Coverge improvements (#22)
* Update cli.py * improved tests * chnages for proper testing * Type check using isinstance * Replace elifs with if when used after return * twitter.com --> www.ibm.com * fix typo * test archive urll parser and dunders * Update test_wrapper.py
This commit is contained in:
Akash
GitHub
parent
4dcda94cb0
commit
56116551ac
@ -7,37 +7,91 @@ import argparse
|
||||
sys.path.append("..")
|
||||
import waybackpy.cli as cli # noqa: E402
|
||||
from waybackpy.wrapper import Url # noqa: E402
|
||||
from waybackpy.__version__ import __version__
|
||||
|
||||
codecov_python = False
|
||||
if sys.version_info > (3, 7):
|
||||
def test_save():
|
||||
obj = Url("https://pypi.org/user/akamhy/", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9")
|
||||
cli._save(obj)
|
||||
else:
|
||||
pass
|
||||
codecov_python = True
|
||||
|
||||
def test_get():
|
||||
args = argparse.Namespace(get='oldest')
|
||||
obj = Url("https://pypi.org/user/akamhy/", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9")
|
||||
cli._get(obj, args)
|
||||
args = argparse.Namespace(get='newest')
|
||||
cli._get(obj, args)
|
||||
args = argparse.Namespace(get='url')
|
||||
cli._get(obj, args)
|
||||
if sys.version_info > (3, 7):
|
||||
args = argparse.Namespace(get='save')
|
||||
cli._get(obj, args)
|
||||
else:
|
||||
pass
|
||||
# Namespace(day=None, get=None, hour=None, minute=None, month=None, near=False,
|
||||
# newest=False, oldest=False, save=False, total=False, url=None, user_agent=None, version=False, year=None)
|
||||
|
||||
if codecov_python:
|
||||
def test_save():
|
||||
args = argparse.Namespace(user_agent=None, url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=True, newest=False, near=False, get=None)
|
||||
reply = cli.args_handler(args)
|
||||
assert "pypi.org/user/akamhy" in reply
|
||||
|
||||
def test_oldest():
|
||||
obj = Url("https://pypi.org/user/akamhy/", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9")
|
||||
cli._oldest(obj)
|
||||
args = argparse.Namespace(user_agent=None, url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=True, save=False, newest=False, near=False, get=None)
|
||||
reply = cli.args_handler(args)
|
||||
assert "pypi.org/user/akamhy" in reply
|
||||
|
||||
def test_newest():
|
||||
obj = Url("https://pypi.org/user/akamhy/", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9")
|
||||
cli._newest(obj)
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=True, near=False, get=None)
|
||||
reply = cli.args_handler(args)
|
||||
assert "pypi.org/user/akamhy" in reply
|
||||
|
||||
def test_total_archives():
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=True, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get=None)
|
||||
reply = cli.args_handler(args)
|
||||
assert isinstance(reply, int)
|
||||
|
||||
def test_near():
|
||||
args = argparse.Namespace(year=2020, month=6, day=1, hour=1, minute=1)
|
||||
obj = Url("https://pypi.org/user/akamhy/", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9")
|
||||
cli._near(obj, args)
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=True, get=None, year=2020, month=7, day=15, hour=1, minute=1)
|
||||
reply = cli.args_handler(args)
|
||||
assert "202007" in reply
|
||||
|
||||
|
||||
def test_get():
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get="url")
|
||||
reply = cli.args_handler(args)
|
||||
assert "waybackpy" in reply
|
||||
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get="oldest")
|
||||
reply = cli.args_handler(args)
|
||||
assert "waybackpy" in reply
|
||||
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get="newest")
|
||||
reply = cli.args_handler(args)
|
||||
assert "waybackpy" in reply
|
||||
|
||||
if codecov_python:
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get="save")
|
||||
reply = cli.args_handler(args)
|
||||
assert "waybackpy" in reply
|
||||
|
||||
args = argparse.Namespace(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 \
|
||||
(KHTML, like Gecko) Version/8.0.8 Safari/600.8.9", url="https://pypi.org/user/akamhy/", total=False, version=False,
|
||||
oldest=False, save=False, newest=False, near=False, get="BullShit")
|
||||
reply = cli.args_handler(args)
|
||||
assert "get the source code of the" in reply
|
||||
|
||||
def test_args_handler():
|
||||
args = argparse.Namespace(version=True)
|
||||
reply = cli.args_handler(args)
|
||||
assert __version__ == reply
|
||||
|
||||
args = argparse.Namespace(url=None, version=False)
|
||||
reply = cli.args_handler(args)
|
||||
assert "Specify an URL" in reply
|
||||
|
||||
def test_main():
|
||||
# This also tests the parse_args method in cli.py
|
||||
cli.main(['temp.py', '--version'])
|
||||
|
||||
@ -23,6 +23,21 @@ def test_clean_url():
|
||||
test_result = target._clean_url()
|
||||
assert answer == test_result
|
||||
|
||||
def test_dunders():
|
||||
url = "https://en.wikipedia.org/wiki/Network_security"
|
||||
user_agent = "UA"
|
||||
target = waybackpy.Url(url, user_agent)
|
||||
assert "waybackpy.Url(url=%s, user_agent=%s)" % (url, user_agent) == repr(target)
|
||||
assert len(target) == len(url)
|
||||
assert str(target) == url
|
||||
|
||||
def test_archive_url_parser():
|
||||
request_url = "https://amazon.com"
|
||||
hdr = {"User-Agent": user_agent} # nosec
|
||||
req = Request(request_url, headers=hdr) # nosec
|
||||
header = waybackpy._get_response(req).headers
|
||||
with pytest.raises(Exception):
|
||||
waybackpy._archive_url_parser(header)
|
||||
|
||||
def test_url_check():
|
||||
broken_url = "http://wwwgooglecom/"
|
||||
@ -40,7 +55,7 @@ def test_save():
|
||||
"commons.wikimedia.org",
|
||||
"www.wiktionary.org",
|
||||
"www.w3schools.com",
|
||||
"twitter.com",
|
||||
"www.ibm.com",
|
||||
]
|
||||
x = random.randint(0, len(url_list) - 1)
|
||||
url1 = url_list[x]
|
||||
|
||||
@ -1,20 +1,21 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import argparse
|
||||
from waybackpy.wrapper import Url
|
||||
from waybackpy.__version__ import __version__
|
||||
|
||||
def _save(obj):
|
||||
print(obj.save())
|
||||
return (obj.save())
|
||||
|
||||
def _oldest(obj):
|
||||
print(obj.oldest())
|
||||
return (obj.oldest())
|
||||
|
||||
def _newest(obj):
|
||||
print(obj.newest())
|
||||
return (obj.newest())
|
||||
|
||||
def _total_archives(obj):
|
||||
print(obj.total_archives())
|
||||
return (obj.total_archives())
|
||||
|
||||
def _near(obj, args):
|
||||
_near_args = {}
|
||||
@ -28,29 +29,54 @@ def _near(obj, args):
|
||||
_near_args["hour"] = args.hour
|
||||
if args.minute:
|
||||
_near_args["minute"] = args.minute
|
||||
print(obj.near(**_near_args))
|
||||
return (obj.near(**_near_args))
|
||||
|
||||
def _get(obj, args):
|
||||
if args.get.lower() == "url":
|
||||
print(obj.get())
|
||||
return (obj.get())
|
||||
|
||||
elif args.get.lower() == "oldest":
|
||||
print(obj.get(obj.oldest()))
|
||||
if args.get.lower() == "oldest":
|
||||
return (obj.get(obj.oldest()))
|
||||
|
||||
elif args.get.lower() == "latest" or args.get.lower() == "newest":
|
||||
print(obj.get(obj.newest()))
|
||||
if args.get.lower() == "latest" or args.get.lower() == "newest":
|
||||
return (obj.get(obj.newest()))
|
||||
|
||||
elif args.get.lower() == "save":
|
||||
print(obj.get(obj.save()))
|
||||
if args.get.lower() == "save":
|
||||
return (obj.get(obj.save()))
|
||||
|
||||
else:
|
||||
print("Use get as \"--get 'source'\", 'source' can be one of the followings: \
|
||||
return ("Use get as \"--get 'source'\", 'source' can be one of the followings: \
|
||||
\n1) url - get the source code of the url specified using --url/-u.\
|
||||
\n2) oldest - get the source code of the oldest archive for the supplied url.\
|
||||
\n3) newest - get the source code of the newest archive for the supplied url.\
|
||||
\n4) save - Create a new archive and get the source code of this new archive for the supplied url.")
|
||||
|
||||
def main():
|
||||
def args_handler(args):
|
||||
if args.version:
|
||||
return (__version__)
|
||||
|
||||
if not args.url:
|
||||
return ("Specify an URL. See --help for help using waybackpy.")
|
||||
|
||||
if args.user_agent:
|
||||
obj = Url(args.url, args.user_agent)
|
||||
else:
|
||||
obj = Url(args.url)
|
||||
|
||||
if args.save:
|
||||
return _save(obj)
|
||||
if args.oldest:
|
||||
return _oldest(obj)
|
||||
if args.newest:
|
||||
return _newest(obj)
|
||||
if args.total:
|
||||
return _total_archives(obj)
|
||||
if args.near:
|
||||
return _near(obj, args)
|
||||
if args.get:
|
||||
return _get(obj, args)
|
||||
return ("Usage: waybackpy --url [URL] --user_agent [USER AGENT] [OPTIONS]. See --help for help using waybackpy.")
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-u", "--url", help="URL on which Wayback machine operations would occur.")
|
||||
parser.add_argument("-ua", "--user_agent", help="User agent, default user_agent is \"waybackpy python package - https://github.com/akamhy/waybackpy\".")
|
||||
@ -60,45 +86,18 @@ def main():
|
||||
parser.add_argument("-t", "--total", action='store_true', help="Total number of archives for the specified URL.")
|
||||
parser.add_argument("-g", "--get", help="Prints the source code of the supplied url. Use '--get help' for extended usage.")
|
||||
parser.add_argument("-v", "--version", action='store_true', help="Prints the waybackpy version.")
|
||||
|
||||
parser.add_argument("-N", "--near", action='store_true', help="Latest/Newest archive for the specified URL.")
|
||||
parser.add_argument("-Y", "--year", type=int, help="Year in integer. For use with --near.")
|
||||
parser.add_argument("-M", "--month", type=int, help="Month in integer. For use with --near.")
|
||||
parser.add_argument("-D", "--day", type=int, help="Day in integer. For use with --near.")
|
||||
parser.add_argument("-H", "--hour", type=int, help="Hour in integer. For use with --near.")
|
||||
parser.add_argument("-MIN", "--minute", type=int, help="Minute in integer. For use with --near.")
|
||||
return parser.parse_args(argv[1:])
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.version:
|
||||
print(__version__)
|
||||
return
|
||||
|
||||
if not args.url:
|
||||
print("Specify an URL. See --help")
|
||||
return
|
||||
|
||||
# create the object with or without the user_agent
|
||||
if args.user_agent:
|
||||
obj = Url(args.url, args.user_agent)
|
||||
else:
|
||||
obj = Url(args.url)
|
||||
|
||||
if args.save:
|
||||
_save(obj)
|
||||
elif args.oldest:
|
||||
_oldest(obj)
|
||||
elif args.newest:
|
||||
_newest(obj)
|
||||
elif args.total:
|
||||
_total_archives(obj)
|
||||
elif args.near:
|
||||
_near(obj, args)
|
||||
elif args.get:
|
||||
_get(obj, args)
|
||||
else:
|
||||
print("Usage: waybackpy --url [URL] --user_agent [USER AGENT] [OPTIONS]. See --help")
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
output = args_handler(args)
|
||||
print(output)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(main(sys.argv))
|
||||
|
||||
@ -51,8 +51,7 @@ def _get_response(req):
|
||||
try:
|
||||
response = urlopen(req) # nosec
|
||||
except Exception as e:
|
||||
exc = WaybackError("Error while retrieving %s" % req.full_ur
|
||||
)
|
||||
exc = WaybackError("Error while retrieving %s" % req.full_url)
|
||||
exc.__cause__ = e
|
||||
raise exc
|
||||
return response
|
||||
|
||||
Loading…
Reference in New Issue
Block a user