lint
This commit is contained in:
parent
6b3b2e2a7d
commit
1a81eb97fb
@ -48,7 +48,8 @@ def _known_urls(obj, args):
|
|||||||
if m:
|
if m:
|
||||||
domain = m.group(1)
|
domain = m.group(1)
|
||||||
else:
|
else:
|
||||||
domain = "waybackpy-known"
|
domain = "domain-unknown"
|
||||||
|
|
||||||
dir_path = os.path.abspath(os.getcwd())
|
dir_path = os.path.abspath(os.getcwd())
|
||||||
file_name = dir_path + "/%s-%d-urls.txt" % (domain, total_urls)
|
file_name = dir_path + "/%s-%d-urls.txt" % (domain, total_urls)
|
||||||
text = "\n".join(url_list) + "\n"
|
text = "\n".join(url_list) + "\n"
|
||||||
@ -118,7 +119,7 @@ def parse_args(argv):
|
|||||||
|
|
||||||
userAgentArg = parser.add_argument_group('User Agent')
|
userAgentArg = parser.add_argument_group('User Agent')
|
||||||
userAgentArg.add_argument("--user_agent", "-ua", help="User agent, default user_agent is \"waybackpy python package - https://github.com/akamhy/waybackpy\"")
|
userAgentArg.add_argument("--user_agent", "-ua", help="User agent, default user_agent is \"waybackpy python package - https://github.com/akamhy/waybackpy\"")
|
||||||
|
|
||||||
saveArg = parser.add_argument_group("Create new archive/save URL")
|
saveArg = parser.add_argument_group("Create new archive/save URL")
|
||||||
saveArg.add_argument("--save", "-s", action='store_true', help="Save the URL on the Wayback machine")
|
saveArg.add_argument("--save", "-s", action='store_true', help="Save the URL on the Wayback machine")
|
||||||
|
|
||||||
|
@ -100,7 +100,7 @@ class Url:
|
|||||||
"""Return the source code of the supplied URL.
|
"""Return the source code of the supplied URL.
|
||||||
If encoding is not supplied, it is auto-detected from the response.
|
If encoding is not supplied, it is auto-detected from the response.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if not url:
|
if not url:
|
||||||
url = self._clean_url()
|
url = self._clean_url()
|
||||||
|
|
||||||
@ -190,13 +190,13 @@ class Url:
|
|||||||
|
|
||||||
if subdomain:
|
if subdomain:
|
||||||
request_url = (
|
request_url = (
|
||||||
"https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey"
|
"https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey"
|
||||||
% self._clean_url()
|
% self._clean_url()
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
request_url = (
|
request_url = (
|
||||||
"http://web.archive.org/cdx/search/cdx?url=%s/*&output=json&fl=original&collapse=urlkey"
|
"http://web.archive.org/cdx/search/cdx?url=%s/*&output=json&fl=original&collapse=urlkey"
|
||||||
% self._clean_url()
|
% self._clean_url()
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -213,7 +213,7 @@ class Url:
|
|||||||
for url in url_list:
|
for url in url_list:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
urlopen(url)
|
urlopen(url) # nosec
|
||||||
except:
|
except:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user