Typing (#128)
* fix: CI yml name * add: mypy configuraion * add: type annotation to waybackpy modules * add: type annotation to test modules * fix: mypy command * add: types-requests to dev deps * fix: disable max-line-length * fix: move pytest.ini into setup.cfg * add: urllib3 to deps * fix: Retry (ref: https://github.com/python/typeshed/issues/6893) * fix: f-string * fix: shorten long lines * add: staticmethod decorator to no-self-use methods * fix: str(headers)->headers_str * fix: error message * fix: revert "str(headers)->headers_str" and ignore assignment CaseInsensitiveDict with str * fix: mypy error
This commit is contained in:
@@ -3,6 +3,7 @@ import os
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
from typing import Generator, List, Optional
|
||||
|
||||
import click
|
||||
import requests
|
||||
@@ -24,7 +25,7 @@ from .wrapper import Url
|
||||
"--user-agent",
|
||||
"--user_agent",
|
||||
default=DEFAULT_USER_AGENT,
|
||||
help="User agent, default value is '%s'." % DEFAULT_USER_AGENT,
|
||||
help=f"User agent, default value is '{DEFAULT_USER_AGENT}'.",
|
||||
)
|
||||
@click.option("-v", "--version", is_flag=True, default=False, help="waybackpy version.")
|
||||
@click.option(
|
||||
@@ -163,34 +164,34 @@ from .wrapper import Url
|
||||
+ "will be printed.",
|
||||
)
|
||||
def main(
|
||||
url,
|
||||
user_agent,
|
||||
version,
|
||||
license,
|
||||
newest,
|
||||
oldest,
|
||||
json,
|
||||
near,
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
hour,
|
||||
minute,
|
||||
save,
|
||||
headers,
|
||||
known_urls,
|
||||
subdomain,
|
||||
file,
|
||||
cdx,
|
||||
start_timestamp,
|
||||
end_timestamp,
|
||||
filter,
|
||||
match_type,
|
||||
gzip,
|
||||
collapse,
|
||||
limit,
|
||||
cdx_print,
|
||||
):
|
||||
url: Optional[str],
|
||||
user_agent: str,
|
||||
version: bool,
|
||||
license: bool,
|
||||
newest: bool,
|
||||
oldest: bool,
|
||||
json: bool,
|
||||
near: bool,
|
||||
year: Optional[int],
|
||||
month: Optional[int],
|
||||
day: Optional[int],
|
||||
hour: Optional[int],
|
||||
minute: Optional[int],
|
||||
save: bool,
|
||||
headers: bool,
|
||||
known_urls: bool,
|
||||
subdomain: bool,
|
||||
file: bool,
|
||||
cdx: bool,
|
||||
start_timestamp: Optional[str],
|
||||
end_timestamp: Optional[str],
|
||||
filter: List[str],
|
||||
match_type: Optional[str],
|
||||
gzip: Optional[str],
|
||||
collapse: List[str],
|
||||
limit: Optional[str],
|
||||
cdx_print: List[str],
|
||||
) -> None:
|
||||
"""\b
|
||||
_ _
|
||||
| | | |
|
||||
@@ -214,7 +215,7 @@ def main(
|
||||
"""
|
||||
|
||||
if version:
|
||||
click.echo("waybackpy version %s" % __version__)
|
||||
click.echo(f"waybackpy version {__version__}")
|
||||
return
|
||||
|
||||
if license:
|
||||
@@ -240,11 +241,14 @@ def main(
|
||||
and not cdx
|
||||
):
|
||||
click.echo(
|
||||
"Only URL passed, but did not specify what to do with the URL. Use --help flag for help using waybackpy."
|
||||
"Only URL passed, but did not specify what to do with the URL. "
|
||||
"Use --help flag for help using waybackpy."
|
||||
)
|
||||
return
|
||||
|
||||
def echo_availability_api(availability_api_instance):
|
||||
def echo_availability_api(
|
||||
availability_api_instance: WaybackMachineAvailabilityAPI,
|
||||
) -> None:
|
||||
click.echo("Archive URL:")
|
||||
if not availability_api_instance.archive_url:
|
||||
archive_url = (
|
||||
@@ -295,13 +299,14 @@ def main(
|
||||
click.echo(save_api.headers)
|
||||
return
|
||||
|
||||
def save_urls_on_file(url_gen):
|
||||
def save_urls_on_file(url_gen: Generator[str, None, None]) -> None:
|
||||
domain = None
|
||||
sys_random = random.SystemRandom()
|
||||
uid = "".join(
|
||||
sys_random.choice(string.ascii_lowercase + string.digits) for _ in range(6)
|
||||
)
|
||||
url_count = 0
|
||||
file_name = None
|
||||
|
||||
for url in url_gen:
|
||||
url_count += 1
|
||||
@@ -310,25 +315,21 @@ def main(
|
||||
|
||||
domain = "domain-unknown"
|
||||
|
||||
if match:
|
||||
if match is not None:
|
||||
domain = match.group(1)
|
||||
|
||||
file_name = "{domain}-urls-{uid}.txt".format(domain=domain, uid=uid)
|
||||
file_name = f"{domain}-urls-{uid}.txt"
|
||||
file_path = os.path.join(os.getcwd(), file_name)
|
||||
if not os.path.isfile(file_path):
|
||||
open(file_path, "w+").close()
|
||||
|
||||
with open(file_path, "a") as f:
|
||||
f.write("{url}\n".format(url=url))
|
||||
with open(file_path, "a") as f:
|
||||
f.write(f"{url}\n")
|
||||
|
||||
click.echo(url)
|
||||
|
||||
if url_count > 0:
|
||||
click.echo(
|
||||
"\n\n'{file_name}' saved in current working directory".format(
|
||||
file_name=file_name
|
||||
)
|
||||
)
|
||||
if url_count > 0 or file_name is not None:
|
||||
click.echo(f"\n\n'{file_name}' saved in current working directory")
|
||||
else:
|
||||
click.echo("No known URLs found. Please try a diffrent input!")
|
||||
|
||||
|
Reference in New Issue
Block a user