add docstrings and lint
This commit is contained in:
@@ -1,3 +1,7 @@
|
|||||||
|
"""
|
||||||
|
Module that makes waybackpy a CLI tool.
|
||||||
|
"""
|
||||||
|
|
||||||
import json as JSON
|
import json as JSON
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
@@ -19,7 +23,10 @@ from .wrapper import Url
|
|||||||
def echo_availability_api(
|
def echo_availability_api(
|
||||||
availability_api_instance: WaybackMachineAvailabilityAPI, json: bool
|
availability_api_instance: WaybackMachineAvailabilityAPI, json: bool
|
||||||
) -> None:
|
) -> None:
|
||||||
click.echo("Archive URL:")
|
"""
|
||||||
|
Output availability API depending functions.
|
||||||
|
Near, oldest and newest output by this method.
|
||||||
|
"""
|
||||||
if not availability_api_instance.archive_url:
|
if not availability_api_instance.archive_url:
|
||||||
archive_url = (
|
archive_url = (
|
||||||
"NO ARCHIVE FOUND - The requested URL is probably "
|
"NO ARCHIVE FOUND - The requested URL is probably "
|
||||||
@@ -29,6 +36,7 @@ def echo_availability_api(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
archive_url = availability_api_instance.archive_url
|
archive_url = availability_api_instance.archive_url
|
||||||
|
click.echo("Archive URL:")
|
||||||
click.echo(archive_url)
|
click.echo(archive_url)
|
||||||
if json:
|
if json:
|
||||||
click.echo("JSON response:")
|
click.echo("JSON response:")
|
||||||
@@ -36,6 +44,10 @@ def echo_availability_api(
|
|||||||
|
|
||||||
|
|
||||||
def save_urls_on_file(url_gen: Generator[str, None, None]) -> None:
|
def save_urls_on_file(url_gen: Generator[str, None, None]) -> None:
|
||||||
|
"""
|
||||||
|
Save output of CDX API on file.
|
||||||
|
Mainly here because of backwards compatibility.
|
||||||
|
"""
|
||||||
domain = None
|
domain = None
|
||||||
sys_random = random.SystemRandom()
|
sys_random = random.SystemRandom()
|
||||||
uid = "".join(
|
uid = "".join(
|
||||||
@@ -51,8 +63,8 @@ def save_urls_on_file(url_gen: Generator[str, None, None]) -> None:
|
|||||||
domain = "domain-unknown" if match is None else match.group(1)
|
domain = "domain-unknown" if match is None else match.group(1)
|
||||||
file_name = f"{domain}-urls-{uid}.txt"
|
file_name = f"{domain}-urls-{uid}.txt"
|
||||||
file_path = os.path.join(os.getcwd(), file_name)
|
file_path = os.path.join(os.getcwd(), file_name)
|
||||||
with open(file_path, "a") as f:
|
with open(file_path, "a") as file:
|
||||||
f.write(f"{url}\n")
|
file.write(f"{url}\n")
|
||||||
|
|
||||||
click.echo(url)
|
click.echo(url)
|
||||||
|
|
||||||
@@ -269,6 +281,7 @@ def main( # pylint: disable=no-value-for-parameter
|
|||||||
"""
|
"""
|
||||||
if version:
|
if version:
|
||||||
click.echo(f"waybackpy version {__version__}")
|
click.echo(f"waybackpy version {__version__}")
|
||||||
|
|
||||||
elif show_license:
|
elif show_license:
|
||||||
click.echo(
|
click.echo(
|
||||||
requests.get(
|
requests.get(
|
||||||
@@ -277,6 +290,7 @@ def main( # pylint: disable=no-value-for-parameter
|
|||||||
)
|
)
|
||||||
elif url is None:
|
elif url is None:
|
||||||
click.echo("No URL detected. Please provide an URL.", err=True)
|
click.echo("No URL detected. Please provide an URL.", err=True)
|
||||||
|
|
||||||
elif (
|
elif (
|
||||||
not version
|
not version
|
||||||
and not oldest
|
and not oldest
|
||||||
@@ -291,14 +305,17 @@ def main( # pylint: disable=no-value-for-parameter
|
|||||||
"Use --help flag for help using waybackpy.",
|
"Use --help flag for help using waybackpy.",
|
||||||
err=True,
|
err=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
elif oldest:
|
elif oldest:
|
||||||
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
||||||
availability_api.oldest()
|
availability_api.oldest()
|
||||||
echo_availability_api(availability_api, json)
|
echo_availability_api(availability_api, json)
|
||||||
|
|
||||||
elif newest:
|
elif newest:
|
||||||
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
||||||
availability_api.newest()
|
availability_api.newest()
|
||||||
echo_availability_api(availability_api, json)
|
echo_availability_api(availability_api, json)
|
||||||
|
|
||||||
elif near:
|
elif near:
|
||||||
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
availability_api = WaybackMachineAvailabilityAPI(url, user_agent=user_agent)
|
||||||
near_args = {}
|
near_args = {}
|
||||||
@@ -309,6 +326,7 @@ def main( # pylint: disable=no-value-for-parameter
|
|||||||
near_args[key] = arg
|
near_args[key] = arg
|
||||||
availability_api.near(**near_args)
|
availability_api.near(**near_args)
|
||||||
echo_availability_api(availability_api, json)
|
echo_availability_api(availability_api, json)
|
||||||
|
|
||||||
elif save:
|
elif save:
|
||||||
save_api = WaybackMachineSaveAPI(url, user_agent=user_agent)
|
save_api = WaybackMachineSaveAPI(url, user_agent=user_agent)
|
||||||
save_api.save()
|
save_api.save()
|
||||||
@@ -319,15 +337,17 @@ def main( # pylint: disable=no-value-for-parameter
|
|||||||
if headers:
|
if headers:
|
||||||
click.echo("Save API headers:")
|
click.echo("Save API headers:")
|
||||||
click.echo(save_api.headers)
|
click.echo(save_api.headers)
|
||||||
|
|
||||||
elif known_urls:
|
elif known_urls:
|
||||||
wayback = Url(url, user_agent)
|
wayback = Url(url, user_agent)
|
||||||
url_gen = wayback.known_urls(subdomain=subdomain)
|
url_gen = wayback.known_urls(subdomain=subdomain)
|
||||||
|
|
||||||
if file:
|
if file:
|
||||||
return save_urls_on_file(url_gen)
|
return save_urls_on_file(url_gen)
|
||||||
else:
|
|
||||||
for url in url_gen:
|
for url in url_gen:
|
||||||
click.echo(url)
|
click.echo(url)
|
||||||
|
|
||||||
elif cdx:
|
elif cdx:
|
||||||
filters = list(cdx_filter)
|
filters = list(cdx_filter)
|
||||||
collapses = list(collapse)
|
collapses = list(collapse)
|
||||||
|
Reference in New Issue
Block a user