Typing (#128)
* fix: CI yml name * add: mypy configuraion * add: type annotation to waybackpy modules * add: type annotation to test modules * fix: mypy command * add: types-requests to dev deps * fix: disable max-line-length * fix: move pytest.ini into setup.cfg * add: urllib3 to deps * fix: Retry (ref: https://github.com/python/typeshed/issues/6893) * fix: f-string * fix: shorten long lines * add: staticmethod decorator to no-self-use methods * fix: str(headers)->headers_str * fix: error message * fix: revert "str(headers)->headers_str" and ignore assignment CaseInsensitiveDict with str * fix: mypy error
This commit is contained in:
@ -12,33 +12,42 @@ from waybackpy.exceptions import (
|
||||
|
||||
now = datetime.utcnow()
|
||||
url = "https://example.com/"
|
||||
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def rndstr(n):
|
||||
def rndstr(n: int) -> str:
|
||||
return "".join(
|
||||
random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
|
||||
)
|
||||
|
||||
|
||||
def test_oldest():
|
||||
def test_oldest() -> None:
|
||||
"""
|
||||
Test the oldest archive of Google.com and also checks the attributes.
|
||||
"""
|
||||
url = "https://example.com/"
|
||||
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
|
||||
)
|
||||
availability_api = WaybackMachineAvailabilityAPI(url, user_agent)
|
||||
oldest = availability_api.oldest()
|
||||
oldest_archive_url = oldest.archive_url
|
||||
assert "2002" in oldest_archive_url
|
||||
oldest_timestamp = oldest.timestamp()
|
||||
assert abs(oldest_timestamp - now) > timedelta(days=7000) # More than 19 years
|
||||
assert availability_api.JSON["archived_snapshots"]["closest"]["available"] is True
|
||||
assert (
|
||||
availability_api.JSON is not None
|
||||
and availability_api.JSON["archived_snapshots"]["closest"]["available"] is True
|
||||
)
|
||||
assert repr(oldest).find("example.com") != -1
|
||||
assert "2002" in str(oldest)
|
||||
|
||||
|
||||
def test_newest():
|
||||
def test_newest() -> None:
|
||||
"""
|
||||
Assuming that the recent most Google Archive was made no more earlier than
|
||||
last one day which is 86400 seconds.
|
||||
@ -54,16 +63,17 @@ def test_newest():
|
||||
assert abs(newest_timestamp - now) < timedelta(seconds=86400 * 3)
|
||||
|
||||
|
||||
def test_invalid_json():
|
||||
def test_invalid_json() -> None:
|
||||
"""
|
||||
When the API is malfunctioning or we don't pass a URL it may return invalid JSON data.
|
||||
When the API is malfunctioning or we don't pass a URL,
|
||||
it may return invalid JSON data.
|
||||
"""
|
||||
with pytest.raises(InvalidJSONInAvailabilityAPIResponse):
|
||||
availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent)
|
||||
_ = availability_api.archive_url
|
||||
|
||||
|
||||
def test_no_archive():
|
||||
def test_no_archive() -> None:
|
||||
"""
|
||||
ArchiveNotInAvailabilityAPIResponse may be raised if Wayback Machine did not
|
||||
replied with the archive despite the fact that we know the site has million
|
||||
@ -74,12 +84,12 @@ def test_no_archive():
|
||||
"""
|
||||
with pytest.raises(ArchiveNotInAvailabilityAPIResponse):
|
||||
availability_api = WaybackMachineAvailabilityAPI(
|
||||
url="https://%s.cn" % rndstr(30), user_agent=user_agent
|
||||
url=f"https://{rndstr(30)}.cn", user_agent=user_agent
|
||||
)
|
||||
_ = availability_api.archive_url
|
||||
|
||||
|
||||
def test_no_api_call_str_repr():
|
||||
def test_no_api_call_str_repr() -> None:
|
||||
"""
|
||||
Some entitled users maybe want to see what is the string representation
|
||||
if they don’t make any API requests.
|
||||
@ -87,17 +97,17 @@ def test_no_api_call_str_repr():
|
||||
str() must not return None so we return ""
|
||||
"""
|
||||
availability_api = WaybackMachineAvailabilityAPI(
|
||||
url="https://%s.gov" % rndstr(30), user_agent=user_agent
|
||||
url=f"https://{rndstr(30)}.gov", user_agent=user_agent
|
||||
)
|
||||
assert "" == str(availability_api)
|
||||
|
||||
|
||||
def test_no_call_timestamp():
|
||||
def test_no_call_timestamp() -> None:
|
||||
"""
|
||||
If no API requests were made the bound timestamp() method returns
|
||||
the datetime.max as a default value.
|
||||
"""
|
||||
availability_api = WaybackMachineAvailabilityAPI(
|
||||
url="https://%s.in" % rndstr(30), user_agent=user_agent
|
||||
url=f"https://{rndstr(30)}.in", user_agent=user_agent
|
||||
)
|
||||
assert datetime.max == availability_api.timestamp()
|
||||
|
@ -1,8 +1,11 @@
|
||||
from waybackpy.cdx_api import WaybackMachineCDXServerAPI
|
||||
|
||||
|
||||
def test_a():
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
def test_a() -> None:
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
url = "https://twitter.com/jack"
|
||||
|
||||
wayback = WaybackMachineCDXServerAPI(
|
||||
@ -21,8 +24,11 @@ def test_a():
|
||||
assert snapshot.timestamp.startswith("2010")
|
||||
|
||||
|
||||
def test_b():
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
def test_b() -> None:
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
url = "https://www.google.com"
|
||||
|
||||
wayback = WaybackMachineCDXServerAPI(
|
||||
|
@ -3,8 +3,11 @@ from datetime import datetime
|
||||
from waybackpy.cdx_snapshot import CDXSnapshot
|
||||
|
||||
|
||||
def test_CDXSnapshot():
|
||||
sample_input = "org,archive)/ 20080126045828 http://github.com text/html 200 Q4YULN754FHV2U6Q5JUT6Q2P57WEWNNY 1415"
|
||||
def test_CDXSnapshot() -> None:
|
||||
sample_input = (
|
||||
"org,archive)/ 20080126045828 http://github.com "
|
||||
"text/html 200 Q4YULN754FHV2U6Q5JUT6Q2P57WEWNNY 1415"
|
||||
)
|
||||
prop_values = sample_input.split(" ")
|
||||
properties = {}
|
||||
(
|
||||
|
@ -1,3 +1,5 @@
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
from waybackpy.cdx_utils import (
|
||||
@ -11,15 +13,18 @@ from waybackpy.cdx_utils import (
|
||||
from waybackpy.exceptions import WaybackError
|
||||
|
||||
|
||||
def test_get_total_pages():
|
||||
def test_get_total_pages() -> None:
|
||||
url = "twitter.com"
|
||||
user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.0.2 Safari/605.1.15"
|
||||
)
|
||||
assert get_total_pages(url=url, user_agent=user_agent) >= 56
|
||||
|
||||
|
||||
def test_full_url():
|
||||
params = {}
|
||||
def test_full_url() -> None:
|
||||
endpoint = "https://web.archive.org/cdx/search/cdx"
|
||||
params: Dict[str, Any] = {}
|
||||
assert endpoint == full_url(endpoint, params)
|
||||
|
||||
params = {"a": "1"}
|
||||
@ -39,36 +44,36 @@ def test_full_url():
|
||||
)
|
||||
|
||||
|
||||
def test_get_response():
|
||||
def test_get_response() -> None:
|
||||
url = "https://github.com"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0"
|
||||
)
|
||||
headers = {"User-Agent": "%s" % user_agent}
|
||||
headers = {"User-Agent": str(user_agent)}
|
||||
response = get_response(url, headers=headers)
|
||||
assert response.status_code == 200
|
||||
assert not isinstance(response, Exception) and response.status_code == 200
|
||||
|
||||
url = "http/wwhfhfvhvjhmom"
|
||||
with pytest.raises(WaybackError):
|
||||
get_response(url, headers=headers)
|
||||
|
||||
|
||||
def test_check_filters():
|
||||
filters = []
|
||||
def test_check_filters() -> None:
|
||||
filters: List[str] = []
|
||||
check_filters(filters)
|
||||
|
||||
filters = ["statuscode:200", "timestamp:20215678901234", "original:https://url.com"]
|
||||
check_filters(filters)
|
||||
|
||||
with pytest.raises(WaybackError):
|
||||
check_filters("not-list")
|
||||
check_filters("not-list") # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(WaybackError):
|
||||
check_filters(["invalid"])
|
||||
|
||||
|
||||
def test_check_collapses():
|
||||
collapses = []
|
||||
def test_check_collapses() -> None:
|
||||
collapses: List[str] = []
|
||||
check_collapses(collapses)
|
||||
|
||||
collapses = ["timestamp:10"]
|
||||
@ -77,7 +82,7 @@ def test_check_collapses():
|
||||
collapses = ["urlkey"]
|
||||
check_collapses(collapses)
|
||||
|
||||
collapses = "urlkey" # NOT LIST
|
||||
collapses = "urlkey" # type: ignore[assignment]
|
||||
with pytest.raises(WaybackError):
|
||||
check_collapses(collapses)
|
||||
|
||||
@ -86,11 +91,11 @@ def test_check_collapses():
|
||||
check_collapses(collapses)
|
||||
|
||||
|
||||
def test_check_match_type():
|
||||
assert check_match_type(None, "url") is None
|
||||
def test_check_match_type() -> None:
|
||||
assert check_match_type(None, "url")
|
||||
match_type = "exact"
|
||||
url = "test_url"
|
||||
assert check_match_type(match_type, url) is None
|
||||
assert check_match_type(match_type, url)
|
||||
|
||||
url = "has * in it"
|
||||
with pytest.raises(WaybackError):
|
||||
|
@ -2,22 +2,27 @@ import random
|
||||
import string
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
from waybackpy.exceptions import MaximumSaveRetriesExceeded
|
||||
from waybackpy.save_api import WaybackMachineSaveAPI
|
||||
|
||||
|
||||
def rndstr(n):
|
||||
def rndstr(n: int) -> str:
|
||||
return "".join(
|
||||
random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
|
||||
)
|
||||
|
||||
|
||||
def test_save():
|
||||
def test_save() -> None:
|
||||
url = "https://github.com/akamhy/waybackpy"
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
save_api.save()
|
||||
archive_url = save_api.archive_url
|
||||
@ -31,15 +36,18 @@ def test_save():
|
||||
assert isinstance(save_api.timestamp(), datetime)
|
||||
|
||||
|
||||
def test_max_redirect_exceeded():
|
||||
def test_max_redirect_exceeded() -> None:
|
||||
with pytest.raises(MaximumSaveRetriesExceeded):
|
||||
url = "https://%s.gov" % rndstr
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
url = f"https://{rndstr}.gov"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent, max_tries=3)
|
||||
save_api.save()
|
||||
|
||||
|
||||
def test_sleep():
|
||||
def test_sleep() -> None:
|
||||
"""
|
||||
sleeping is actually very important for SaveAPI
|
||||
interface stability.
|
||||
@ -47,7 +55,10 @@ def test_sleep():
|
||||
is as intended.
|
||||
"""
|
||||
url = "https://example.com"
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
s_time = int(time.time())
|
||||
save_api.sleep(6) # multiple of 3 sleep for 10 seconds
|
||||
@ -60,76 +71,150 @@ def test_sleep():
|
||||
assert (e_time - s_time) >= 5
|
||||
|
||||
|
||||
def test_timestamp():
|
||||
def test_timestamp() -> None:
|
||||
url = "https://example.com"
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
now = datetime.utcnow()
|
||||
save_api._archive_url = (
|
||||
"https://web.archive.org/web/%s/" % now.strftime("%Y%m%d%H%M%S") + url
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
|
||||
save_api._archive_url = f"https://web.archive.org/web/{now}/{url}/"
|
||||
save_api.timestamp()
|
||||
assert save_api.cached_save is False
|
||||
save_api._archive_url = "https://web.archive.org/web/%s/" % "20100124063622" + url
|
||||
now = "20100124063622"
|
||||
save_api._archive_url = f"https://web.archive.org/web/{now}/{url}/"
|
||||
save_api.timestamp()
|
||||
assert save_api.cached_save is True
|
||||
|
||||
|
||||
def test_archive_url_parser():
|
||||
def test_archive_url_parser() -> None:
|
||||
"""
|
||||
Testing three regex for matches and also tests the response URL.
|
||||
"""
|
||||
url = "https://example.com"
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
|
||||
save_api.headers = """
|
||||
START
|
||||
Content-Location: /web/20201126185327/https://www.scribbr.com/citing-sources/et-al
|
||||
END
|
||||
"""
|
||||
|
||||
assert (
|
||||
save_api.archive_url_parser()
|
||||
== "https://web.archive.org/web/20201126185327/https://www.scribbr.com/citing-sources/et-al"
|
||||
h = (
|
||||
"\nSTART\nContent-Location: "
|
||||
"/web/20201126185327/https://www.scribbr.com/citing-sources/et-al"
|
||||
"\nEND\n"
|
||||
)
|
||||
save_api.headers = h # type: ignore[assignment]
|
||||
|
||||
save_api.headers = """
|
||||
{'Server': 'nginx/1.15.8', 'Date': 'Sat, 02 Jan 2021 09:40:25 GMT', 'Content-Type': 'text/html; charset=UTF-8', 'Transfer-Encoding': 'chunked', 'Connection': 'keep-alive', 'X-Archive-Orig-Server': 'nginx', 'X-Archive-Orig-Date': 'Sat, 02 Jan 2021 09:40:09 GMT', 'X-Archive-Orig-Transfer-Encoding': 'chunked', 'X-Archive-Orig-Connection': 'keep-alive', 'X-Archive-Orig-Vary': 'Accept-Encoding', 'X-Archive-Orig-Last-Modified': 'Fri, 01 Jan 2021 12:19:00 GMT', 'X-Archive-Orig-Strict-Transport-Security': 'max-age=31536000, max-age=0;', 'X-Archive-Guessed-Content-Type': 'text/html', 'X-Archive-Guessed-Charset': 'utf-8', 'Memento-Datetime': 'Sat, 02 Jan 2021 09:40:09 GMT', 'Link': '<https://www.scribbr.com/citing-sources/et-al/>; rel="original", <https://web.archive.org/web/timemap/link/https://www.scribbr.com/citing-sources/et-al/>; rel="timemap"; type="application/link-format", <https://web.archive.org/web/https://www.scribbr.com/citing-sources/et-al/>; rel="timegate", <https://web.archive.org/web/20200601082911/https://www.scribbr.com/citing-sources/et-al/>; rel="first memento"; datetime="Mon, 01 Jun 2020 08:29:11 GMT", <https://web.archive.org/web/20201126185327/https://www.scribbr.com/citing-sources/et-al/>; rel="prev memento"; datetime="Thu, 26 Nov 2020 18:53:27 GMT", <https://web.archive.org/web/20210102094009/https://www.scribbr.com/citing-sources/et-al/>; rel="memento"; datetime="Sat, 02 Jan 2021 09:40:09 GMT", <https://web.archive.org/web/20210102094009/https://www.scribbr.com/citing-sources/et-al/>; rel="last memento"; datetime="Sat, 02 Jan 2021 09:40:09 GMT"', 'Content-Security-Policy': "default-src 'self' 'unsafe-eval' 'unsafe-inline' data: blob: archive.org web.archive.org analytics.archive.org pragma.archivelab.org", 'X-Archive-Src': 'spn2-20210102092956-wwwb-spn20.us.archive.org-8001.warc.gz', 'Server-Timing': 'captures_list;dur=112.646325, exclusion.robots;dur=0.172010, exclusion.robots.policy;dur=0.158205, RedisCDXSource;dur=2.205932, esindex;dur=0.014647, LoadShardBlock;dur=82.205012, PetaboxLoader3.datanode;dur=70.750239, CDXLines.iter;dur=24.306278, load_resource;dur=26.520179', 'X-App-Server': 'wwwb-app200', 'X-ts': '200', 'X-location': 'All', 'X-Cache-Key': 'httpsweb.archive.org/web/20210102094009/https://www.scribbr.com/citing-sources/et-al/IN', 'X-RL': '0', 'X-Page-Cache': 'MISS', 'X-Archive-Screenname': '0', 'Content-Encoding': 'gzip'}
|
||||
"""
|
||||
|
||||
assert (
|
||||
save_api.archive_url_parser()
|
||||
== "https://web.archive.org/web/20210102094009/https://www.scribbr.com/citing-sources/et-al/"
|
||||
expected_url = (
|
||||
"https://web.archive.org/web/20201126185327/"
|
||||
"https://www.scribbr.com/citing-sources/et-al"
|
||||
)
|
||||
assert save_api.archive_url_parser() == expected_url
|
||||
|
||||
save_api.headers = """
|
||||
START
|
||||
X-Cache-Key: https://web.archive.org/web/20171128185327/https://www.scribbr.com/citing-sources/et-al/US
|
||||
END
|
||||
"""
|
||||
headers = {
|
||||
"Server": "nginx/1.15.8",
|
||||
"Date": "Sat, 02 Jan 2021 09:40:25 GMT",
|
||||
"Content-Type": "text/html; charset=UTF-8",
|
||||
"Transfer-Encoding": "chunked",
|
||||
"Connection": "keep-alive",
|
||||
"X-Archive-Orig-Server": "nginx",
|
||||
"X-Archive-Orig-Date": "Sat, 02 Jan 2021 09:40:09 GMT",
|
||||
"X-Archive-Orig-Transfer-Encoding": "chunked",
|
||||
"X-Archive-Orig-Connection": "keep-alive",
|
||||
"X-Archive-Orig-Vary": "Accept-Encoding",
|
||||
"X-Archive-Orig-Last-Modified": "Fri, 01 Jan 2021 12:19:00 GMT",
|
||||
"X-Archive-Orig-Strict-Transport-Security": "max-age=31536000, max-age=0;",
|
||||
"X-Archive-Guessed-Content-Type": "text/html",
|
||||
"X-Archive-Guessed-Charset": "utf-8",
|
||||
"Memento-Datetime": "Sat, 02 Jan 2021 09:40:09 GMT",
|
||||
"Link": (
|
||||
'<https://www.scribbr.com/citing-sources/et-al/>; rel="original", '
|
||||
"<https://web.archive.org/web/timemap/link/https://www.scribbr.com/"
|
||||
'citing-sources/et-al/>; rel="timemap"; type="application/link-format", '
|
||||
"<https://web.archive.org/web/https://www.scribbr.com/citing-sources/"
|
||||
'et-al/>; rel="timegate", <https://web.archive.org/web/20200601082911/'
|
||||
'https://www.scribbr.com/citing-sources/et-al/>; rel="first memento"; '
|
||||
'datetime="Mon, 01 Jun 2020 08:29:11 GMT", <https://web.archive.org/web/'
|
||||
"20201126185327/https://www.scribbr.com/citing-sources/et-al/>; "
|
||||
'rel="prev memento"; datetime="Thu, 26 Nov 2020 18:53:27 GMT", '
|
||||
"<https://web.archive.org/web/20210102094009/https://www.scribbr.com/"
|
||||
'citing-sources/et-al/>; rel="memento"; datetime="Sat, 02 Jan 2021 '
|
||||
'09:40:09 GMT", <https://web.archive.org/web/20210102094009/'
|
||||
"https://www.scribbr.com/citing-sources/et-al/>; "
|
||||
'rel="last memento"; datetime="Sat, 02 Jan 2021 09:40:09 GMT"'
|
||||
),
|
||||
"Content-Security-Policy": (
|
||||
"default-src 'self' 'unsafe-eval' 'unsafe-inline' "
|
||||
"data: blob: archive.org web.archive.org analytics.archive.org "
|
||||
"pragma.archivelab.org",
|
||||
),
|
||||
"X-Archive-Src": "spn2-20210102092956-wwwb-spn20.us.archive.org-8001.warc.gz",
|
||||
"Server-Timing": (
|
||||
"captures_list;dur=112.646325, exclusion.robots;dur=0.172010, "
|
||||
"exclusion.robots.policy;dur=0.158205, RedisCDXSource;dur=2.205932, "
|
||||
"esindex;dur=0.014647, LoadShardBlock;dur=82.205012, "
|
||||
"PetaboxLoader3.datanode;dur=70.750239, CDXLines.iter;dur=24.306278, "
|
||||
"load_resource;dur=26.520179"
|
||||
),
|
||||
"X-App-Server": "wwwb-app200",
|
||||
"X-ts": "200",
|
||||
"X-location": "All",
|
||||
"X-Cache-Key": (
|
||||
"httpsweb.archive.org/web/20210102094009/"
|
||||
"https://www.scribbr.com/citing-sources/et-al/IN",
|
||||
),
|
||||
"X-RL": "0",
|
||||
"X-Page-Cache": "MISS",
|
||||
"X-Archive-Screenname": "0",
|
||||
"Content-Encoding": "gzip",
|
||||
}
|
||||
|
||||
assert (
|
||||
save_api.archive_url_parser()
|
||||
== "https://web.archive.org/web/20171128185327/https://www.scribbr.com/citing-sources/et-al/"
|
||||
save_api.headers = cast(CaseInsensitiveDict[str], headers)
|
||||
|
||||
expected_url2 = (
|
||||
"https://web.archive.org/web/20210102094009/"
|
||||
"https://www.scribbr.com/citing-sources/et-al/"
|
||||
)
|
||||
assert save_api.archive_url_parser() == expected_url2
|
||||
|
||||
save_api.headers = "TEST TEST TEST AND NO MATCH - TEST FOR RESPONSE URL MATCHING"
|
||||
save_api.response_url = "https://web.archive.org/web/20171128185327/https://www.scribbr.com/citing-sources/et-al"
|
||||
assert (
|
||||
save_api.archive_url_parser()
|
||||
== "https://web.archive.org/web/20171128185327/https://www.scribbr.com/citing-sources/et-al"
|
||||
expected_url_3 = (
|
||||
"https://web.archive.org/web/20171128185327/"
|
||||
"https://www.scribbr.com/citing-sources/et-al/US"
|
||||
)
|
||||
h = f"START\nX-Cache-Key: {expected_url_3}\nEND\n"
|
||||
save_api.headers = h # type: ignore[assignment]
|
||||
|
||||
expected_url4 = (
|
||||
"https://web.archive.org/web/20171128185327/"
|
||||
"https://www.scribbr.com/citing-sources/et-al/"
|
||||
)
|
||||
assert save_api.archive_url_parser() == expected_url4
|
||||
|
||||
h = "TEST TEST TEST AND NO MATCH - TEST FOR RESPONSE URL MATCHING"
|
||||
save_api.headers = h # type: ignore[assignment]
|
||||
save_api.response_url = (
|
||||
"https://web.archive.org/web/20171128185327/"
|
||||
"https://www.scribbr.com/citing-sources/et-al"
|
||||
)
|
||||
expected_url5 = (
|
||||
"https://web.archive.org/web/20171128185327/"
|
||||
"https://www.scribbr.com/citing-sources/et-al"
|
||||
)
|
||||
assert save_api.archive_url_parser() == expected_url5
|
||||
|
||||
|
||||
def test_archive_url():
|
||||
def test_archive_url() -> None:
|
||||
"""
|
||||
Checks the attribute archive_url's value when the save method was not
|
||||
explicitly invoked by the end-user but the save method was invoked implicitly
|
||||
by the archive_url method which is an attribute due to @property.
|
||||
"""
|
||||
url = "https://example.com"
|
||||
user_agent = "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/14.1.1 Safari/604.1"
|
||||
)
|
||||
save_api = WaybackMachineSaveAPI(url, user_agent)
|
||||
save_api.saved_archive = (
|
||||
"https://web.archive.org/web/20220124063056/https://example.com/"
|
||||
|
@ -6,13 +6,13 @@ from waybackpy.utils import (
|
||||
)
|
||||
|
||||
|
||||
def test_default_user_agent():
|
||||
def test_default_user_agent() -> None:
|
||||
assert (
|
||||
DEFAULT_USER_AGENT
|
||||
== "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__
|
||||
== f"waybackpy {__version__} - https://github.com/akamhy/waybackpy"
|
||||
)
|
||||
|
||||
|
||||
def test_latest_version():
|
||||
def test_latest_version() -> None:
|
||||
package_name = "waybackpy"
|
||||
assert latest_version_github(package_name) == latest_version_pypi(package_name)
|
||||
|
Reference in New Issue
Block a user