don't use pagination API if total pages <= 2

This commit is contained in:
Akash Mahanty 2021-01-12 09:46:07 +05:30
parent 259a024eb1
commit a65990aee3

View File

@ -83,10 +83,10 @@ class Cdx:
"""
endpoint = "https://web.archive.org/cdx/search/cdx"
if use_page == True:
total_pages = _get_total_pages(self.url, self.user_agent)
#If we only have two or less pages of archives then we care for accuracy
# pagination API can be lagged sometimes
if use_page == True and total_pages >= 2:
blank_pages = 0
for i in range(total_pages):
payload["page"] = str(i)