don't use pagination API if total pages <= 2
This commit is contained in:
parent
259a024eb1
commit
a65990aee3
@ -83,10 +83,10 @@ class Cdx:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
endpoint = "https://web.archive.org/cdx/search/cdx"
|
endpoint = "https://web.archive.org/cdx/search/cdx"
|
||||||
|
|
||||||
if use_page == True:
|
|
||||||
|
|
||||||
total_pages = _get_total_pages(self.url, self.user_agent)
|
total_pages = _get_total_pages(self.url, self.user_agent)
|
||||||
|
#If we only have two or less pages of archives then we care for accuracy
|
||||||
|
# pagination API can be lagged sometimes
|
||||||
|
if use_page == True and total_pages >= 2:
|
||||||
blank_pages = 0
|
blank_pages = 0
|
||||||
for i in range(total_pages):
|
for i in range(total_pages):
|
||||||
payload["page"] = str(i)
|
payload["page"] = str(i)
|
||||||
|
Loading…
Reference in New Issue
Block a user