added RedirectSaveError - for failed saves if the URL is a permanent … (#93)

* added RedirectSaveError - for failed saves if the URL is a permanent redirect.

* check if url is redirect before throwing exceptions, res.url is the redirect url if redirected at all

* update tests and cli errors
This commit is contained in:
Akash Mahanty 2021-04-02 10:38:17 +05:30 committed by GitHub
parent db8f902cff
commit dd1917c77e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 32 additions and 3 deletions

View File

@ -29,7 +29,7 @@ def test_save():
get=None,
)
reply = cli.args_handler(args)
assert "could happen because either your waybackpy" in str(reply)
assert "could happen because either your waybackpy" or "cannot be archived by wayback machine as it is a redirect" in str(reply)
def test_json():

View File

@ -29,6 +29,8 @@ def _save(obj):
version=__version__, header=header
)
)
if "URL cannot be archived by wayback machine as it is a redirect" in e:
return ("URL cannot be archived by wayback machine as it is a redirect")
raise WaybackError(err)

View File

@ -13,6 +13,13 @@ class WaybackError(Exception):
"""
class RedirectSaveError(WaybackError):
"""
Raised when the original URL is redirected and the
redirect URL is archived but not the original URL.
"""
class URLError(Exception):
"""
Raised when malformed URLs are passed as arguments.

View File

@ -3,7 +3,7 @@ import time
import requests
from datetime import datetime
from .exceptions import WaybackError, URLError
from .exceptions import WaybackError, URLError, RedirectSaveError
from .__version__ import __version__
from urllib3.util.retry import Retry
@ -302,7 +302,9 @@ def _get_total_pages(url, user_agent):
return int((_get_response(total_pages_url, headers=headers).text).strip())
def _archive_url_parser(header, url, latest_version=__version__, instance=None):
def _archive_url_parser(
header, url, latest_version=__version__, instance=None, response=None
):
"""Returns the archive after parsing it from the response header.
Parameters
@ -388,6 +390,16 @@ def _archive_url_parser(header, url, latest_version=__version__, instance=None):
if m:
return m.group(1)
if response:
if response.url:
if "web.archive.org/web" in response.url:
m = re.search(
r"web\.archive\.org/web/(?:[0-9]*?)/(?:.*)$",
str(response.url).strip(),
)
if m:
return m.group(0)
if instance:
newest_archive = None
try:
@ -414,6 +426,13 @@ def _archive_url_parser(header, url, latest_version=__version__, instance=None):
"Wayback Machine is malfunctioning or it refused to archive your URL."
"\nHeader:\n{header}".format(url=url, header=header)
)
if "save redirected" == header.strip():
raise RedirectSaveError(
"URL cannot be archived by wayback machine as it is a redirect.\nHeader:\n{header}".format(
header=header
)
)
else:
exc_message = (
"No archive URL found in the API response. "

View File

@ -225,6 +225,7 @@ class Url:
self.url,
latest_version=self.latest_version,
instance=self,
response=response,
)
m = re.search(