2020-05-04 04:35:52 +02:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
from datetime import datetime
|
2020-05-04 06:19:40 +02:00
|
|
|
try:
|
|
|
|
from urllib.request import Request, urlopen
|
|
|
|
from urllib.error import HTTPError
|
|
|
|
except ImportError:
|
|
|
|
from urllib2 import Request, urlopen, HTTPError
|
|
|
|
|
2020-05-04 04:35:52 +02:00
|
|
|
|
|
|
|
default_UA = "waybackpy python package"
|
|
|
|
|
|
|
|
class TooManyArchivingRequestsError(Exception):
|
|
|
|
"""
|
|
|
|
Error when a single url reqeusted for archiving too many times in a short timespam.
|
|
|
|
Wayback machine doesn't supports archivng any url too many times in a short period of time.
|
|
|
|
"""
|
|
|
|
|
|
|
|
class ArchivingNotAllowed(Exception):
|
|
|
|
"""
|
|
|
|
Files like robots.txt are set to deny robot archiving.
|
|
|
|
Wayback machine respects these file, will not archive.
|
|
|
|
"""
|
|
|
|
|
|
|
|
class PageNotSavedError(Exception):
|
|
|
|
"""
|
2020-05-04 05:31:33 +02:00
|
|
|
When unable to save a webpage.
|
|
|
|
"""
|
|
|
|
|
|
|
|
class ArchiveNotFound(Exception):
|
|
|
|
"""
|
|
|
|
When a page was never archived but client asks for old archive.
|
2020-05-04 04:35:52 +02:00
|
|
|
"""
|
|
|
|
|
2020-05-04 12:03:00 +02:00
|
|
|
class UrlNotFound(Exception):
|
|
|
|
"""
|
|
|
|
Raised when 404 UrlNotFound.
|
|
|
|
"""
|
|
|
|
|
|
|
|
class BadGateWayError(Exception):
|
|
|
|
"""
|
|
|
|
Raised when 502 bad gateway.
|
|
|
|
"""
|
|
|
|
|
2020-05-04 04:35:52 +02:00
|
|
|
class InvalidUrlError(Exception):
|
|
|
|
"""
|
2020-05-04 05:31:33 +02:00
|
|
|
Raised when url doesn't follow the standard url format.
|
2020-05-04 04:35:52 +02:00
|
|
|
"""
|
|
|
|
|
2020-05-04 05:26:01 +02:00
|
|
|
def clean_url(url):
|
|
|
|
return str(url).strip().replace(" ","_")
|
|
|
|
|
2020-05-04 04:35:52 +02:00
|
|
|
def save(url,UA=default_UA):
|
|
|
|
base_save_url = "https://web.archive.org/save/"
|
2020-05-04 12:03:00 +02:00
|
|
|
request_url = (base_save_url + clean_url(url))
|
2020-05-04 04:35:52 +02:00
|
|
|
hdr = { 'User-Agent' : '%s' % UA }
|
|
|
|
req = Request(request_url, headers=hdr)
|
|
|
|
if "." not in url:
|
|
|
|
raise InvalidUrlError("'%s' is not a vaild url." % url)
|
|
|
|
try:
|
|
|
|
response = urlopen(req) #nosec
|
2020-05-04 06:19:40 +02:00
|
|
|
except HTTPError as e:
|
2020-05-04 04:35:52 +02:00
|
|
|
if e.code == 502:
|
2020-05-04 12:03:00 +02:00
|
|
|
raise BadGateWayError(e)
|
2020-05-04 04:35:52 +02:00
|
|
|
elif e.code == 429:
|
|
|
|
raise TooManyArchivingRequestsError(e)
|
2020-05-04 12:03:00 +02:00
|
|
|
elif e.code == 404:
|
|
|
|
raise UrlNotFound(e)
|
|
|
|
else:
|
|
|
|
raise PageNotSaved(e)
|
2020-05-04 04:35:52 +02:00
|
|
|
|
|
|
|
header = response.headers
|
|
|
|
if "exclusion.robots.policy" in str(header):
|
|
|
|
raise ArchivingNotAllowed("Can not archive %s. Disabled by site owner." % (url))
|
|
|
|
archive_id = header['Content-Location']
|
|
|
|
archived_url = "https://web.archive.org" + archive_id
|
|
|
|
return archived_url
|
|
|
|
|
|
|
|
def near(
|
|
|
|
url,
|
|
|
|
year=datetime.utcnow().strftime('%Y'),
|
|
|
|
month=datetime.utcnow().strftime('%m'),
|
|
|
|
day=datetime.utcnow().strftime('%d'),
|
|
|
|
hour=datetime.utcnow().strftime('%H'),
|
|
|
|
minute=datetime.utcnow().strftime('%M'),
|
|
|
|
UA=default_UA,
|
|
|
|
):
|
|
|
|
timestamp = str(year)+str(month)+str(day)+str(hour)+str(minute)
|
2020-05-04 05:26:01 +02:00
|
|
|
request_url = "https://archive.org/wayback/available?url=%s×tamp=%s" % (clean_url(url), str(timestamp))
|
2020-05-04 04:35:52 +02:00
|
|
|
hdr = { 'User-Agent' : '%s' % UA }
|
|
|
|
req = Request(request_url, headers=hdr)
|
|
|
|
response = urlopen(req) #nosec
|
|
|
|
import json
|
2020-05-04 06:19:40 +02:00
|
|
|
data = json.loads(response.read().decode('utf8'))
|
2020-05-04 04:35:52 +02:00
|
|
|
if not data["archived_snapshots"]:
|
2020-05-04 05:31:33 +02:00
|
|
|
raise ArchiveNotFound("'%s' is not yet archived." % url)
|
2020-05-04 04:35:52 +02:00
|
|
|
|
|
|
|
archive_url = (data["archived_snapshots"]["closest"]["url"])
|
|
|
|
return archive_url
|
|
|
|
|
2020-05-04 04:51:42 +02:00
|
|
|
def oldest(url,UA=default_UA,year=1994):
|
|
|
|
return near(url,year=year,UA=UA)
|
2020-05-04 04:35:52 +02:00
|
|
|
|
|
|
|
def newest(url,UA=default_UA):
|
|
|
|
return near(url,UA=UA)
|