Format and lint codes and fix packaging (#125)

* add: configure files (setup.py->setup.py+setup.cfg+pyproject.toml)

* add: __download_url__

* format with black and isort

* fix: flake8 section in setup.cfg

* add: E501 to flake ignore

* fix: metadata.name does not accept attr

* fix: merge __version__.py into __init__.py

* fix: flake8 errors in tests/

* fix: datetime.datetime -> datetime

* fix: banner

* fix: ignore W605 for banner

* fix: way to install deps in CI

* add: versem to setuptools

* fix: drop python<=3.6 (#126) from package and CI
This commit is contained in:
eggplants 2022-02-03 22:43:39 +09:00 committed by GitHub
parent 947647f2e7
commit e61447effd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 179 additions and 152 deletions

View File

@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
python-version: ['3.6', '3.10'] python-version: ['3.7', '3.10']
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
@ -24,7 +24,7 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip python -m pip install --upgrade pip
pip install setuptools wheel pip install -U setuptools wheel
- name: Build test the package - name: Build test the package
run: | run: |
python setup.py sdist bdist_wheel python setup.py sdist bdist_wheel

View File

@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
python-version: ['3.9'] python-version: ['3.10']
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
@ -25,14 +25,14 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install '.[dev]'
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
- name: Lint with flake8 - name: Lint with flake8
run: | run: |
# stop the build if there are Python syntax errors or undefined names # stop the build if there are Python syntax errors or undefined names
flake8 waybackpy/ --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - name: Lint with black
# flake8 waybackpy/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --per-file-ignores="waybackpy/__init__.py:F401" run: |
black . --check --diff
# - name: Static type test with mypy # - name: Static type test with mypy
# run: | # run: |
# mypy # mypy

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[build-system]
requires = ["wheel", "setuptools"]
build-backend = "setuptools.build_meta"

View File

@ -1,8 +1,9 @@
black
click click
requests
pytest
pytest-cov
codecov codecov
flake8 flake8
mypy mypy
black setuptools>=46.4.0
pytest
pytest-cov
requests

View File

@ -1,7 +1,67 @@
[metadata] [metadata]
description-file = README.md name = waybackpy
license_file = LICENSE version = attr: waybackpy.__version__
description = attr: waybackpy.__description__
long_description = file: README.md
long_description_content_type = text/markdown
license = attr: waybackpy.__license__
author = attr: waybackpy.__author__
author_email = attr: waybackpy.__author_email__
url = attr: waybackpy.__url__
download_url = attr: waybackpy.__download_url__
project_urls =
Documentation = https://github.com/akamhy/waybackpy/wiki
Source = https://github.com/akamhy/waybackpy
Tracker = https://github.com/akamhy/waybackpy/issues
keywords =
Archive Website
Wayback Machine
Internet Archive
Wayback Machine CLI
Wayback Machine Python
Internet Archiving
Availability API
CDX API
savepagenow
classifiers =
Development Status :: 4 - Beta
Intended Audience :: Developers
Natural Language :: English
License :: OSI Approved :: MIT License
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: Implementation :: CPython
[options]
packages = find:
python_requires = >= 3.7
install_requires =
click
requests
[options.extras_require]
dev =
black
codecov
flake8
mypy
pytest
pytest-cov
setuptools>=46.4.0
[options.entry_points]
console_scripts =
waybackpy = waybackpy.cli:main
[isort]
profile = black
[flake8] [flake8]
indent-size = 4
max-line-length = 88 max-line-length = 88
extend-ignore = E203,W503 extend-ignore = E203,W503,E501,W605

View File

@ -1,66 +1,3 @@
import os.path
from setuptools import setup from setuptools import setup
readme_path = os.path.join(os.path.dirname(__file__), "README.md") setup()
with open(readme_path, encoding="utf-8") as f:
long_description = f.read()
about = {}
version_path = os.path.join(os.path.dirname(__file__), "waybackpy", "__version__.py")
with open(version_path, encoding="utf-8") as f:
exec(f.read(), about)
version = str(about["__version__"])
download_url = "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
version=version
)
setup(
name=about["__title__"],
packages=["waybackpy"],
version=version,
description=about["__description__"],
long_description=long_description,
long_description_content_type="text/markdown",
license=about["__license__"],
author=about["__author__"],
author_email=about["__author_email__"],
url=about["__url__"],
download_url=download_url,
keywords=[
"Archive Website",
"Wayback Machine",
"Internet Archive",
"Wayback Machine CLI",
"Wayback Machine Python",
"Internet Archiving",
"Availability API",
"CDX API",
"savepagenow",
],
install_requires=["requests", "click"],
python_requires=">=3.4",
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Natural Language :: English",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: Implementation :: CPython",
],
entry_points={"console_scripts": ["waybackpy = waybackpy.cli:main"]},
project_urls={
"Documentation": "https://github.com/akamhy/waybackpy/wiki",
"Source": "https://github.com/akamhy/waybackpy",
"Tracker": "https://github.com/akamhy/waybackpy/issues",
},
)

View File

@ -1,19 +1,22 @@
import pytest
import random import random
import string import string
from datetime import datetime, timedelta from datetime import datetime, timedelta
import pytest
from waybackpy.availability_api import WaybackMachineAvailabilityAPI from waybackpy.availability_api import WaybackMachineAvailabilityAPI
from waybackpy.exceptions import ( from waybackpy.exceptions import (
InvalidJSONInAvailabilityAPIResponse,
ArchiveNotInAvailabilityAPIResponse, ArchiveNotInAvailabilityAPIResponse,
InvalidJSONInAvailabilityAPIResponse,
) )
now = datetime.utcnow() now = datetime.utcnow()
url = "https://example.com/" url = "https://example.com/"
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
rndstr = lambda n: "".join(
def rndstr(n):
return "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(n) random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
) )
@ -57,7 +60,7 @@ def test_invalid_json():
""" """
with pytest.raises(InvalidJSONInAvailabilityAPIResponse): with pytest.raises(InvalidJSONInAvailabilityAPIResponse):
availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent) availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent)
archive_url = availability_api.archive_url _ = availability_api.archive_url
def test_no_archive(): def test_no_archive():
@ -73,7 +76,7 @@ def test_no_archive():
availability_api = WaybackMachineAvailabilityAPI( availability_api = WaybackMachineAvailabilityAPI(
url="https://%s.cn" % rndstr(30), user_agent=user_agent url="https://%s.cn" % rndstr(30), user_agent=user_agent
) )
archive_url = availability_api.archive_url _ = availability_api.archive_url
def test_no_api_call_str_repr(): def test_no_api_call_str_repr():

View File

@ -1,4 +1,3 @@
import pytest
from datetime import datetime from datetime import datetime
from waybackpy.cdx_snapshot import CDXSnapshot from waybackpy.cdx_snapshot import CDXSnapshot

View File

@ -1,13 +1,14 @@
import pytest import pytest
from waybackpy.exceptions import WaybackError
from waybackpy.cdx_utils import ( from waybackpy.cdx_utils import (
get_total_pages, check_collapses,
check_filters,
check_match_type,
full_url, full_url,
get_response, get_response,
check_filters, get_total_pages,
check_collapses,
check_match_type,
) )
from waybackpy.exceptions import WaybackError
def test_get_total_pages(): def test_get_total_pages():
@ -86,10 +87,10 @@ def test_check_collapses():
def test_check_match_type(): def test_check_match_type():
assert None == check_match_type(None, "url") assert check_match_type(None, "url") is None
match_type = "exact" match_type = "exact"
url = "test_url" url = "test_url"
assert None == check_match_type(match_type, url) assert check_match_type(match_type, url) is None
url = "has * in it" url = "has * in it"
with pytest.raises(WaybackError): with pytest.raises(WaybackError):

View File

@ -1,13 +1,16 @@
import pytest
import time
import random import random
import string import string
import time
from datetime import datetime from datetime import datetime
from waybackpy.save_api import WaybackMachineSaveAPI import pytest
from waybackpy.exceptions import MaximumSaveRetriesExceeded
rndstr = lambda n: "".join( from waybackpy.exceptions import MaximumSaveRetriesExceeded
from waybackpy.save_api import WaybackMachineSaveAPI
def rndstr(n):
return "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(n) random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
) )
@ -23,8 +26,9 @@ def test_save():
cached_save = save_api.cached_save cached_save = save_api.cached_save
assert cached_save in [True, False] assert cached_save in [True, False]
assert archive_url.find("github.com/akamhy/waybackpy") != -1 assert archive_url.find("github.com/akamhy/waybackpy") != -1
assert timestamp is not None
assert str(headers).find("github.com/akamhy/waybackpy") != -1 assert str(headers).find("github.com/akamhy/waybackpy") != -1
assert type(save_api.timestamp()) == type(datetime(year=2020, month=10, day=2)) assert isinstance(save_api.timestamp(), datetime)
def test_max_redirect_exceeded(): def test_max_redirect_exceeded():

View File

@ -1,9 +1,9 @@
from waybackpy import __version__
from waybackpy.utils import ( from waybackpy.utils import (
latest_version_pypi,
latest_version_github,
DEFAULT_USER_AGENT, DEFAULT_USER_AGENT,
latest_version_github,
latest_version_pypi,
) )
from waybackpy.__version__ import __version__
def test_default_user_agent(): def test_default_user_agent():

View File

@ -1,14 +1,37 @@
from .wrapper import Url __title__ = "waybackpy"
__description__ = (
"Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
"Archive pages and retrieve archived pages easily."
)
__url__ = "https://akamhy.github.io/waybackpy/"
__version__ = "3.0.2"
__download_url__ = (
"https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
version=__version__
)
)
__author__ = "Akash Mahanty"
__author_email__ = "akamhy@yahoo.com"
__license__ = "MIT"
__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."
from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI from .save_api import WaybackMachineSaveAPI
from .availability_api import WaybackMachineAvailabilityAPI from .wrapper import Url
from .__version__ import (
__title__, __all__ = [
__description__, "__author__",
__url__, "__author_email__",
__version__, "__copyright__",
__author__, "__description__",
__author_email__, "__license__",
__license__, "__title__",
__copyright__, "__url__",
) "__download_url__",
"__version__",
"WaybackMachineAvailabilityAPI",
"WaybackMachineCDXServerAPI",
"WaybackMachineSaveAPI",
"Url",
]

View File

@ -1,11 +0,0 @@
__title__ = "waybackpy"
__description__ = (
"Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
"Archive pages and retrieve archived pages easily."
)
__url__ = "https://akamhy.github.io/waybackpy/"
__version__ = "3.0.2"
__author__ = "Akash Mahanty"
__author_email__ = "akamhy@yahoo.com"
__license__ = "MIT"
__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."

View File

@ -1,12 +1,14 @@
import time
import json import json
import requests import time
from datetime import datetime from datetime import datetime
from .utils import DEFAULT_USER_AGENT
import requests
from .exceptions import ( from .exceptions import (
ArchiveNotInAvailabilityAPIResponse, ArchiveNotInAvailabilityAPIResponse,
InvalidJSONInAvailabilityAPIResponse, InvalidJSONInAvailabilityAPIResponse,
) )
from .utils import DEFAULT_USER_AGENT
class WaybackMachineAvailabilityAPI: class WaybackMachineAvailabilityAPI:

View File

@ -1,14 +1,13 @@
from .exceptions import WaybackError
from .cdx_snapshot import CDXSnapshot from .cdx_snapshot import CDXSnapshot
from .cdx_utils import ( from .cdx_utils import (
get_total_pages,
get_response,
check_filters,
check_collapses, check_collapses,
check_filters,
check_match_type, check_match_type,
full_url, full_url,
get_response,
get_total_pages,
) )
from .exceptions import WaybackError
from .utils import DEFAULT_USER_AGENT from .utils import DEFAULT_USER_AGENT

View File

@ -1,7 +1,9 @@
import re import re
import requests import requests
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from .exceptions import WaybackError from .exceptions import WaybackError
from .utils import DEFAULT_USER_AGENT from .utils import DEFAULT_USER_AGENT

View File

@ -1,15 +1,17 @@
import click
import re
import os
import json as JSON import json as JSON
import os
import random import random
import requests import re
import string import string
from .__version__ import __version__
from .utils import DEFAULT_USER_AGENT import click
import requests
from . import __version__
from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI from .save_api import WaybackMachineSaveAPI
from .availability_api import WaybackMachineAvailabilityAPI from .utils import DEFAULT_USER_AGENT
from .wrapper import Url from .wrapper import Url

View File

@ -1,13 +1,13 @@
import re import re
import time import time
import requests
from datetime import datetime from datetime import datetime
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from .utils import DEFAULT_USER_AGENT import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from .exceptions import MaximumSaveRetriesExceeded from .exceptions import MaximumSaveRetriesExceeded
from .utils import DEFAULT_USER_AGENT
class WaybackMachineSaveAPI: class WaybackMachineSaveAPI:

View File

@ -1,5 +1,6 @@
import requests import requests
from .__version__ import __version__
from . import __version__
DEFAULT_USER_AGENT = "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__ DEFAULT_USER_AGENT = "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__

View File

@ -1,8 +1,9 @@
from .save_api import WaybackMachineSaveAPI from datetime import datetime, timedelta
from .availability_api import WaybackMachineAvailabilityAPI from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI
from .utils import DEFAULT_USER_AGENT from .utils import DEFAULT_USER_AGENT
from datetime import datetime, timedelta
""" """
The Url class is not recommended to be used anymore, instead use the The Url class is not recommended to be used anymore, instead use the