From e61447effddb1ab5d68e58ff07420f18f47a1c5b Mon Sep 17 00:00:00 2001 From: eggplants Date: Thu, 3 Feb 2022 22:43:39 +0900 Subject: [PATCH] Format and lint codes and fix packaging (#125) * add: configure files (setup.py->setup.py+setup.cfg+pyproject.toml) * add: __download_url__ * format with black and isort * fix: flake8 section in setup.cfg * add: E501 to flake ignore * fix: metadata.name does not accept attr * fix: merge __version__.py into __init__.py * fix: flake8 errors in tests/ * fix: datetime.datetime -> datetime * fix: banner * fix: ignore W605 for banner * fix: way to install deps in CI * add: versem to setuptools * fix: drop python<=3.6 (#126) from package and CI --- .github/workflows/build_test.yml | 4 +- .github/workflows/unit_test.yml | 12 +++--- pyproject.toml | 3 ++ requirements-dev.txt | 9 +++-- setup.cfg | 66 ++++++++++++++++++++++++++++++-- setup.py | 65 +------------------------------ tests/test_availability_api.py | 17 ++++---- tests/test_cdx_snapshot.py | 1 - tests/test_cdx_utils.py | 15 ++++---- tests/test_save_api.py | 20 ++++++---- tests/test_utils.py | 6 +-- waybackpy/__init__.py | 47 +++++++++++++++++------ waybackpy/__version__.py | 11 ------ waybackpy/availability_api.py | 8 ++-- waybackpy/cdx_api.py | 9 ++--- waybackpy/cdx_utils.py | 4 +- waybackpy/cli.py | 16 ++++---- waybackpy/save_api.py | 10 ++--- waybackpy/utils.py | 3 +- waybackpy/wrapper.py | 5 ++- 20 files changed, 179 insertions(+), 152 deletions(-) create mode 100644 pyproject.toml delete mode 100644 waybackpy/__version__.py diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index ef61a4a..aac0944 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.6', '3.10'] + python-version: ['3.7', '3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install -U setuptools wheel - name: Build test the package run: | python setup.py sdist bdist_wheel diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index a076b00..8fd5b95 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9'] + python-version: ['3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -25,14 +25,14 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + pip install '.[dev]' - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 waybackpy/ --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 waybackpy/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --per-file-ignores="waybackpy/__init__.py:F401" + flake8 . --count --show-source --statistics + - name: Lint with black + run: | + black . --check --diff # - name: Static type test with mypy # run: | # mypy diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..10376dc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["wheel", "setuptools"] +build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index 24b2b39..288ffd3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,9 @@ +black click -requests -pytest -pytest-cov codecov flake8 mypy -black +setuptools>=46.4.0 +pytest +pytest-cov +requests diff --git a/setup.cfg b/setup.cfg index 8980e9c..197b071 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,67 @@ [metadata] -description-file = README.md -license_file = LICENSE +name = waybackpy +version = attr: waybackpy.__version__ +description = attr: waybackpy.__description__ +long_description = file: README.md +long_description_content_type = text/markdown +license = attr: waybackpy.__license__ +author = attr: waybackpy.__author__ +author_email = attr: waybackpy.__author_email__ +url = attr: waybackpy.__url__ +download_url = attr: waybackpy.__download_url__ +project_urls = + Documentation = https://github.com/akamhy/waybackpy/wiki + Source = https://github.com/akamhy/waybackpy + Tracker = https://github.com/akamhy/waybackpy/issues +keywords = + Archive Website + Wayback Machine + Internet Archive + Wayback Machine CLI + Wayback Machine Python + Internet Archiving + Availability API + CDX API + savepagenow +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + Natural Language :: English + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + +[options] +packages = find: +python_requires = >= 3.7 +install_requires = + click + requests + +[options.extras_require] +dev = + black + codecov + flake8 + mypy + pytest + pytest-cov + setuptools>=46.4.0 + + +[options.entry_points] +console_scripts = + waybackpy = waybackpy.cli:main + +[isort] +profile = black [flake8] +indent-size = 4 max-line-length = 88 -extend-ignore = E203,W503 +extend-ignore = E203,W503,E501,W605 diff --git a/setup.py b/setup.py index b7fc3e4..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,66 +1,3 @@ -import os.path from setuptools import setup -readme_path = os.path.join(os.path.dirname(__file__), "README.md") -with open(readme_path, encoding="utf-8") as f: - long_description = f.read() - -about = {} -version_path = os.path.join(os.path.dirname(__file__), "waybackpy", "__version__.py") -with open(version_path, encoding="utf-8") as f: - exec(f.read(), about) - -version = str(about["__version__"]) - -download_url = "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format( - version=version -) - -setup( - name=about["__title__"], - packages=["waybackpy"], - version=version, - description=about["__description__"], - long_description=long_description, - long_description_content_type="text/markdown", - license=about["__license__"], - author=about["__author__"], - author_email=about["__author_email__"], - url=about["__url__"], - download_url=download_url, - keywords=[ - "Archive Website", - "Wayback Machine", - "Internet Archive", - "Wayback Machine CLI", - "Wayback Machine Python", - "Internet Archiving", - "Availability API", - "CDX API", - "savepagenow", - ], - install_requires=["requests", "click"], - python_requires=">=3.4", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Natural Language :: English", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - ], - entry_points={"console_scripts": ["waybackpy = waybackpy.cli:main"]}, - project_urls={ - "Documentation": "https://github.com/akamhy/waybackpy/wiki", - "Source": "https://github.com/akamhy/waybackpy", - "Tracker": "https://github.com/akamhy/waybackpy/issues", - }, -) +setup() diff --git a/tests/test_availability_api.py b/tests/test_availability_api.py index 1260129..3b2b812 100644 --- a/tests/test_availability_api.py +++ b/tests/test_availability_api.py @@ -1,21 +1,24 @@ -import pytest import random import string from datetime import datetime, timedelta +import pytest + from waybackpy.availability_api import WaybackMachineAvailabilityAPI from waybackpy.exceptions import ( - InvalidJSONInAvailabilityAPIResponse, ArchiveNotInAvailabilityAPIResponse, + InvalidJSONInAvailabilityAPIResponse, ) now = datetime.utcnow() url = "https://example.com/" user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" -rndstr = lambda n: "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(n) -) + +def rndstr(n): + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(n) + ) def test_oldest(): @@ -57,7 +60,7 @@ def test_invalid_json(): """ with pytest.raises(InvalidJSONInAvailabilityAPIResponse): availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent) - archive_url = availability_api.archive_url + _ = availability_api.archive_url def test_no_archive(): @@ -73,7 +76,7 @@ def test_no_archive(): availability_api = WaybackMachineAvailabilityAPI( url="https://%s.cn" % rndstr(30), user_agent=user_agent ) - archive_url = availability_api.archive_url + _ = availability_api.archive_url def test_no_api_call_str_repr(): diff --git a/tests/test_cdx_snapshot.py b/tests/test_cdx_snapshot.py index 0f9034f..baea40e 100644 --- a/tests/test_cdx_snapshot.py +++ b/tests/test_cdx_snapshot.py @@ -1,4 +1,3 @@ -import pytest from datetime import datetime from waybackpy.cdx_snapshot import CDXSnapshot diff --git a/tests/test_cdx_utils.py b/tests/test_cdx_utils.py index 27435f2..406e7e6 100644 --- a/tests/test_cdx_utils.py +++ b/tests/test_cdx_utils.py @@ -1,13 +1,14 @@ import pytest -from waybackpy.exceptions import WaybackError + from waybackpy.cdx_utils import ( - get_total_pages, + check_collapses, + check_filters, + check_match_type, full_url, get_response, - check_filters, - check_collapses, - check_match_type, + get_total_pages, ) +from waybackpy.exceptions import WaybackError def test_get_total_pages(): @@ -86,10 +87,10 @@ def test_check_collapses(): def test_check_match_type(): - assert None == check_match_type(None, "url") + assert check_match_type(None, "url") is None match_type = "exact" url = "test_url" - assert None == check_match_type(match_type, url) + assert check_match_type(match_type, url) is None url = "has * in it" with pytest.raises(WaybackError): diff --git a/tests/test_save_api.py b/tests/test_save_api.py index be9b56c..7cd725d 100644 --- a/tests/test_save_api.py +++ b/tests/test_save_api.py @@ -1,15 +1,18 @@ -import pytest -import time import random import string +import time from datetime import datetime -from waybackpy.save_api import WaybackMachineSaveAPI -from waybackpy.exceptions import MaximumSaveRetriesExceeded +import pytest -rndstr = lambda n: "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(n) -) +from waybackpy.exceptions import MaximumSaveRetriesExceeded +from waybackpy.save_api import WaybackMachineSaveAPI + + +def rndstr(n): + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(n) + ) def test_save(): @@ -23,8 +26,9 @@ def test_save(): cached_save = save_api.cached_save assert cached_save in [True, False] assert archive_url.find("github.com/akamhy/waybackpy") != -1 + assert timestamp is not None assert str(headers).find("github.com/akamhy/waybackpy") != -1 - assert type(save_api.timestamp()) == type(datetime(year=2020, month=10, day=2)) + assert isinstance(save_api.timestamp(), datetime) def test_max_redirect_exceeded(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 8a05ca8..6f6d509 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,9 @@ +from waybackpy import __version__ from waybackpy.utils import ( - latest_version_pypi, - latest_version_github, DEFAULT_USER_AGENT, + latest_version_github, + latest_version_pypi, ) -from waybackpy.__version__ import __version__ def test_default_user_agent(): diff --git a/waybackpy/__init__.py b/waybackpy/__init__.py index c86b736..602ca00 100644 --- a/waybackpy/__init__.py +++ b/waybackpy/__init__.py @@ -1,14 +1,37 @@ -from .wrapper import Url +__title__ = "waybackpy" +__description__ = ( + "Python package that interfaces with the Internet Archive's Wayback Machine APIs. " + "Archive pages and retrieve archived pages easily." +) +__url__ = "https://akamhy.github.io/waybackpy/" +__version__ = "3.0.2" +__download_url__ = ( + "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format( + version=__version__ + ) +) +__author__ = "Akash Mahanty" +__author_email__ = "akamhy@yahoo.com" +__license__ = "MIT" +__copyright__ = "Copyright 2020-2022 Akash Mahanty et al." + +from .availability_api import WaybackMachineAvailabilityAPI from .cdx_api import WaybackMachineCDXServerAPI from .save_api import WaybackMachineSaveAPI -from .availability_api import WaybackMachineAvailabilityAPI -from .__version__ import ( - __title__, - __description__, - __url__, - __version__, - __author__, - __author_email__, - __license__, - __copyright__, -) +from .wrapper import Url + +__all__ = [ + "__author__", + "__author_email__", + "__copyright__", + "__description__", + "__license__", + "__title__", + "__url__", + "__download_url__", + "__version__", + "WaybackMachineAvailabilityAPI", + "WaybackMachineCDXServerAPI", + "WaybackMachineSaveAPI", + "Url", +] diff --git a/waybackpy/__version__.py b/waybackpy/__version__.py deleted file mode 100644 index e2c4906..0000000 --- a/waybackpy/__version__.py +++ /dev/null @@ -1,11 +0,0 @@ -__title__ = "waybackpy" -__description__ = ( - "Python package that interfaces with the Internet Archive's Wayback Machine APIs. " - "Archive pages and retrieve archived pages easily." -) -__url__ = "https://akamhy.github.io/waybackpy/" -__version__ = "3.0.2" -__author__ = "Akash Mahanty" -__author_email__ = "akamhy@yahoo.com" -__license__ = "MIT" -__copyright__ = "Copyright 2020-2022 Akash Mahanty et al." diff --git a/waybackpy/availability_api.py b/waybackpy/availability_api.py index 72f247f..6e76bb8 100644 --- a/waybackpy/availability_api.py +++ b/waybackpy/availability_api.py @@ -1,12 +1,14 @@ -import time import json -import requests +import time from datetime import datetime -from .utils import DEFAULT_USER_AGENT + +import requests + from .exceptions import ( ArchiveNotInAvailabilityAPIResponse, InvalidJSONInAvailabilityAPIResponse, ) +from .utils import DEFAULT_USER_AGENT class WaybackMachineAvailabilityAPI: diff --git a/waybackpy/cdx_api.py b/waybackpy/cdx_api.py index 5d122f5..a04f8af 100644 --- a/waybackpy/cdx_api.py +++ b/waybackpy/cdx_api.py @@ -1,14 +1,13 @@ -from .exceptions import WaybackError from .cdx_snapshot import CDXSnapshot from .cdx_utils import ( - get_total_pages, - get_response, - check_filters, check_collapses, + check_filters, check_match_type, full_url, + get_response, + get_total_pages, ) - +from .exceptions import WaybackError from .utils import DEFAULT_USER_AGENT diff --git a/waybackpy/cdx_utils.py b/waybackpy/cdx_utils.py index ff31918..06f043c 100644 --- a/waybackpy/cdx_utils.py +++ b/waybackpy/cdx_utils.py @@ -1,7 +1,9 @@ import re + import requests -from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + from .exceptions import WaybackError from .utils import DEFAULT_USER_AGENT diff --git a/waybackpy/cli.py b/waybackpy/cli.py index bf536e5..f1117c2 100644 --- a/waybackpy/cli.py +++ b/waybackpy/cli.py @@ -1,15 +1,17 @@ -import click -import re -import os import json as JSON +import os import random -import requests +import re import string -from .__version__ import __version__ -from .utils import DEFAULT_USER_AGENT + +import click +import requests + +from . import __version__ +from .availability_api import WaybackMachineAvailabilityAPI from .cdx_api import WaybackMachineCDXServerAPI from .save_api import WaybackMachineSaveAPI -from .availability_api import WaybackMachineAvailabilityAPI +from .utils import DEFAULT_USER_AGENT from .wrapper import Url diff --git a/waybackpy/save_api.py b/waybackpy/save_api.py index b549aa4..530e03a 100644 --- a/waybackpy/save_api.py +++ b/waybackpy/save_api.py @@ -1,13 +1,13 @@ import re import time -import requests - from datetime import datetime -from urllib3.util.retry import Retry -from requests.adapters import HTTPAdapter -from .utils import DEFAULT_USER_AGENT +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + from .exceptions import MaximumSaveRetriesExceeded +from .utils import DEFAULT_USER_AGENT class WaybackMachineSaveAPI: diff --git a/waybackpy/utils.py b/waybackpy/utils.py index 0c2be13..7201403 100644 --- a/waybackpy/utils.py +++ b/waybackpy/utils.py @@ -1,5 +1,6 @@ import requests -from .__version__ import __version__ + +from . import __version__ DEFAULT_USER_AGENT = "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__ diff --git a/waybackpy/wrapper.py b/waybackpy/wrapper.py index 7ca8d22..3121b77 100644 --- a/waybackpy/wrapper.py +++ b/waybackpy/wrapper.py @@ -1,8 +1,9 @@ -from .save_api import WaybackMachineSaveAPI +from datetime import datetime, timedelta + from .availability_api import WaybackMachineAvailabilityAPI from .cdx_api import WaybackMachineCDXServerAPI +from .save_api import WaybackMachineSaveAPI from .utils import DEFAULT_USER_AGENT -from datetime import datetime, timedelta """ The Url class is not recommended to be used anymore, instead use the