Format and lint codes and fix packaging (#125)

* add: configure files (setup.py->setup.py+setup.cfg+pyproject.toml)

* add: __download_url__

* format with black and isort

* fix: flake8 section in setup.cfg

* add: E501 to flake ignore

* fix: metadata.name does not accept attr

* fix: merge __version__.py into __init__.py

* fix: flake8 errors in tests/

* fix: datetime.datetime -> datetime

* fix: banner

* fix: ignore W605 for banner

* fix: way to install deps in CI

* add: versem to setuptools

* fix: drop python<=3.6 (#126) from package and CI

.github/workflows/build_test.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.6', '3.10']
+        python-version: ['3.7', '3.10']
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel
+        pip install -U setuptools wheel
     - name: Build test the package
       run: |
         python setup.py sdist bdist_wheel

.github/workflows/unit_test.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9']
+        python-version: ['3.10']
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -25,14 +25,14 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install '.[dev]'
-        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 waybackpy/ --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+    - name: Lint with black
-        # flake8 waybackpy/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --per-file-ignores="waybackpy/__init__.py:F401"
+      run: |
+        black . --check --diff
     # - name: Static type test with mypy
     #   run: |
     #     mypy

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["wheel", "setuptools"]
+build-backend = "setuptools.build_meta"

requirements-dev.txt

@@ -1,8 +1,9 @@
+black
 click
-requests
-pytest
-pytest-cov
 codecov
 flake8
 mypy
-black
+setuptools>=46.4.0
+pytest
+pytest-cov
+requests

setup.cfg

@@ -1,7 +1,67 @@
 [metadata]
-description-file = README.md
+name = waybackpy
-license_file = LICENSE
+version = attr: waybackpy.__version__
+description = attr: waybackpy.__description__
+long_description = file: README.md
+long_description_content_type = text/markdown
+license = attr: waybackpy.__license__
+author = attr: waybackpy.__author__
+author_email = attr: waybackpy.__author_email__
+url = attr: waybackpy.__url__
+download_url = attr: waybackpy.__download_url__
+project_urls =
+    Documentation = https://github.com/akamhy/waybackpy/wiki
+    Source = https://github.com/akamhy/waybackpy
+    Tracker = https://github.com/akamhy/waybackpy/issues
+keywords =
+    Archive Website
+    Wayback Machine
+    Internet Archive
+    Wayback Machine CLI
+    Wayback Machine Python
+    Internet Archiving
+    Availability API
+    CDX API
+    savepagenow
+classifiers =
+    Development Status :: 4 - Beta
+    Intended Audience :: Developers
+    Natural Language :: English
+    License :: OSI Approved :: MIT License
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: Implementation :: CPython
+
+[options]
+packages = find:
+python_requires = >= 3.7
+install_requires =
+    click
+    requests
+
+[options.extras_require]
+dev =
+    black
+    codecov
+    flake8
+    mypy
+    pytest
+    pytest-cov
+    setuptools>=46.4.0
+
+
+[options.entry_points]
+console_scripts =
+    waybackpy = waybackpy.cli:main
+
+[isort]
+profile = black
 
 [flake8]
+indent-size = 4
 max-line-length = 88
-extend-ignore = E203,W503
+extend-ignore = E203,W503,E501,W605

setup.py

@@ -1,66 +1,3 @@
-import os.path
 from setuptools import setup
 
-readme_path = os.path.join(os.path.dirname(__file__), "README.md")
+setup()
-with open(readme_path, encoding="utf-8") as f:
-    long_description = f.read()
-
-about = {}
-version_path = os.path.join(os.path.dirname(__file__), "waybackpy", "__version__.py")
-with open(version_path, encoding="utf-8") as f:
-    exec(f.read(), about)
-
-version = str(about["__version__"])
-
-download_url = "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
-    version=version
-)
-
-setup(
-    name=about["__title__"],
-    packages=["waybackpy"],
-    version=version,
-    description=about["__description__"],
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    license=about["__license__"],
-    author=about["__author__"],
-    author_email=about["__author_email__"],
-    url=about["__url__"],
-    download_url=download_url,
-    keywords=[
-        "Archive Website",
-        "Wayback Machine",
-        "Internet Archive",
-        "Wayback Machine CLI",
-        "Wayback Machine Python",
-        "Internet Archiving",
-        "Availability API",
-        "CDX API",
-        "savepagenow",
-    ],
-    install_requires=["requests", "click"],
-    python_requires=">=3.4",
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Intended Audience :: Developers",
-        "Natural Language :: English",
-        "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: Implementation :: CPython",
-    ],
-    entry_points={"console_scripts": ["waybackpy = waybackpy.cli:main"]},
-    project_urls={
-        "Documentation": "https://github.com/akamhy/waybackpy/wiki",
-        "Source": "https://github.com/akamhy/waybackpy",
-        "Tracker": "https://github.com/akamhy/waybackpy/issues",
-    },
-)

tests/test_availability_api.py

@@ -1,21 +1,24 @@
-import pytest
 import random
 import string
 from datetime import datetime, timedelta
 
+import pytest
+
 from waybackpy.availability_api import WaybackMachineAvailabilityAPI
 from waybackpy.exceptions import (
-    InvalidJSONInAvailabilityAPIResponse,
     ArchiveNotInAvailabilityAPIResponse,
+    InvalidJSONInAvailabilityAPIResponse,
 )
 
 now = datetime.utcnow()
 url = "https://example.com/"
 user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
 
-rndstr = lambda n: "".join(
+
-    random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
+def rndstr(n):
-)
+    return "".join(
+        random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
+    )
 
 
 def test_oldest():
@@ -57,7 +60,7 @@ def test_invalid_json():
     """
     with pytest.raises(InvalidJSONInAvailabilityAPIResponse):
         availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent)
-        archive_url = availability_api.archive_url
+        _ = availability_api.archive_url
 
 
 def test_no_archive():
@@ -73,7 +76,7 @@ def test_no_archive():
         availability_api = WaybackMachineAvailabilityAPI(
             url="https://%s.cn" % rndstr(30), user_agent=user_agent
         )
-        archive_url = availability_api.archive_url
+        _ = availability_api.archive_url
 
 
 def test_no_api_call_str_repr():

tests/test_cdx_snapshot.py

@@ -1,4 +1,3 @@
-import pytest
 from datetime import datetime
 
 from waybackpy.cdx_snapshot import CDXSnapshot

tests/test_cdx_utils.py

@@ -1,13 +1,14 @@
 import pytest
-from waybackpy.exceptions import WaybackError
+
 from waybackpy.cdx_utils import (
-    get_total_pages,
+    check_collapses,
+    check_filters,
+    check_match_type,
     full_url,
     get_response,
-    check_filters,
+    get_total_pages,
-    check_collapses,
-    check_match_type,
 )
+from waybackpy.exceptions import WaybackError
 
 
 def test_get_total_pages():
@@ -86,10 +87,10 @@ def test_check_collapses():
 
 
 def test_check_match_type():
-    assert None == check_match_type(None, "url")
+    assert check_match_type(None, "url") is None
     match_type = "exact"
     url = "test_url"
-    assert None == check_match_type(match_type, url)
+    assert check_match_type(match_type, url) is None
 
     url = "has * in it"
     with pytest.raises(WaybackError):

tests/test_save_api.py

@@ -1,15 +1,18 @@
-import pytest
-import time
 import random
 import string
+import time
 from datetime import datetime
 
-from waybackpy.save_api import WaybackMachineSaveAPI
+import pytest
-from waybackpy.exceptions import MaximumSaveRetriesExceeded
 
-rndstr = lambda n: "".join(
+from waybackpy.exceptions import MaximumSaveRetriesExceeded
-    random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
+from waybackpy.save_api import WaybackMachineSaveAPI
-)
+
+
+def rndstr(n):
+    return "".join(
+        random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
+    )
 
 
 def test_save():
@@ -23,8 +26,9 @@ def test_save():
     cached_save = save_api.cached_save
     assert cached_save in [True, False]
     assert archive_url.find("github.com/akamhy/waybackpy") != -1
+    assert timestamp is not None
     assert str(headers).find("github.com/akamhy/waybackpy") != -1
-    assert type(save_api.timestamp()) == type(datetime(year=2020, month=10, day=2))
+    assert isinstance(save_api.timestamp(), datetime)
 
 
 def test_max_redirect_exceeded():

tests/test_utils.py

@@ -1,9 +1,9 @@
+from waybackpy import __version__
 from waybackpy.utils import (
-    latest_version_pypi,
-    latest_version_github,
     DEFAULT_USER_AGENT,
+    latest_version_github,
+    latest_version_pypi,
 )
-from waybackpy.__version__ import __version__
 
 
 def test_default_user_agent():

waybackpy/__init__.py

@@ -1,14 +1,37 @@
-from .wrapper import Url
+__title__ = "waybackpy"
+__description__ = (
+    "Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
+    "Archive pages and retrieve archived pages easily."
+)
+__url__ = "https://akamhy.github.io/waybackpy/"
+__version__ = "3.0.2"
+__download_url__ = (
+    "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
+        version=__version__
+    )
+)
+__author__ = "Akash Mahanty"
+__author_email__ = "akamhy@yahoo.com"
+__license__ = "MIT"
+__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."
+
+from .availability_api import WaybackMachineAvailabilityAPI
 from .cdx_api import WaybackMachineCDXServerAPI
 from .save_api import WaybackMachineSaveAPI
-from .availability_api import WaybackMachineAvailabilityAPI
+from .wrapper import Url
-from .__version__ import (
+
-    __title__,
+__all__ = [
-    __description__,
+    "__author__",
-    __url__,
+    "__author_email__",
-    __version__,
+    "__copyright__",
-    __author__,
+    "__description__",
-    __author_email__,
+    "__license__",
-    __license__,
+    "__title__",
-    __copyright__,
+    "__url__",
-)
+    "__download_url__",
+    "__version__",
+    "WaybackMachineAvailabilityAPI",
+    "WaybackMachineCDXServerAPI",
+    "WaybackMachineSaveAPI",
+    "Url",
+]

waybackpy/__version__.py

@@ -1,11 +0,0 @@
-__title__ = "waybackpy"
-__description__ = (
-    "Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
-    "Archive pages and retrieve archived pages easily."
-)
-__url__ = "https://akamhy.github.io/waybackpy/"
-__version__ = "3.0.2"
-__author__ = "Akash Mahanty"
-__author_email__ = "akamhy@yahoo.com"
-__license__ = "MIT"
-__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."

waybackpy/availability_api.py

@@ -1,12 +1,14 @@
-import time
 import json
-import requests
+import time
 from datetime import datetime
-from .utils import DEFAULT_USER_AGENT
+
+import requests
+
 from .exceptions import (
     ArchiveNotInAvailabilityAPIResponse,
     InvalidJSONInAvailabilityAPIResponse,
 )
+from .utils import DEFAULT_USER_AGENT
 
 
 class WaybackMachineAvailabilityAPI:

waybackpy/cdx_api.py

@@ -1,14 +1,13 @@
-from .exceptions import WaybackError
 from .cdx_snapshot import CDXSnapshot
 from .cdx_utils import (
-    get_total_pages,
-    get_response,
-    check_filters,
     check_collapses,
+    check_filters,
     check_match_type,
     full_url,
+    get_response,
+    get_total_pages,
 )
-
+from .exceptions import WaybackError
 from .utils import DEFAULT_USER_AGENT
 
 

waybackpy/cdx_utils.py

@@ -1,7 +1,9 @@
 import re
+
 import requests
-from urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
 from .exceptions import WaybackError
 from .utils import DEFAULT_USER_AGENT
 

waybackpy/cli.py

@@ -1,15 +1,17 @@
-import click
-import re
-import os
 import json as JSON
+import os
 import random
-import requests
+import re
 import string
-from .__version__ import __version__
+
-from .utils import DEFAULT_USER_AGENT
+import click
+import requests
+
+from . import __version__
+from .availability_api import WaybackMachineAvailabilityAPI
 from .cdx_api import WaybackMachineCDXServerAPI
 from .save_api import WaybackMachineSaveAPI
-from .availability_api import WaybackMachineAvailabilityAPI
+from .utils import DEFAULT_USER_AGENT
 from .wrapper import Url
 
 

waybackpy/save_api.py

@@ -1,13 +1,13 @@
 import re
 import time
-import requests
-
 from datetime import datetime
-from urllib3.util.retry import Retry
-from requests.adapters import HTTPAdapter
 
-from .utils import DEFAULT_USER_AGENT
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
 from .exceptions import MaximumSaveRetriesExceeded
+from .utils import DEFAULT_USER_AGENT
 
 
 class WaybackMachineSaveAPI:

waybackpy/utils.py

@@ -1,5 +1,6 @@
 import requests
-from .__version__ import __version__
+
+from . import __version__
 
 DEFAULT_USER_AGENT = "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__
 

waybackpy/wrapper.py

@@ -1,8 +1,9 @@
-from .save_api import WaybackMachineSaveAPI
+from datetime import datetime, timedelta
+
 from .availability_api import WaybackMachineAvailabilityAPI
 from .cdx_api import WaybackMachineCDXServerAPI
+from .save_api import WaybackMachineSaveAPI
 from .utils import DEFAULT_USER_AGENT
-from datetime import datetime, timedelta
 
 """
 The Url class is not recommended to be used anymore, instead use the