Format and lint codes and fix packaging (#125)

* add: configure files (setup.py->setup.py+setup.cfg+pyproject.toml)

* add: __download_url__

* format with black and isort

* fix: flake8 section in setup.cfg

* add: E501 to flake ignore

* fix: metadata.name does not accept attr

* fix: merge __version__.py into __init__.py

* fix: flake8 errors in tests/

* fix: datetime.datetime -> datetime

* fix: banner

* fix: ignore W605 for banner

* fix: way to install deps in CI

* add: versem to setuptools

* fix: drop python<=3.6 (#126) from package and CI
This commit is contained in:
eggplants 2022-02-03 22:43:39 +09:00 committed by GitHub
parent 947647f2e7
commit e61447effd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 179 additions and 152 deletions

View File

@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.6', '3.10']
python-version: ['3.7', '3.10']
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
@ -24,7 +24,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
pip install -U setuptools wheel
- name: Build test the package
run: |
python setup.py sdist bdist_wheel

View File

@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.9']
python-version: ['3.10']
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
@ -25,14 +25,14 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
pip install '.[dev]'
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 waybackpy/ --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
# flake8 waybackpy/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --per-file-ignores="waybackpy/__init__.py:F401"
flake8 . --count --show-source --statistics
- name: Lint with black
run: |
black . --check --diff
# - name: Static type test with mypy
# run: |
# mypy

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[build-system]
requires = ["wheel", "setuptools"]
build-backend = "setuptools.build_meta"

View File

@ -1,8 +1,9 @@
black
click
requests
pytest
pytest-cov
codecov
flake8
mypy
black
setuptools>=46.4.0
pytest
pytest-cov
requests

View File

@ -1,7 +1,67 @@
[metadata]
description-file = README.md
license_file = LICENSE
name = waybackpy
version = attr: waybackpy.__version__
description = attr: waybackpy.__description__
long_description = file: README.md
long_description_content_type = text/markdown
license = attr: waybackpy.__license__
author = attr: waybackpy.__author__
author_email = attr: waybackpy.__author_email__
url = attr: waybackpy.__url__
download_url = attr: waybackpy.__download_url__
project_urls =
Documentation = https://github.com/akamhy/waybackpy/wiki
Source = https://github.com/akamhy/waybackpy
Tracker = https://github.com/akamhy/waybackpy/issues
keywords =
Archive Website
Wayback Machine
Internet Archive
Wayback Machine CLI
Wayback Machine Python
Internet Archiving
Availability API
CDX API
savepagenow
classifiers =
Development Status :: 4 - Beta
Intended Audience :: Developers
Natural Language :: English
License :: OSI Approved :: MIT License
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: Implementation :: CPython
[options]
packages = find:
python_requires = >= 3.7
install_requires =
click
requests
[options.extras_require]
dev =
black
codecov
flake8
mypy
pytest
pytest-cov
setuptools>=46.4.0
[options.entry_points]
console_scripts =
waybackpy = waybackpy.cli:main
[isort]
profile = black
[flake8]
indent-size = 4
max-line-length = 88
extend-ignore = E203,W503
extend-ignore = E203,W503,E501,W605

View File

@ -1,66 +1,3 @@
import os.path
from setuptools import setup
readme_path = os.path.join(os.path.dirname(__file__), "README.md")
with open(readme_path, encoding="utf-8") as f:
long_description = f.read()
about = {}
version_path = os.path.join(os.path.dirname(__file__), "waybackpy", "__version__.py")
with open(version_path, encoding="utf-8") as f:
exec(f.read(), about)
version = str(about["__version__"])
download_url = "https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
version=version
)
setup(
name=about["__title__"],
packages=["waybackpy"],
version=version,
description=about["__description__"],
long_description=long_description,
long_description_content_type="text/markdown",
license=about["__license__"],
author=about["__author__"],
author_email=about["__author_email__"],
url=about["__url__"],
download_url=download_url,
keywords=[
"Archive Website",
"Wayback Machine",
"Internet Archive",
"Wayback Machine CLI",
"Wayback Machine Python",
"Internet Archiving",
"Availability API",
"CDX API",
"savepagenow",
],
install_requires=["requests", "click"],
python_requires=">=3.4",
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Natural Language :: English",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: Implementation :: CPython",
],
entry_points={"console_scripts": ["waybackpy = waybackpy.cli:main"]},
project_urls={
"Documentation": "https://github.com/akamhy/waybackpy/wiki",
"Source": "https://github.com/akamhy/waybackpy",
"Tracker": "https://github.com/akamhy/waybackpy/issues",
},
)
setup()

View File

@ -1,19 +1,22 @@
import pytest
import random
import string
from datetime import datetime, timedelta
import pytest
from waybackpy.availability_api import WaybackMachineAvailabilityAPI
from waybackpy.exceptions import (
InvalidJSONInAvailabilityAPIResponse,
ArchiveNotInAvailabilityAPIResponse,
InvalidJSONInAvailabilityAPIResponse,
)
now = datetime.utcnow()
url = "https://example.com/"
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
rndstr = lambda n: "".join(
def rndstr(n):
return "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
)
@ -57,7 +60,7 @@ def test_invalid_json():
"""
with pytest.raises(InvalidJSONInAvailabilityAPIResponse):
availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent)
archive_url = availability_api.archive_url
_ = availability_api.archive_url
def test_no_archive():
@ -73,7 +76,7 @@ def test_no_archive():
availability_api = WaybackMachineAvailabilityAPI(
url="https://%s.cn" % rndstr(30), user_agent=user_agent
)
archive_url = availability_api.archive_url
_ = availability_api.archive_url
def test_no_api_call_str_repr():

View File

@ -1,4 +1,3 @@
import pytest
from datetime import datetime
from waybackpy.cdx_snapshot import CDXSnapshot

View File

@ -1,13 +1,14 @@
import pytest
from waybackpy.exceptions import WaybackError
from waybackpy.cdx_utils import (
get_total_pages,
check_collapses,
check_filters,
check_match_type,
full_url,
get_response,
check_filters,
check_collapses,
check_match_type,
get_total_pages,
)
from waybackpy.exceptions import WaybackError
def test_get_total_pages():
@ -86,10 +87,10 @@ def test_check_collapses():
def test_check_match_type():
assert None == check_match_type(None, "url")
assert check_match_type(None, "url") is None
match_type = "exact"
url = "test_url"
assert None == check_match_type(match_type, url)
assert check_match_type(match_type, url) is None
url = "has * in it"
with pytest.raises(WaybackError):

View File

@ -1,13 +1,16 @@
import pytest
import time
import random
import string
import time
from datetime import datetime
from waybackpy.save_api import WaybackMachineSaveAPI
from waybackpy.exceptions import MaximumSaveRetriesExceeded
import pytest
rndstr = lambda n: "".join(
from waybackpy.exceptions import MaximumSaveRetriesExceeded
from waybackpy.save_api import WaybackMachineSaveAPI
def rndstr(n):
return "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(n)
)
@ -23,8 +26,9 @@ def test_save():
cached_save = save_api.cached_save
assert cached_save in [True, False]
assert archive_url.find("github.com/akamhy/waybackpy") != -1
assert timestamp is not None
assert str(headers).find("github.com/akamhy/waybackpy") != -1
assert type(save_api.timestamp()) == type(datetime(year=2020, month=10, day=2))
assert isinstance(save_api.timestamp(), datetime)
def test_max_redirect_exceeded():

View File

@ -1,9 +1,9 @@
from waybackpy import __version__
from waybackpy.utils import (
latest_version_pypi,
latest_version_github,
DEFAULT_USER_AGENT,
latest_version_github,
latest_version_pypi,
)
from waybackpy.__version__ import __version__
def test_default_user_agent():

View File

@ -1,14 +1,37 @@
from .wrapper import Url
__title__ = "waybackpy"
__description__ = (
"Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
"Archive pages and retrieve archived pages easily."
)
__url__ = "https://akamhy.github.io/waybackpy/"
__version__ = "3.0.2"
__download_url__ = (
"https://github.com/akamhy/waybackpy/archive/{version}.tar.gz".format(
version=__version__
)
)
__author__ = "Akash Mahanty"
__author_email__ = "akamhy@yahoo.com"
__license__ = "MIT"
__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."
from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI
from .availability_api import WaybackMachineAvailabilityAPI
from .__version__ import (
__title__,
__description__,
__url__,
__version__,
__author__,
__author_email__,
__license__,
__copyright__,
)
from .wrapper import Url
__all__ = [
"__author__",
"__author_email__",
"__copyright__",
"__description__",
"__license__",
"__title__",
"__url__",
"__download_url__",
"__version__",
"WaybackMachineAvailabilityAPI",
"WaybackMachineCDXServerAPI",
"WaybackMachineSaveAPI",
"Url",
]

View File

@ -1,11 +0,0 @@
__title__ = "waybackpy"
__description__ = (
"Python package that interfaces with the Internet Archive's Wayback Machine APIs. "
"Archive pages and retrieve archived pages easily."
)
__url__ = "https://akamhy.github.io/waybackpy/"
__version__ = "3.0.2"
__author__ = "Akash Mahanty"
__author_email__ = "akamhy@yahoo.com"
__license__ = "MIT"
__copyright__ = "Copyright 2020-2022 Akash Mahanty et al."

View File

@ -1,12 +1,14 @@
import time
import json
import requests
import time
from datetime import datetime
from .utils import DEFAULT_USER_AGENT
import requests
from .exceptions import (
ArchiveNotInAvailabilityAPIResponse,
InvalidJSONInAvailabilityAPIResponse,
)
from .utils import DEFAULT_USER_AGENT
class WaybackMachineAvailabilityAPI:

View File

@ -1,14 +1,13 @@
from .exceptions import WaybackError
from .cdx_snapshot import CDXSnapshot
from .cdx_utils import (
get_total_pages,
get_response,
check_filters,
check_collapses,
check_filters,
check_match_type,
full_url,
get_response,
get_total_pages,
)
from .exceptions import WaybackError
from .utils import DEFAULT_USER_AGENT

View File

@ -1,7 +1,9 @@
import re
import requests
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from .exceptions import WaybackError
from .utils import DEFAULT_USER_AGENT

View File

@ -1,15 +1,17 @@
import click
import re
import os
import json as JSON
import os
import random
import requests
import re
import string
from .__version__ import __version__
from .utils import DEFAULT_USER_AGENT
import click
import requests
from . import __version__
from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI
from .availability_api import WaybackMachineAvailabilityAPI
from .utils import DEFAULT_USER_AGENT
from .wrapper import Url

View File

@ -1,13 +1,13 @@
import re
import time
import requests
from datetime import datetime
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from .utils import DEFAULT_USER_AGENT
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from .exceptions import MaximumSaveRetriesExceeded
from .utils import DEFAULT_USER_AGENT
class WaybackMachineSaveAPI:

View File

@ -1,5 +1,6 @@
import requests
from .__version__ import __version__
from . import __version__
DEFAULT_USER_AGENT = "waybackpy %s - https://github.com/akamhy/waybackpy" % __version__

View File

@ -1,8 +1,9 @@
from .save_api import WaybackMachineSaveAPI
from datetime import datetime, timedelta
from .availability_api import WaybackMachineAvailabilityAPI
from .cdx_api import WaybackMachineCDXServerAPI
from .save_api import WaybackMachineSaveAPI
from .utils import DEFAULT_USER_AGENT
from datetime import datetime, timedelta
"""
The Url class is not recommended to be used anymore, instead use the