diff options
| author | Stefano Rivera <stefanor@debian.org> | 2023-06-10 17:32:24 -0400 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2023-06-12 16:59:19 +0000 |
| commit | 461de18777946ee6b6eb3a2f55637e7eab86920f (patch) | |
| tree | 7eaff3c29a1deaad6f11bd06d88479bfb463ebbd | |
| parent | 672870762c4d577a1b3f0a92588f88398e1a479e (diff) | |
0.13.1-1 (patches unapplied)import/0.13.1-1ubuntu/mantic-proposedubuntu/mantic-develubuntu/mantic
Imported using git-ubuntu import.
Notes
Notes:
* New upstream release. (Closes: #1019705)
* Refresh patches.
* Migrate to flit (and built with pybuild-plugin-pyproject), following
upstream.
* Bump Standards-Version to 4.6.2, no changes needed.
44 files changed, 883 insertions, 585 deletions
diff --git a/.bumpversion.cfg b/.bumpversion.cfg index afa609b..437b22e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.12.12 -files = setup.py cachecontrol/__init__.py docs/conf.py +current_version = 0.13.0 +files = cachecontrol/__init__.py docs/conf.py commit = True tag = True diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..08c05fb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +on: + release: + types: + - published + +name: release + +jobs: + pypi: + name: upload release to PyPI + runs-on: ubuntu-latest + + permissions: + # Used to authenticate to PyPI via OIDC. + # Used to sign the release's artifacts with sigstore-python. + id-token: write + + # Used to attach signing artifacts to the published release. + contents: write + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: ">= 3.6" + + - name: deps + run: python -m pip install -U build + + - name: build + run: python -m build + + - name: publish + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: sign + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: ./dist/*.tar.gz ./dist/*.whl + release-signing-artifacts: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e20117e..01611fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,13 +14,14 @@ jobs: runs-on: "${{ matrix.os }}" strategy: + fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - os: ["macos-10.15", "windows-latest", "ubuntu-latest"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: ["macos-latest", "windows-latest", "ubuntu-latest"] steps: - - uses: "actions/checkout@v2" - - uses: "actions/setup-python@v2" + - uses: "actions/checkout@v3" + - uses: "actions/setup-python@v4" with: python-version: "${{ matrix.python-version }}" - name: "Install dependencies" @@ -28,7 +29,7 @@ jobs: python -VV python -m site python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade virtualenv tox tox-gh-actions + python -m pip install --upgrade virtualenv tox tox-gh-actions - name: "Run tox targets for ${{ matrix.python-version }}" run: "python -m tox" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4630561 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,86 @@ +# Contributing to CacheControl + +Thank you for your interest in contributing to `CacheControl`! + +The information below will help you set up a local development environment +and perform common development tasks. + +## Requirements + +`CacheControl`'s only external development requirement is Python 3.7 or newer. + +## Development steps + +First, clone this repository: + +```bash +git clone https://github.com/psf/cachecontrol +cd cachecontrol +``` + +Then, bootstrap your local development environment: + +```bash +make bootstrap +# OPTIONAL: enter the new environment, if you'd like to run things directly +source .venv/bin/activate +``` + +Once you've run `make bootstrap`, you can run the other `make` targets +to perform particular tasks. + +Any changes you make to the `cachecontrol` source tree will take effect +immediately in the development environment. + +### Linting + +You can run the current formatters with: + +```bash +make format +``` + +### Testing + +You can run the unit tests locally with: + +```bash +# run the test suite in the current environment +make test + +# OPTIONAL: use `tox` to fan out across multiple interpreters +make test-all +``` + +### Documentation + +You can build the Sphinx-based documentation with: + +```bash +# puts the generated HTML in docs/_build/html/ +make doc +``` + +### Releasing + +**NOTE**: If you're a non-maintaining contributor, you don't need the steps +here! They're documented for completeness and for onboarding future maintainers. + +Releases of `CacheControl` are managed by GitHub Actions. + +To perform a release: + +1. Update `CacheControl`'s `__version__` attribute. It can be found under + `cachecontrol/__init__.py`. + +1. Create a new tag corresponding to your new version, with a `v` prefix. For example: + + ```bash + # IMPORTANT: don't forget the `v` prefix! + git tag v1.2.3 + ``` + +1. Push your changes to `master` and to the new remote tag. + +1. Create, save, and publish a GitHub release for your new tag, including any + `CHANGELOG` entries. @@ -7,13 +7,14 @@ VENV_CMD=python3 -m venv ACTIVATE = $(VENV)/bin/activate CHEESE=https://pypi.python.org/pypi BUMPTYPE=patch +BUMPPRE=0 $(VENV)/bin/pip3: $(VENV_CMD) $(VENV) bootstrap: $(VENV)/bin/pip3 - $(VENV)/bin/pip3 install -r dev_requirements.txt + $(VENV)/bin/pip3 install -e .[dev] format: $(VENV)/bin/black . @@ -49,13 +50,6 @@ test: coverage: $(VENV)/bin/py.test --cov cachecontrol -release: dist - $(VENV)/bin/twine upload dist/* - dist: clean - $(VENV)/bin/python setup.py sdist bdist_wheel + $(VENV)/bin/python -m build ls -l dist - -bump: - $(VENV)/bin/bumpversion $(BUMPTYPE) - git push && git push --tags @@ -11,8 +11,8 @@ :target: https://pypi.python.org/pypi/cachecontrol :alt: Latest Version -.. image:: https://travis-ci.org/ionrock/cachecontrol.png?branch=master - :target: https://travis-ci.org/ionrock/cachecontrol +.. image:: https://github.com/psf/cachecontrol/actions/workflows/tests.yml/badge.svg + :target: https://github.com/psf/cachecontrol/actions/workflows/tests.yml CacheControl is a port of the caching algorithms in httplib2_ for use with requests_ session object. diff --git a/cachecontrol/__init__.py b/cachecontrol/__init__.py index 4e4b779..abbf8c0 100644 --- a/cachecontrol/__init__.py +++ b/cachecontrol/__init__.py @@ -8,11 +8,21 @@ Make it easy to import from cachecontrol without long namespaces. """ __author__ = "Eric Larson" __email__ = "eric@ionrock.org" -__version__ = "0.12.12" +__version__ = "0.13.1" -from .wrapper import CacheControl -from .adapter import CacheControlAdapter -from .controller import CacheController +from cachecontrol.adapter import CacheControlAdapter +from cachecontrol.controller import CacheController +from cachecontrol.wrapper import CacheControl + +__all__ = [ + "__author__", + "__email__", + "__version__", + "CacheControlAdapter", + "CacheController", + "CacheControl", +] import logging + logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/cachecontrol/_cmd.py b/cachecontrol/_cmd.py index ccee007..684a4a8 100644 --- a/cachecontrol/_cmd.py +++ b/cachecontrol/_cmd.py @@ -1,8 +1,11 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import logging +from argparse import ArgumentParser +from typing import TYPE_CHECKING import requests @@ -10,16 +13,19 @@ from cachecontrol.adapter import CacheControlAdapter from cachecontrol.cache import DictCache from cachecontrol.controller import logger -from argparse import ArgumentParser +if TYPE_CHECKING: + from argparse import Namespace + from cachecontrol.controller import CacheController -def setup_logging(): + +def setup_logging() -> None: logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() logger.addHandler(handler) -def get_session(): +def get_session() -> requests.Session: adapter = CacheControlAdapter( DictCache(), cache_etags=True, serializer=None, heuristic=None ) @@ -27,17 +33,17 @@ def get_session(): sess.mount("http://", adapter) sess.mount("https://", adapter) - sess.cache_controller = adapter.controller + sess.cache_controller = adapter.controller # type: ignore[attr-defined] return sess -def get_args(): +def get_args() -> Namespace: parser = ArgumentParser() parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() -def main(args=None): +def main() -> None: args = get_args() sess = get_session() @@ -48,10 +54,13 @@ def main(args=None): setup_logging() # try setting the cache - sess.cache_controller.cache_response(resp.request, resp.raw) + cache_controller: CacheController = ( + sess.cache_controller # type: ignore[attr-defined] + ) + cache_controller.cache_response(resp.request, resp.raw) # Now try to get it - if sess.cache_controller.cached_request(resp.request): + if cache_controller.cached_request(resp.request): print("Cached!") else: print("Not cached :(") diff --git a/cachecontrol/adapter.py b/cachecontrol/adapter.py index 22b4963..bf4a23d 100644 --- a/cachecontrol/adapter.py +++ b/cachecontrol/adapter.py @@ -1,16 +1,26 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -import types import functools +import types import zlib +from typing import TYPE_CHECKING, Any, Collection, Mapping from requests.adapters import HTTPAdapter -from .controller import CacheController, PERMANENT_REDIRECT_STATUSES -from .cache import DictCache -from .filewrapper import CallbackFileWrapper +from cachecontrol.cache import DictCache +from cachecontrol.controller import PERMANENT_REDIRECT_STATUSES, CacheController +from cachecontrol.filewrapper import CallbackFileWrapper + +if TYPE_CHECKING: + from requests import PreparedRequest, Response + from urllib3 import HTTPResponse + + from cachecontrol.cache import BaseCache + from cachecontrol.heuristics import BaseHeuristic + from cachecontrol.serialize import Serializer class CacheControlAdapter(HTTPAdapter): @@ -18,16 +28,16 @@ class CacheControlAdapter(HTTPAdapter): def __init__( self, - cache=None, - cache_etags=True, - controller_class=None, - serializer=None, - heuristic=None, - cacheable_methods=None, - *args, - **kw - ): - super(CacheControlAdapter, self).__init__(*args, **kw) + cache: BaseCache | None = None, + cache_etags: bool = True, + controller_class: type[CacheController] | None = None, + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + cacheable_methods: Collection[str] | None = None, + *args: Any, + **kw: Any, + ) -> None: + super().__init__(*args, **kw) self.cache = DictCache() if cache is None else cache self.heuristic = heuristic self.cacheable_methods = cacheable_methods or ("GET",) @@ -37,7 +47,16 @@ class CacheControlAdapter(HTTPAdapter): self.cache, cache_etags=cache_etags, serializer=serializer ) - def send(self, request, cacheable_methods=None, **kw): + def send( + self, + request: PreparedRequest, + stream: bool = False, + timeout: None | float | tuple[float, float] | tuple[float, None] = None, + verify: bool | str = True, + cert: (None | bytes | str | tuple[bytes | str, bytes | str]) = None, + proxies: Mapping[str, str] | None = None, + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Send a request. Use the request information to see if it exists in the cache and cache the response if we need to and can. @@ -54,13 +73,17 @@ class CacheControlAdapter(HTTPAdapter): # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) - resp = super(CacheControlAdapter, self).send(request, **kw) + resp = super().send(request, stream, timeout, verify, cert, proxies) return resp def build_response( - self, request, response, from_cache=False, cacheable_methods=None - ): + self, + request: PreparedRequest, + response: HTTPResponse, + from_cache: bool = False, + cacheable_methods: Collection[str] | None = None, + ) -> Response: """ Build a response by making a request or using the cache. @@ -102,36 +125,37 @@ class CacheControlAdapter(HTTPAdapter): else: # Wrap the response file with a wrapper that will cache the # response when the stream has been consumed. - response._fp = CallbackFileWrapper( - response._fp, + response._fp = CallbackFileWrapper( # type: ignore[attr-defined] + response._fp, # type: ignore[attr-defined] functools.partial( self.controller.cache_response, request, response ), ) if response.chunked: - super_update_chunk_length = response._update_chunk_length + super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined] - def _update_chunk_length(self): + def _update_chunk_length(self: HTTPResponse) -> None: super_update_chunk_length() if self.chunk_left == 0: - self._fp._close() + self._fp._close() # type: ignore[attr-defined] - response._update_chunk_length = types.MethodType( + response._update_chunk_length = types.MethodType( # type: ignore[attr-defined] _update_chunk_length, response ) - resp = super(CacheControlAdapter, self).build_response(request, response) + resp: Response = super().build_response(request, response) # type: ignore[no-untyped-call] # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: + assert request.url is not None cache_url = self.controller.cache_url(request.url) self.cache.delete(cache_url) # Give the request a from_cache attr to let people use it - resp.from_cache = from_cache + resp.from_cache = from_cache # type: ignore[attr-defined] return resp - def close(self): + def close(self) -> None: self.cache.close() - super(CacheControlAdapter, self).close() + super().close() # type: ignore[no-untyped-call] diff --git a/cachecontrol/cache.py b/cachecontrol/cache.py index 2a965f5..3293b00 100644 --- a/cachecontrol/cache.py +++ b/cachecontrol/cache.py @@ -6,38 +6,46 @@ The cache object API for implementing caches. The default is a thread safe in-memory dictionary. """ +from __future__ import annotations + from threading import Lock +from typing import IO, TYPE_CHECKING, MutableMapping +if TYPE_CHECKING: + from datetime import datetime -class BaseCache(object): - def get(self, key): +class BaseCache: + def get(self, key: str) -> bytes | None: raise NotImplementedError() - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: int | datetime | None = None + ) -> None: raise NotImplementedError() - def delete(self, key): + def delete(self, key: str) -> None: raise NotImplementedError() - def close(self): + def close(self) -> None: pass class DictCache(BaseCache): - - def __init__(self, init_dict=None): + def __init__(self, init_dict: MutableMapping[str, bytes] | None = None) -> None: self.lock = Lock() self.data = init_dict or {} - def get(self, key): + def get(self, key: str) -> bytes | None: return self.data.get(key, None) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: int | datetime | None = None + ) -> None: with self.lock: self.data.update({key: value}) - def delete(self, key): + def delete(self, key: str) -> None: with self.lock: if key in self.data: self.data.pop(key) @@ -55,10 +63,11 @@ class SeparateBodyBaseCache(BaseCache): Similarly, the body should be loaded separately via ``get_body()``. """ - def set_body(self, key, body): + + def set_body(self, key: str, body: bytes) -> None: raise NotImplementedError() - def get_body(self, key): + def get_body(self, key: str) -> IO[bytes] | None: """ Return the body as file-like object. """ diff --git a/cachecontrol/caches/__init__.py b/cachecontrol/caches/__init__.py index 3782729..44a1af8 100644 --- a/cachecontrol/caches/__init__.py +++ b/cachecontrol/caches/__init__.py @@ -2,8 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from .file_cache import FileCache, SeparateBodyFileCache -from .redis_cache import RedisCache - +from cachecontrol.caches.file_cache import FileCache, SeparateBodyFileCache +from cachecontrol.caches.redis_cache import RedisCache __all__ = ["FileCache", "SeparateBodyFileCache", "RedisCache"] diff --git a/cachecontrol/caches/file_cache.py b/cachecontrol/caches/file_cache.py index f82e7b8..a4ddb5e 100644 --- a/cachecontrol/caches/file_cache.py +++ b/cachecontrol/caches/file_cache.py @@ -1,22 +1,23 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import hashlib import os from textwrap import dedent +from typing import IO, TYPE_CHECKING -from ..cache import BaseCache, SeparateBodyBaseCache -from ..controller import CacheController +from cachecontrol.cache import BaseCache, SeparateBodyBaseCache +from cachecontrol.controller import CacheController -try: - FileNotFoundError -except NameError: - # py2.X - FileNotFoundError = (IOError, OSError) +if TYPE_CHECKING: + from datetime import datetime + from filelock import BaseFileLock -def _secure_open_write(filename, fmode): + +def _secure_open_write(filename: str, fmode: int) -> IO[bytes]: # We only want to write to this file, so open it in write only mode flags = os.O_WRONLY @@ -39,7 +40,7 @@ def _secure_open_write(filename, fmode): # there try: os.remove(filename) - except (IOError, OSError): + except OSError: # The file must not exist already, so we can just skip ahead to opening pass @@ -62,16 +63,16 @@ class _FileCacheMixin: def __init__( self, - directory, - forever=False, - filemode=0o0600, - dirmode=0o0700, - lock_class=None, - ): - + directory: str, + forever: bool = False, + filemode: int = 0o0600, + dirmode: int = 0o0700, + lock_class: type[BaseFileLock] | None = None, + ) -> None: try: if lock_class is None: from filelock import FileLock + lock_class = FileLock except ImportError: notice = dedent( @@ -90,17 +91,17 @@ class _FileCacheMixin: self.lock_class = lock_class @staticmethod - def encode(x): + def encode(x: str) -> str: return hashlib.sha224(x.encode()).hexdigest() - def _fn(self, name): + def _fn(self, name: str) -> str: # NOTE: This method should not change as some may depend on it. # See: https://github.com/ionrock/cachecontrol/issues/63 hashed = self.encode(name) parts = list(hashed[:5]) + [hashed] return os.path.join(self.directory, *parts) - def get(self, key): + def get(self, key: str) -> bytes | None: name = self._fn(key) try: with open(name, "rb") as fh: @@ -109,18 +110,20 @@ class _FileCacheMixin: except FileNotFoundError: return None - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: int | datetime | None = None + ) -> None: name = self._fn(key) self._write(name, value) - def _write(self, path, data: bytes): + def _write(self, path: str, data: bytes) -> None: """ Safely write the data to the given path. """ # Make sure the directory exists try: os.makedirs(os.path.dirname(path), self.dirmode) - except (IOError, OSError): + except OSError: pass with self.lock_class(path + ".lock"): @@ -128,7 +131,7 @@ class _FileCacheMixin: with _secure_open_write(path, self.filemode) as fh: fh.write(data) - def _delete(self, key, suffix): + def _delete(self, key: str, suffix: str) -> None: name = self._fn(key) + suffix if not self.forever: try: @@ -143,7 +146,7 @@ class FileCache(_FileCacheMixin, BaseCache): downloads. """ - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") @@ -153,23 +156,23 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache): peak memory usage. """ - def get_body(self, key): + def get_body(self, key: str) -> IO[bytes] | None: name = self._fn(key) + ".body" try: return open(name, "rb") except FileNotFoundError: return None - def set_body(self, key, body): + def set_body(self, key: str, body: bytes) -> None: name = self._fn(key) + ".body" self._write(name, body) - def delete(self, key): + def delete(self, key: str) -> None: self._delete(key, "") self._delete(key, ".body") -def url_to_file_path(url, filecache): +def url_to_file_path(url: str, filecache: FileCache) -> str: """Return the file cache path based on the URL. This does not ensure the file exists! diff --git a/cachecontrol/caches/redis_cache.py b/cachecontrol/caches/redis_cache.py index 7bcb38a..f859e71 100644 --- a/cachecontrol/caches/redis_cache.py +++ b/cachecontrol/caches/redis_cache.py @@ -1,39 +1,48 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from __future__ import division -from datetime import datetime +from datetime import datetime, timezone +from typing import TYPE_CHECKING + from cachecontrol.cache import BaseCache +if TYPE_CHECKING: + from redis import Redis -class RedisCache(BaseCache): - def __init__(self, conn): +class RedisCache(BaseCache): + def __init__(self, conn: Redis[bytes]) -> None: self.conn = conn - def get(self, key): + def get(self, key: str) -> bytes | None: return self.conn.get(key) - def set(self, key, value, expires=None): + def set( + self, key: str, value: bytes, expires: int | datetime | None = None + ) -> None: if not expires: self.conn.set(key, value) elif isinstance(expires, datetime): - expires = expires - datetime.utcnow() - self.conn.setex(key, int(expires.total_seconds()), value) + now_utc = datetime.now(timezone.utc) + if expires.tzinfo is None: + now_utc = now_utc.replace(tzinfo=None) + delta = expires - now_utc + self.conn.setex(key, int(delta.total_seconds()), value) else: self.conn.setex(key, expires, value) - def delete(self, key): + def delete(self, key: str) -> None: self.conn.delete(key) - def clear(self): + def clear(self) -> None: """Helper for clearing all the keys in a database. Use with caution!""" for key in self.conn.keys(): self.conn.delete(key) - def close(self): + def close(self) -> None: """Redis uses connection pooling, no need to close the connection.""" pass diff --git a/cachecontrol/compat.py b/cachecontrol/compat.py deleted file mode 100644 index 72c456c..0000000 --- a/cachecontrol/compat.py +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - -try: - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - - -try: - import cPickle as pickle -except ImportError: - import pickle - -# Handle the case where the requests module has been patched to not have -# urllib3 bundled as part of its source. -try: - from requests.packages.urllib3.response import HTTPResponse -except ImportError: - from urllib3.response import HTTPResponse - -try: - from requests.packages.urllib3.util import is_fp_closed -except ImportError: - from urllib3.util import is_fp_closed - -# Replicate some six behaviour -try: - text_type = unicode -except NameError: - text_type = str diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py index 184fe66..1de50ce 100644 --- a/cachecontrol/controller.py +++ b/cachecontrol/controller.py @@ -5,17 +5,27 @@ """ The httplib2 algorithms ported for use with requests. """ +from __future__ import annotations + +import calendar import logging import re -import calendar import time from email.utils import parsedate_tz +from typing import TYPE_CHECKING, Collection, Mapping from requests.structures import CaseInsensitiveDict -from .cache import DictCache, SeparateBodyBaseCache -from .serialize import Serializer +from cachecontrol.cache import DictCache, SeparateBodyBaseCache +from cachecontrol.serialize import Serializer + +if TYPE_CHECKING: + from typing import Literal + from requests import PreparedRequest + from urllib3 import HTTPResponse + + from cachecontrol.cache import BaseCache logger = logging.getLogger(__name__) @@ -24,20 +34,26 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) -def parse_uri(uri): +def parse_uri(uri: str) -> tuple[str, str, str, str, str]: """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) """ - groups = URI.match(uri).groups() + match = URI.match(uri) + assert match is not None + groups = match.groups() return (groups[1], groups[3], groups[4], groups[6], groups[8]) -class CacheController(object): +class CacheController: """An interface to see if request should cached or not.""" def __init__( - self, cache=None, cache_etags=True, serializer=None, status_codes=None + self, + cache: BaseCache | None = None, + cache_etags: bool = True, + serializer: Serializer | None = None, + status_codes: Collection[int] | None = None, ): self.cache = DictCache() if cache is None else cache self.cache_etags = cache_etags @@ -45,7 +61,7 @@ class CacheController(object): self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308) @classmethod - def _urlnorm(cls, uri): + def _urlnorm(cls, uri: str) -> str: """Normalize the URL to create a safe key for the cache""" (scheme, authority, path, query, fragment) = parse_uri(uri) if not scheme or not authority: @@ -65,10 +81,10 @@ class CacheController(object): return defrag_uri @classmethod - def cache_url(cls, uri): + def cache_url(cls, uri: str) -> str: return cls._urlnorm(uri) - def parse_cache_control(self, headers): + def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]: known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 "max-age": (int, True), @@ -87,7 +103,7 @@ class CacheController(object): cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) - retval = {} + retval: dict[str, int | None] = {} for cc_directive in cc_headers.split(","): if not cc_directive.strip(): @@ -122,11 +138,12 @@ class CacheController(object): return retval - def _load_from_cache(self, request): + def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None: """ Load a cached response, or return None if it's not available. """ cache_url = request.url + assert cache_url is not None cache_data = self.cache.get(cache_url) if cache_data is None: logger.debug("No cache entry available") @@ -142,11 +159,12 @@ class CacheController(object): logger.warning("Cache entry deserialization failed, entry ignored") return result - def cached_request(self, request): + def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]: """ Return a cached response if it exists in the cache, otherwise return False. """ + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Looking up "%s" in the cache', cache_url) cc = self.parse_cache_control(request.headers) @@ -182,7 +200,7 @@ class CacheController(object): logger.debug(msg) return resp - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if not headers or "date" not in headers: if "etag" not in headers: # Without date or etag, the cached response can never be used @@ -193,7 +211,9 @@ class CacheController(object): return False now = time.time() - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) current_age = max(0, now - date) logger.debug("Current age based on date: %i", current_age) @@ -207,28 +227,30 @@ class CacheController(object): freshness_lifetime = 0 # Check the max-age pragma in the cache control header - if "max-age" in resp_cc: - freshness_lifetime = resp_cc["max-age"] + max_age = resp_cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime) # If there isn't a max-age, check for an expires header elif "expires" in headers: expires = parsedate_tz(headers["expires"]) if expires is not None: - expire_time = calendar.timegm(expires) - date + expire_time = calendar.timegm(expires[:6]) - date freshness_lifetime = max(0, expire_time) logger.debug("Freshness lifetime from expires: %i", freshness_lifetime) # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. - if "max-age" in cc: - freshness_lifetime = cc["max-age"] + max_age = cc.get("max-age") + if max_age is not None: + freshness_lifetime = max_age logger.debug( "Freshness lifetime from request max-age: %i", freshness_lifetime ) - if "min-fresh" in cc: - min_fresh = cc["min-fresh"] + min_fresh = cc.get("min-fresh") + if min_fresh is not None: # adjust our current age by our min fresh current_age += min_fresh logger.debug("Adjusted current age from min-fresh: %i", current_age) @@ -247,12 +269,12 @@ class CacheController(object): # return the original handler return False - def conditional_headers(self, request): + def conditional_headers(self, request: PreparedRequest) -> dict[str, str]: resp = self._load_from_cache(request) new_headers = {} if resp: - headers = CaseInsensitiveDict(resp.headers) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers) if "etag" in headers: new_headers["If-None-Match"] = headers["ETag"] @@ -262,7 +284,14 @@ class CacheController(object): return new_headers - def _cache_set(self, cache_url, request, response, body=None, expires_time=None): + def _cache_set( + self, + cache_url: str, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + expires_time: int | None = None, + ) -> None: """ Store the data in the cache. """ @@ -285,7 +314,13 @@ class CacheController(object): expires=expires_time, ) - def cache_response(self, request, response, body=None, status_codes=None): + def cache_response( + self, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + status_codes: Collection[int] | None = None, + ) -> None: """ Algorithm for caching requests. @@ -300,10 +335,14 @@ class CacheController(object): ) return - response_headers = CaseInsensitiveDict(response.headers) + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) else: date = 0 @@ -322,6 +361,7 @@ class CacheController(object): cc_req = self.parse_cache_control(request.headers) cc = self.parse_cache_control(response_headers) + assert request.url is not None cache_url = self.cache_url(request.url) logger.debug('Updating cache with response from "%s"', cache_url) @@ -354,11 +394,11 @@ class CacheController(object): if response_headers.get("expires"): expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date expires_time = max(expires_time, 14 * 86400) - logger.debug("etag object cached for {0} seconds".format(expires_time)) + logger.debug(f"etag object cached for {expires_time} seconds") logger.debug("Caching due to etag") self._cache_set(cache_url, request, response, body, expires_time) @@ -372,11 +412,14 @@ class CacheController(object): # is no date header then we can't do anything about expiring # the cache. elif "date" in response_headers: - date = calendar.timegm(parsedate_tz(response_headers["date"])) + time_tuple = parsedate_tz(response_headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) # cache when there is a max-age > 0 - if "max-age" in cc and cc["max-age"] > 0: + max_age = cc.get("max-age") + if max_age is not None and max_age > 0: logger.debug("Caching b/c date exists and max-age > 0") - expires_time = cc["max-age"] + expires_time = max_age self._cache_set( cache_url, request, @@ -391,12 +434,12 @@ class CacheController(object): if response_headers["expires"]: expires = parsedate_tz(response_headers["expires"]) if expires is not None: - expires_time = calendar.timegm(expires) - date + expires_time = calendar.timegm(expires[:6]) - date else: expires_time = None logger.debug( - "Caching b/c of expires header. expires in {0} seconds".format( + "Caching b/c of expires header. expires in {} seconds".format( expires_time ) ) @@ -408,13 +451,16 @@ class CacheController(object): expires_time, ) - def update_cached_response(self, request, response): + def update_cached_response( + self, request: PreparedRequest, response: HTTPResponse + ) -> HTTPResponse: """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. This should only ever be called when we've sent an ETag and gotten a 304 as the response. """ + assert request.url is not None cache_url = self.cache_url(request.url) cached_response = self._load_from_cache(request) @@ -432,11 +478,11 @@ class CacheController(object): excluded_headers = ["content-length"] cached_response.headers.update( - dict( - (k, v) - for k, v in response.headers.items() + { + k: v + for k, v in response.headers.items() # type: ignore[no-untyped-call] if k.lower() not in excluded_headers - ) + } ) # we want a 200 b/c we have content via the cache diff --git a/cachecontrol/filewrapper.py b/cachecontrol/filewrapper.py index f5ed5f6..2514390 100644 --- a/cachecontrol/filewrapper.py +++ b/cachecontrol/filewrapper.py @@ -1,12 +1,17 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from tempfile import NamedTemporaryFile import mmap +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from http.client import HTTPResponse -class CallbackFileWrapper(object): +class CallbackFileWrapper: """ Small wrapper around a fp object which will tee everything read into a buffer, and when that file is closed it will execute a callback with the @@ -25,12 +30,14 @@ class CallbackFileWrapper(object): performance impact. """ - def __init__(self, fp, callback): + def __init__( + self, fp: HTTPResponse, callback: Callable[[bytes], None] | None + ) -> None: self.__buf = NamedTemporaryFile("rb+", delete=True) self.__fp = fp self.__callback = callback - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: # The vaguaries of garbage collection means that self.__fp is # not always set. By using __getattribute__ and the private # name[0] allows looking up the attribute value and raising an @@ -42,7 +49,7 @@ class CallbackFileWrapper(object): fp = self.__getattribute__("_CallbackFileWrapper__fp") return getattr(fp, name) - def __is_fp_closed(self): + def __is_fp_closed(self) -> bool: try: return self.__fp.fp is None @@ -50,7 +57,8 @@ class CallbackFileWrapper(object): pass try: - return self.__fp.closed + closed: bool = self.__fp.closed + return closed except AttributeError: pass @@ -59,7 +67,7 @@ class CallbackFileWrapper(object): # TODO: Add some logging here... return False - def _close(self): + def _close(self) -> None: if self.__callback: if self.__buf.tell() == 0: # Empty file: @@ -86,8 +94,8 @@ class CallbackFileWrapper(object): # Important when caching big files. self.__buf.close() - def read(self, amt=None): - data = self.__fp.read(amt) + def read(self, amt: int | None = None) -> bytes: + data: bytes = self.__fp.read(amt) if data: # We may be dealing with b'', a sign that things are over: # it's passed e.g. after we've already closed self.__buf. @@ -97,8 +105,8 @@ class CallbackFileWrapper(object): return data - def _safe_read(self, amt): - data = self.__fp._safe_read(amt) + def _safe_read(self, amt: int) -> bytes: + data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined] if amt == 2 and data == b"\r\n": # urllib executes this read to toss the CRLF at the end # of the chunk. diff --git a/cachecontrol/heuristics.py b/cachecontrol/heuristics.py index ebe4a96..323262b 100644 --- a/cachecontrol/heuristics.py +++ b/cachecontrol/heuristics.py @@ -1,29 +1,31 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import calendar import time - +from datetime import datetime, timedelta, timezone from email.utils import formatdate, parsedate, parsedate_tz +from typing import TYPE_CHECKING, Any, Mapping -from datetime import datetime, timedelta +if TYPE_CHECKING: + from urllib3 import HTTPResponse TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -def expire_after(delta, date=None): - date = date or datetime.utcnow() +def expire_after(delta: timedelta, date: datetime | None = None) -> datetime: + date = date or datetime.now(timezone.utc) return date + delta -def datetime_to_header(dt): +def datetime_to_header(dt: datetime) -> str: return formatdate(calendar.timegm(dt.timetuple())) -class BaseHeuristic(object): - - def warning(self, response): +class BaseHeuristic: + def warning(self, response: HTTPResponse) -> str | None: """ Return a valid 1xx warning header value describing the cache adjustments. @@ -34,7 +36,7 @@ class BaseHeuristic(object): """ return '110 - "Response is Stale"' - def update_headers(self, response): + def update_headers(self, response: HTTPResponse) -> dict[str, str]: """Update the response headers with any new headers. NOTE: This SHOULD always include some Warning header to @@ -43,7 +45,7 @@ class BaseHeuristic(object): """ return {} - def apply(self, response): + def apply(self, response: HTTPResponse) -> HTTPResponse: updated_headers = self.update_headers(response) if updated_headers: @@ -61,12 +63,12 @@ class OneDayCache(BaseHeuristic): future. """ - def update_headers(self, response): + def update_headers(self, response: HTTPResponse) -> dict[str, str]: headers = {} if "expires" not in response.headers: date = parsedate(response.headers["date"]) - expires = expire_after(timedelta(days=1), date=datetime(*date[:6])) + expires = expire_after(timedelta(days=1), date=datetime(*date[:6], tzinfo=timezone.utc)) # type: ignore[misc] headers["expires"] = datetime_to_header(expires) headers["cache-control"] = "public" return headers @@ -77,14 +79,14 @@ class ExpiresAfter(BaseHeuristic): Cache **all** requests for a defined time period. """ - def __init__(self, **kw): + def __init__(self, **kw: Any) -> None: self.delta = timedelta(**kw) - def update_headers(self, response): + def update_headers(self, response: HTTPResponse) -> dict[str, str]: expires = expire_after(self.delta) return {"expires": datetime_to_header(expires), "cache-control": "public"} - def warning(self, response): + def warning(self, response: HTTPResponse) -> str | None: tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -101,12 +103,23 @@ class LastModified(BaseHeuristic): http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397 Unlike mozilla we limit this to 24-hr. """ + cacheable_by_default_statuses = { - 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501 + 200, + 203, + 204, + 206, + 300, + 301, + 404, + 405, + 410, + 414, + 501, } - def update_headers(self, resp): - headers = resp.headers + def update_headers(self, resp: HTTPResponse) -> dict[str, str]: + headers: Mapping[str, str] = resp.headers if "expires" in headers: return {} @@ -120,9 +133,11 @@ class LastModified(BaseHeuristic): if "date" not in headers or "last-modified" not in headers: return {} - date = calendar.timegm(parsedate_tz(headers["date"])) + time_tuple = parsedate_tz(headers["date"]) + assert time_tuple is not None + date = calendar.timegm(time_tuple[:6]) last_modified = parsedate(headers["last-modified"]) - if date is None or last_modified is None: + if last_modified is None: return {} now = time.time() @@ -135,5 +150,5 @@ class LastModified(BaseHeuristic): expires = date + freshness_lifetime return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} - def warning(self, resp): + def warning(self, resp: HTTPResponse) -> str | None: return None diff --git a/cachecontrol/py.typed b/cachecontrol/py.typed new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/cachecontrol/py.typed diff --git a/cachecontrol/serialize.py b/cachecontrol/serialize.py index 70135d3..28d3dc9 100644 --- a/cachecontrol/serialize.py +++ b/cachecontrol/serialize.py @@ -1,78 +1,76 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -import base64 import io -import json -import zlib +from typing import IO, TYPE_CHECKING, Any, Mapping, cast import msgpack from requests.structures import CaseInsensitiveDict +from urllib3 import HTTPResponse -from .compat import HTTPResponse, pickle, text_type +if TYPE_CHECKING: + from requests import PreparedRequest -def _b64_decode_bytes(b): - return base64.b64decode(b.encode("ascii")) +class Serializer: + serde_version = "4" - -def _b64_decode_str(s): - return _b64_decode_bytes(s).decode("utf8") - - -_default_body_read = object() - - -class Serializer(object): - def dumps(self, request, response, body=None): - response_headers = CaseInsensitiveDict(response.headers) + def dumps( + self, + request: PreparedRequest, + response: HTTPResponse, + body: bytes | None = None, + ) -> bytes: + response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + response.headers + ) if body is None: # When a body isn't passed in, we'll read the response. We # also update the response with a new file handler to be # sure it acts as though it was never read. body = response.read(decode_content=False) - response._fp = io.BytesIO(body) - - # NOTE: This is all a bit weird, but it's really important that on - # Python 2.x these objects are unicode and not str, even when - # they contain only ascii. The problem here is that msgpack - # understands the difference between unicode and bytes and we - # have it set to differentiate between them, however Python 2 - # doesn't know the difference. Forcing these to unicode will be - # enough to have msgpack know the difference. + response._fp = io.BytesIO(body) # type: ignore[attr-defined] + response.length_remaining = len(body) + data = { - u"response": { - u"body": body, # Empty bytestring if body is stored separately - u"headers": dict( - (text_type(k), text_type(v)) for k, v in response.headers.items() - ), - u"status": response.status, - u"version": response.version, - u"reason": text_type(response.reason), - u"strict": response.strict, - u"decode_content": response.decode_content, + "response": { + "body": body, # Empty bytestring if body is stored separately + "headers": {str(k): str(v) for k, v in response.headers.items()}, # type: ignore[no-untyped-call] + "status": response.status, + "version": response.version, + "reason": str(response.reason), + "decode_content": response.decode_content, } } # Construct our vary headers - data[u"vary"] = {} - if u"vary" in response_headers: - varied_headers = response_headers[u"vary"].split(",") + data["vary"] = {} + if "vary" in response_headers: + varied_headers = response_headers["vary"].split(",") for header in varied_headers: - header = text_type(header).strip() + header = str(header).strip() header_value = request.headers.get(header, None) if header_value is not None: - header_value = text_type(header_value) - data[u"vary"][header] = header_value + header_value = str(header_value) + data["vary"][header] = header_value + + return b",".join([f"cc={self.serde_version}".encode(), self.serialize(data)]) - return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)]) + def serialize(self, data: dict[str, Any]) -> bytes: + return cast(bytes, msgpack.dumps(data, use_bin_type=True)) - def loads(self, request, data, body_file=None): + def loads( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: # Short circuit if we've been given an empty set of data if not data: - return + return None # Determine what version of the serializer the data was serialized # with @@ -88,18 +86,23 @@ class Serializer(object): ver = b"cc=0" # Get the version number out of the cc=N - ver = ver.split(b"=", 1)[-1].decode("ascii") + verstr = ver.split(b"=", 1)[-1].decode("ascii") # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{}".format(ver))(request, data, body_file) + return getattr(self, f"_loads_v{verstr}")(request, data, body_file) # type: ignore[no-any-return] except AttributeError: # This is a version we don't have a loads function for, so we'll # just treat it as a miss and return None - return - - def prepare_response(self, request, cached, body_file=None): + return None + + def prepare_response( + self, + request: PreparedRequest, + cached: Mapping[str, Any], + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: """Verify our vary headers match and construct a real urllib3 HTTPResponse object. """ @@ -108,23 +111,26 @@ class Serializer(object): # This case is also handled in the controller code when creating # a cache entry, but is left here for backwards compatibility. if "*" in cached.get("vary", {}): - return + return None # Ensure that the Vary headers for the cached response match our # request for header, value in cached.get("vary", {}).items(): if request.headers.get(header, None) != value: - return + return None body_raw = cached["response"].pop("body") - headers = CaseInsensitiveDict(data=cached["response"]["headers"]) + headers: CaseInsensitiveDict[str] = CaseInsensitiveDict( + data=cached["response"]["headers"] + ) if headers.get("transfer-encoding", "") == "chunked": headers.pop("transfer-encoding") cached["response"]["headers"] = headers try: + body: IO[bytes] if body_file is None: body = io.BytesIO(body_raw) else: @@ -138,53 +144,63 @@ class Serializer(object): # TypeError: 'str' does not support the buffer interface body = io.BytesIO(body_raw.encode("utf8")) + # Discard any `strict` parameter serialized by older version of cachecontrol. + cached["response"].pop("strict", None) + return HTTPResponse(body=body, preload_content=False, **cached["response"]) - def _loads_v0(self, request, data, body_file=None): + def _loads_v0( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> None: # The original legacy cache data. This doesn't contain enough # information to construct everything we need, so we'll treat this as # a miss. - return - - def _loads_v1(self, request, data, body_file=None): - try: - cached = pickle.loads(data) - except ValueError: - return - - return self.prepare_response(request, cached, body_file) - - def _loads_v2(self, request, data, body_file=None): - assert body_file is None - try: - cached = json.loads(zlib.decompress(data).decode("utf8")) - except (ValueError, zlib.error): - return - - # We need to decode the items that we've base64 encoded - cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) - cached["response"]["headers"] = dict( - (_b64_decode_str(k), _b64_decode_str(v)) - for k, v in cached["response"]["headers"].items() - ) - cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"]) - cached["vary"] = dict( - (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v) - for k, v in cached["vary"].items() - ) - - return self.prepare_response(request, cached, body_file) - - def _loads_v3(self, request, data, body_file): + return None + + def _loads_v1( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v1" pickled cache format. This is no longer supported + # for security reasons, so we treat it as a miss. + return None + + def _loads_v2( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: + # The "v2" compressed base64 cache format. + # This has been removed due to age and poor size/performance + # characteristics, so we treat it as a miss. + return None + + def _loads_v3( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> None: # Due to Python 2 encoding issues, it's impossible to know for sure # exactly how to load v3 entries, thus we'll treat these as a miss so # that they get rewritten out as v4 entries. - return - - def _loads_v4(self, request, data, body_file=None): + return None + + def _loads_v4( + self, + request: PreparedRequest, + data: bytes, + body_file: IO[bytes] | None = None, + ) -> HTTPResponse | None: try: cached = msgpack.loads(data, raw=False) except ValueError: - return + return None return self.prepare_response(request, cached, body_file) diff --git a/cachecontrol/wrapper.py b/cachecontrol/wrapper.py index b6ee7f2..37ee07c 100644 --- a/cachecontrol/wrapper.py +++ b/cachecontrol/wrapper.py @@ -1,22 +1,32 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations -from .adapter import CacheControlAdapter -from .cache import DictCache +from typing import TYPE_CHECKING, Collection +from cachecontrol.adapter import CacheControlAdapter +from cachecontrol.cache import DictCache -def CacheControl( - sess, - cache=None, - cache_etags=True, - serializer=None, - heuristic=None, - controller_class=None, - adapter_class=None, - cacheable_methods=None, -): +if TYPE_CHECKING: + import requests + + from cachecontrol.cache import BaseCache + from cachecontrol.controller import CacheController + from cachecontrol.heuristics import BaseHeuristic + from cachecontrol.serialize import Serializer + +def CacheControl( + sess: requests.Session, + cache: BaseCache | None = None, + cache_etags: bool = True, + serializer: Serializer | None = None, + heuristic: BaseHeuristic | None = None, + controller_class: type[CacheController] | None = None, + adapter_class: type[CacheControlAdapter] | None = None, + cacheable_methods: Collection[str] | None = None, +) -> requests.Session: cache = DictCache() if cache is None else cache adapter_class = adapter_class or CacheControlAdapter adapter = adapter_class( diff --git a/debian/changelog b/debian/changelog index d3c5e6a..46c040f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +python-cachecontrol (0.13.1-1) unstable; urgency=medium + + * New upstream release. (Closes: #1019705) + * Refresh patches. + * Migrate to flit (and built with pybuild-plugin-pyproject), following + upstream. + * Bump Standards-Version to 4.6.2, no changes needed. + + -- Stefano Rivera <stefanor@debian.org> Sat, 10 Jun 2023 17:32:24 -0400 + python-cachecontrol (0.12.12-2) unstable; urgency=medium * Support the nocheck build profile. diff --git a/debian/control b/debian/control index 9b8050b..f2a6573 100644 --- a/debian/control +++ b/debian/control @@ -7,6 +7,8 @@ Priority: optional Build-Depends: debhelper-compat (= 13), dh-python, + flit, + pybuild-plugin-pyproject, python3-all, python3-cherrypy3 <!nocheck>, python3-filelock <!nocheck>, @@ -15,8 +17,7 @@ Build-Depends: python3-pytest <!nocheck>, python3-redis <!nocheck>, python3-requests <!nocheck>, - python3-setuptools -Standards-Version: 4.6.1 +Standards-Version: 4.6.2 Rules-Requires-Root: no Vcs-Git: https://salsa.debian.org/python-team/packages/python-cachecontrol.git Vcs-Browser: https://salsa.debian.org/python-team/packages/python-cachecontrol diff --git a/debian/patches/no-doesitcache-script.patch b/debian/patches/no-doesitcache-script.patch index e8d4221..0206c0c 100644 --- a/debian/patches/no-doesitcache-script.patch +++ b/debian/patches/no-doesitcache-script.patch @@ -1,22 +1,22 @@ From: Barry Warsaw <barry@python.org> Date: Mon, 31 Oct 2016 17:31:16 -0400 -Subject: Edit setup.py entry_points. +Subject: Remove unnecessary scripts Don't install the undocumented and unnecessary doesitcache script. --- - setup.py | 2 +- + pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -diff --git a/setup.py b/setup.py -index 779d9bf..e002c24 100644 ---- a/setup.py -+++ b/setup.py -@@ -23,7 +23,7 @@ setup_params = dict( - long_description=long_description, - install_requires=["requests", "msgpack>=0.5.2"], - extras_require={"filecache": ["filelock>=3.8.0"], "redis": ["redis>=2.10.5"]}, -- entry_points={"console_scripts": ["doesitcache = cachecontrol._cmd:main"]}, -+ ## entry_points={"console_scripts": ["doesitcache = cachecontrol._cmd:main"]}, - python_requires=">=3.6", - classifiers=[ - "Development Status :: 4 - Beta", +diff --git a/pyproject.toml b/pyproject.toml +index ef55536..e44db2a 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -62,7 +62,7 @@ dev = [ + ] + + [project.scripts] +-doesitcache = "cachecontrol._cmd:main" ++#doesitcache = "cachecontrol._cmd:main" + + [tool.mypy] + show_error_codes = true diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 46d00b0..0000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - --e . - -tox -pytest-cov -pytest -mock -cherrypy -sphinx -redis -filelock -bumpversion -twine -black -wheel diff --git a/docs/conf.py b/docs/conf.py index 84d2434..957a728 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os + +from cachecontrol import __version__ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -44,17 +45,17 @@ source_suffix = ".rst" master_doc = "index" # General information about the project. -project = u"CacheControl" -copyright = u"2013, Eric Larson" +project = "CacheControl" +copyright = "2013, Eric Larson" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = "0.12.12" +version = __version__ # The full version, including alpha/beta/rc tags. -release = "0.12.12" +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -124,7 +125,7 @@ html_theme = "default" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -188,8 +189,8 @@ latex_documents = [ ( "index", "CacheControl.tex", - u"CacheControl Documentation", - u"Eric Larson", + "CacheControl Documentation", + "Eric Larson", "manual", ) ] @@ -220,7 +221,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ("index", "cachecontrol", u"CacheControl Documentation", [u"Eric Larson"], 1) + ("index", "cachecontrol", "CacheControl Documentation", ["Eric Larson"], 1) ] # If true, show URL addresses after external links. @@ -236,8 +237,8 @@ texinfo_documents = [ ( "index", "CacheControl", - u"CacheControl Documentation", - u"Eric Larson", + "CacheControl Documentation", + "Eric Larson", "CacheControl", "One line description of project.", "Miscellaneous", diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 6b2f9ff..be9e939 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -7,6 +7,28 @@ Release Notes =============== +0.13.1 +====== + +* Support for old serialization formats has been removed. +* Move the serialization implementation into own method. +* Drop support for Python older than 3.7. + +0.13.0 +====== + +**YANKED** + +The project has been moved to the `PSF <https://github.com/psf>`_ organization. + +* Discard the ``strict`` attribute when serializing and deserializing responses. +* Fix the ``IncompleteRead`` error thrown by ``urllib3 2.0``. +* Remove usage of ``utcnow`` in favor of timezone-aware datetimes. +* Remove the ``compat`` module. +* Use Python's ``unittest.mock`` library instead of ``mock``. +* Add type annotations. +* Exclude the ``tests`` directory from the wheel. + 0.12.11 ======= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ef55536 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "cachecontrol" + +[tool.flit.sdist] +include = ["tests/"] + +[project] +name = "CacheControl" +dynamic = ["version"] +description = "httplib2 caching for requests" +readme = "README.rst" +license = { file = "LICENSE.txt" } +authors = [ + { name = "Eric Larson", email = "ericlarson@ionrock.com" }, + { name = "Frost Ming", email = "me@frostming.com" }, + { name = "William Woodruff", email = "william@yossarian.net" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Internet :: WWW/HTTP", +] +keywords = ["requests", "http", "caching", "web"] +dependencies = ["requests >= 2.16.0", "msgpack >= 0.5.2"] +requires-python = ">=3.7" + +[project.urls] +Homepage = "https://pypi.org/project/CacheControl/" +Issues = "https://github.com/psf/cachecontrol/issues" +Source = "https://github.com/psf/cachecontrol" + +[project.optional-dependencies] +# End-user extras. +filecache = ["filelock >= 3.8.0"] +redis = ["redis>=2.10.5"] + +# Development extras. +dev = [ + "CacheControl[filecache,redis]", + "build", + "mypy", + "tox", + "pytest-cov", + "pytest", + "cherrypy", + "sphinx", + "black", + "types-redis", + "types-requests", +] + +[project.scripts] +doesitcache = "cachecontrol._cmd:main" + +[tool.mypy] +show_error_codes = true +strict = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + +[[tool.mypy.overrides]] +module = "msgpack" +ignore_missing_imports = true + +[tool.pytest.ini_options] +norecursedirs = ["bin", "lib", "include", "build"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 53862d7..0000000 --- a/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - -[metadata] -license_files = - LICENSE.txt - -[tool:pytest] -norecursedirs = bin lib include build - -[bdist_wheel] -universal = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 779d9bf..0000000 --- a/setup.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Eric Larson -# -# SPDX-License-Identifier: Apache-2.0 - -import setuptools - -long_description = open("README.rst").read() - -VERSION = "0.12.12" - -setup_params = dict( - name="CacheControl", - version=VERSION, - author="Eric Larson", - author_email="eric@ionrock.org", - url="https://github.com/ionrock/cachecontrol", - keywords="requests http caching web", - packages=setuptools.find_packages(), - package_data={"": ["LICENSE.txt"]}, - package_dir={"cachecontrol": "cachecontrol"}, - include_package_data=True, - description="httplib2 caching for requests", - long_description=long_description, - install_requires=["requests", "msgpack>=0.5.2"], - extras_require={"filecache": ["filelock>=3.8.0"], "redis": ["redis>=2.10.5"]}, - entry_points={"console_scripts": ["doesitcache = cachecontrol._cmd:main"]}, - python_requires=">=3.6", - classifiers=[ - "Development Status :: 4 - Beta", - "Environment :: Web Environment", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Internet :: WWW/HTTP", - ], -) - - -if __name__ == "__main__": - setuptools.setup(**setup_params) diff --git a/tests/conftest.py b/tests/conftest.py index 2f1de2d..e6231e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ import pytest import cherrypy -class SimpleApp(object): +class SimpleApp: def __init__(self): self.etag_count = 0 @@ -53,7 +53,7 @@ class SimpleApp(object): def update_etag_string(self): self.etag_count += 1 - self.etag_string = '"ETAG-{}"'.format(self.etag_count) + self.etag_string = f'"ETAG-{self.etag_count}"' def update_etag(self, env, start_response): self.update_etag_string() @@ -86,16 +86,16 @@ class SimpleApp(object): def permanent_redirect(self, env, start_response): headers = [("Location", "/permalink")] start_response("301 Moved Permanently", headers) - return ["See: /permalink".encode("utf-8")] + return [b"See: /permalink"] def permalink(self, env, start_response): start_response("200 OK", [("Content-Type", "text/plain")]) - return ["The permanent resource".encode("utf-8")] + return [b"The permanent resource"] def multiple_choices(self, env, start_response): headers = [("Link", "/permalink")] start_response("300 Multiple Choices", headers) - return ["See: /permalink".encode("utf-8")] + return [b"See: /permalink"] def stream(self, env, start_response): headers = [("Content-Type", "text/plain"), ("Cache-Control", "max-age=5000")] @@ -104,6 +104,16 @@ class SimpleApp(object): for i in range(10): yield pformat(i).encode("utf8") + def fixed_length(self, env, start_response): + body = b"0123456789" + headers = [ + ("Content-Type", "text/plain"), + ("Cache-Control", "max-age=5000"), + ("Content-Length", str(len(body))) + ] + start_response("200 OK", headers) + return [body] + def __call__(self, env, start_response): func = self.dispatch(env) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index a682057..fcac683 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -import mock +from unittest import mock import pytest from requests import Session @@ -34,7 +34,7 @@ def sess(url, request): sess.close() -class TestSessionActions(object): +class TestSessionActions: def test_get_caches(self, url, sess): r2 = sess.get(url) diff --git a/tests/test_cache_control.py b/tests/test_cache_control.py index 63ed3d8..7d893cd 100644 --- a/tests/test_cache_control.py +++ b/tests/test_cache_control.py @@ -5,21 +5,22 @@ """ Unit tests that verify our caching methods work correctly. """ -import pytest -from mock import ANY, Mock import time from tempfile import mkdtemp +from unittest.mock import ANY, Mock + +import pytest from cachecontrol import CacheController from cachecontrol.cache import DictCache from cachecontrol.caches import SeparateBodyFileCache -from .utils import NullSerializer, DummyResponse, DummyRequest -TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" +from .utils import DummyRequest, DummyResponse, NullSerializer +TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT" -class TestCacheControllerResponse(object): +class TestCacheControllerResponse: url = "http://url.com/" def req(self, headers=None): @@ -126,13 +127,16 @@ class TestCacheControllerResponse(object): cache = DictCache({}) cc = CacheController(cache) req = DummyRequest(url="http://localhost/", headers={"if-match": "xyz"}) - resp = DummyResponse(status=304, headers={ - "ETag": "xyz", - "x-value": "b", - "Date": time.strftime(TIME_FMT, time.gmtime()), - "Cache-Control": "max-age=60", - "Content-Length": "200" - }) + resp = DummyResponse( + status=304, + headers={ + "ETag": "xyz", + "x-value": "b", + "Date": time.strftime(TIME_FMT, time.gmtime()), + "Cache-Control": "max-age=60", + "Content-Length": "200", + }, + ) # First, ensure the response from update_cached_response() matches the # cached one: result = cc.update_cached_response(req, resp) @@ -176,13 +180,16 @@ class TestCacheControllerResponse(object): cc = CacheController(cache) url = "http://localhost:123/x" req = DummyRequest(url=url, headers={}) - cached_resp = DummyResponse(status=200, headers={ - "ETag": etag, - "x-value:": "a", - "Content-Length": "100", - "Cache-Control": "max-age=60", - "Date": time.strftime(TIME_FMT, time.gmtime()), - }) + cached_resp = DummyResponse( + status=200, + headers={ + "ETag": etag, + "x-value:": "a", + "Content-Length": "100", + "Cache-Control": "max-age=60", + "Date": time.strftime(TIME_FMT, time.gmtime()), + }, + ) cc._cache_set(url, req, cached_resp, b"my body") # Now we get another request, and it's a 304, with new value for @@ -191,13 +198,16 @@ class TestCacheControllerResponse(object): # Set our content length to 200. That would be a mistake in # the server, but we'll handle it gracefully... for now. req = DummyRequest(url=url, headers={"if-match": etag}) - resp = DummyResponse(status=304, headers={ - "ETag": etag, - "x-value": "b", - "Date": time.strftime(TIME_FMT, time.gmtime()), - "Cache-Control": "max-age=60", - "Content-Length": "200" - }) + resp = DummyResponse( + status=304, + headers={ + "ETag": etag, + "x-value": "b", + "Date": time.strftime(TIME_FMT, time.gmtime()), + "Cache-Control": "max-age=60", + "Content-Length": "200", + }, + ) # First, ensure the response from update_cached_response() matches the # cached one: result = cc.update_cached_response(req, resp) @@ -211,10 +221,10 @@ class TestCacheControllerResponse(object): assert r.read() == b"my body" -class TestCacheControlRequest(object): +class TestCacheControlRequest: url = "http://foo.com/bar" - def setup(self): + def setup_method(self): self.c = CacheController(DictCache(), serializer=NullSerializer()) def req(self, headers): @@ -222,7 +232,9 @@ class TestCacheControlRequest(object): return self.c.cached_request(mock_request) def test_cache_request_no_headers(self): - cached_resp = Mock(headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200) + cached_resp = Mock( + headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200 + ) self.c.cache = DictCache({self.url: cached_resp}) resp = self.req({}) assert not resp diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py index 4684087..f0be802 100644 --- a/tests/test_chunked_response.py +++ b/tests/test_chunked_response.py @@ -4,7 +4,6 @@ """ Test for supporting streamed responses (Transfer-Encoding: chunked) """ -from __future__ import print_function, unicode_literals import pytest import requests @@ -21,7 +20,7 @@ def sess(): sess.close() -class TestChunkedResponses(object): +class TestChunkedResponses: def test_cache_chunked_response(self, url, sess): """ diff --git a/tests/test_etag.py b/tests/test_etag.py index 7523bf8..b496311 100644 --- a/tests/test_etag.py +++ b/tests/test_etag.py @@ -1,20 +1,19 @@ # SPDX-FileCopyrightText: 2015 Eric Larson # # SPDX-License-Identifier: Apache-2.0 +from contextlib import ExitStack, suppress +from unittest.mock import Mock, patch +from urllib.parse import urljoin import pytest - -from mock import Mock, patch - import requests from cachecontrol import CacheControl from cachecontrol.cache import DictCache -from cachecontrol.compat import urljoin -from .utils import NullSerializer +from tests.utils import NullSerializer -class TestETag(object): +class TestETag: """Test our equal priority caching with ETags Equal Priority Caching is a term I've defined to describe when @@ -81,7 +80,7 @@ class TestETag(object): assert self.cache.get(self.etag_url) == resp.raw -class TestDisabledETags(object): +class TestDisabledETags: """Test our use of ETags when the response is stale and the response has an ETag. """ @@ -118,7 +117,7 @@ class TestDisabledETags(object): assert r.status_code == 200 -class TestReleaseConnection(object): +class TestReleaseConnection: """ On 304s we still make a request using our connection pool, yet we do not call the parent adapter, which releases the connection @@ -134,11 +133,20 @@ class TestReleaseConnection(object): resp = Mock(status=304, headers={}) - # This is how the urllib3 response is created in - # requests.adapters - response_mod = "requests.adapters.HTTPResponse.from_httplib" + # These are various ways the the urllib3 response can created + # in requests.adapters. Which one is actually used depends + # on which version if `requests` is in use, as well as perhaps + # other parameters. + response_mods = [ + "requests.adapters.HTTPResponse.from_httplib", + "urllib3.HTTPConnectionPool.urlopen", + ] + + with ExitStack() as stack: + for mod in response_mods: + with suppress(ImportError, AttributeError): + stack.enter_context(patch(mod, Mock(return_value=resp))) - with patch(response_mod, Mock(return_value=resp)): sess.get(etag_url) assert resp.read.called assert resp.release_conn.called diff --git a/tests/test_expires_heuristics.py b/tests/test_expires_heuristics.py index 475541b..2a5fd23 100644 --- a/tests/test_expires_heuristics.py +++ b/tests/test_expires_heuristics.py @@ -4,26 +4,27 @@ import calendar import time - +from datetime import datetime, timezone from email.utils import formatdate, parsedate -from datetime import datetime +from pprint import pprint +from unittest.mock import Mock -from mock import Mock from requests import Session, get from cachecontrol import CacheControl -from cachecontrol.heuristics import LastModified, ExpiresAfter, OneDayCache -from cachecontrol.heuristics import TIME_FMT -from cachecontrol.heuristics import BaseHeuristic -from .utils import DummyResponse - -from pprint import pprint +from cachecontrol.heuristics import ( + TIME_FMT, + BaseHeuristic, + ExpiresAfter, + LastModified, + OneDayCache, +) +from .utils import DummyResponse -class TestHeuristicWithoutWarning(object): - - def setup(self): +class TestHeuristicWithoutWarning: + def setup_method(self): class NoopHeuristic(BaseHeuristic): warning = Mock() @@ -35,17 +36,14 @@ class TestHeuristicWithoutWarning(object): def test_no_header_change_means_no_warning_header(self, url): the_url = url + "optional_cacheable_request" - resp = self.sess.get(the_url) + self.sess.get(the_url) assert not self.heuristic.warning.called -class TestHeuristicWith3xxResponse(object): - - def setup(self): - +class TestHeuristicWith3xxResponse: + def setup_method(self): class DummyHeuristic(BaseHeuristic): - def update_headers(self, resp): return {"x-dummy-header": "foobar"} @@ -62,17 +60,15 @@ class TestHeuristicWith3xxResponse(object): assert "x-dummy-header" in resp.headers -class TestUseExpiresHeuristic(object): - +class TestUseExpiresHeuristic: def test_expires_heuristic_arg(self): sess = Session() cached_sess = CacheControl(sess, heuristic=Mock()) assert cached_sess -class TestOneDayCache(object): - - def setup(self): +class TestOneDayCache: + def setup_method(self): self.sess = Session() self.cached_sess = CacheControl(self.sess, heuristic=OneDayCache()) @@ -90,9 +86,8 @@ class TestOneDayCache(object): assert r.from_cache -class TestExpiresAfter(object): - - def setup(self): +class TestExpiresAfter: + def setup_method(self): self.sess = Session() self.cache_sess = CacheControl(self.sess, heuristic=ExpiresAfter(days=1)) @@ -111,9 +106,8 @@ class TestExpiresAfter(object): assert r.from_cache -class TestLastModified(object): - - def setup(self): +class TestLastModified: + def setup_method(self): self.sess = Session() self.cached_sess = CacheControl(self.sess, heuristic=LastModified()) @@ -135,12 +129,11 @@ def datetime_to_header(dt): return formatdate(calendar.timegm(dt.timetuple())) -class TestModifiedUnitTests(object): - +class TestModifiedUnitTests: def last_modified(self, period): return time.strftime(TIME_FMT, time.gmtime(self.time_now - period)) - def setup(self): + def setup_method(self): self.heuristic = LastModified() self.time_now = time.time() day_in_seconds = 86400 @@ -164,7 +157,9 @@ class TestModifiedUnitTests(object): resp = DummyResponse(200, {"Date": self.now, "Last-Modified": self.week_ago}) modified = self.heuristic.update_headers(resp) assert ["expires"] == list(modified.keys()) - assert datetime(*parsedate(modified["expires"])[:6]) > datetime.now() + + expected = datetime(*parsedate(modified["expires"])[:6], tzinfo=timezone.utc) + assert expected > datetime.now(timezone.utc) def test_last_modified_is_not_used_when_cache_control_present(self): resp = DummyResponse( @@ -192,7 +187,8 @@ class TestModifiedUnitTests(object): ) modified = self.heuristic.update_headers(resp) assert ["expires"] == list(modified.keys()) - assert datetime(*parsedate(modified["expires"])[:6]) > datetime.now() + expected = datetime(*parsedate(modified["expires"])[:6], tzinfo=timezone.utc) + assert expected > datetime.now(timezone.utc) def test_warning_not_added_when_response_more_recent_than_24_hours(self): resp = DummyResponse(200, {"Date": self.now, "Last-Modified": self.week_ago}) diff --git a/tests/test_max_age.py b/tests/test_max_age.py index 4755c57..09a00ce 100644 --- a/tests/test_max_age.py +++ b/tests/test_max_age.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from __future__ import print_function import pytest from requests import Session @@ -11,7 +10,7 @@ from cachecontrol.cache import DictCache from .utils import NullSerializer -class TestMaxAge(object): +class TestMaxAge: @pytest.fixture() def sess(self, url): diff --git a/tests/test_redirects.py b/tests/test_redirects.py index 56571f6..a37bb2b 100644 --- a/tests/test_redirects.py +++ b/tests/test_redirects.py @@ -10,9 +10,8 @@ import requests from cachecontrol import CacheControl -class TestPermanentRedirects(object): - - def setup(self): +class TestPermanentRedirects: + def setup_method(self): self.sess = CacheControl(requests.Session()) def test_redirect_response_is_cached(self, url): @@ -32,9 +31,8 @@ class TestPermanentRedirects(object): assert not resp.from_cache -class TestMultipleChoicesRedirects(object): - - def setup(self): +class TestMultipleChoicesRedirects: + def setup_method(self): self.sess = CacheControl(requests.Session()) def test_multiple_choices_is_cacheable(self, url): diff --git a/tests/test_regressions.py b/tests/test_regressions.py index a072fd7..7109c2a 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -12,7 +12,7 @@ from cachecontrol.filewrapper import CallbackFileWrapper from requests import Session -class Test39(object): +class Test39: @pytest.mark.skipif( sys.version.startswith("2"), reason="Only run this for python 3.x" diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 4301be4..1ae4f00 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -2,33 +2,33 @@ # # SPDX-License-Identifier: Apache-2.0 +import pickle +from unittest.mock import Mock + import msgpack import requests -from mock import Mock - -from cachecontrol.compat import pickle from cachecontrol.serialize import Serializer -class TestSerializer(object): - def setup(self): +class TestSerializer: + def setup_method(self): self.serializer = Serializer() self.response_data = { - u"response": { + "response": { # Encode the body as bytes b/c it will eventually be # converted back into a BytesIO object. - u"body": "Hello World".encode("utf-8"), - u"headers": { - u"Content-Type": u"text/plain", - u"Expires": u"87654", - u"Cache-Control": u"public", + "body": b"Hello World", + "headers": { + "Content-Type": "text/plain", + "Expires": "87654", + "Cache-Control": "public", }, - u"status": 200, - u"version": 11, - u"reason": u"", - u"strict": True, - u"decode_content": True, + "status": 200, + "version": 11, + "reason": "", + "strict": True, + "decode_content": True, } } @@ -38,18 +38,17 @@ class TestSerializer(object): resp = self.serializer.loads(req, data) assert resp is None - def test_read_version_v1(self): + def test_load_by_version_v1(self): + data = b"cc=1,somedata" req = Mock() - resp = self.serializer._loads_v1(req, pickle.dumps(self.response_data)) - # We have to decode our urllib3 data back into a unicode string. - assert resp.data == "Hello World".encode("utf-8") + resp = self.serializer.loads(req, data) + assert resp is None - def test_read_version_v2(self): + def test_load_by_version_v2(self): + data = b"cc=2,somedata" req = Mock() - compressed_base64_json = b"x\x9c%O\xb9\n\x83@\x10\xfd\x97\xa9-\x92%E\x14R\xe4 +\x16\t\xe6\x10\xbb\xb0\xc7\xe0\x81\xb8\xb2\xbb*A\xfc\xf7\x8c\xa6|\xe7\xbc\x99\xc0\xa2\xebL\xeb\x10\xa2\t\xa4\xd1_\x88\xe0\xc93'\xf9\xbe\xc8X\xf8\x95<=@\x00\x1a\x95\xd1\xf8Q\xa6\xf5\xd8z\x88\xbc\xed1\x80\x12\x85F\xeb\x96h\xca\xc2^\xf3\xac\xd7\xe7\xed\x1b\xf3SC5\x04w\xfa\x1c\x8e\x92_;Y\x1c\x96\x9a\x94]k\xc1\xdf~u\xc7\xc9 \x8fDG\xa0\xe2\xac\x92\xbc\xa9\xc9\xf1\xc8\xcbQ\xe4I\xa3\xc6U\xb9_\x14\xbb\xbdh\xc2\x1c\xd0R\xe1LK$\xd9\x9c\x17\xbe\xa7\xc3l\xb3Y\x80\xad\x94\xff\x0b\x03\xed\xa9V\x17[2\x83\xb0\xf4\xd14\xcf?E\x03Im" - resp = self.serializer._loads_v2(req, compressed_base64_json) - # We have to decode our urllib3 data back into a unicode string. - assert resp.data == "Hello World".encode("utf-8") + resp = self.serializer.loads(req, data) + assert resp is None def test_load_by_version_v3(self): data = b"cc=3,somedata" @@ -61,32 +60,7 @@ class TestSerializer(object): req = Mock() resp = self.serializer._loads_v4(req, msgpack.dumps(self.response_data)) # We have to decode our urllib3 data back into a unicode string. - assert resp.data == "Hello World".encode("utf-8") - - def test_read_v1_serialized_with_py2_TypeError(self): - # This tests how the code handles in reading data that was pickled - # with an old version of cachecontrol running under Python 2 - req = Mock() - py2_pickled_data = b"".join( - [ - b"(dp1\nS'response'\np2\n(dp3\nS'body'\np4\nS'Hello World'\n", - b"p5\nsS'version'\np6\nS'2'\nsS'status'\np7\nI200\n", - b"sS'reason'\np8\nS''\nsS'decode_content'\np9\nI01\n", - b"sS'strict'\np10\nS''\nsS'headers'\np11\n(dp12\n", - b"S'Content-Type'\np13\nS'text/plain'\np14\n", - b"sS'Cache-Control'\np15\nS'public'\np16\n", - b"sS'Expires'\np17\nS'87654'\np18\nsss.", - ] - ) - resp = self.serializer._loads_v1(req, py2_pickled_data) - # We have to decode our urllib3 data back into a unicode - # string. - assert resp.data == "Hello World".encode("utf-8") - - def test_read_v2_corrupted_cache(self): - # This should prevent a regression of bug #134 - req = Mock() - assert self.serializer._loads_v2(req, b"") is None + assert resp.data == b"Hello World" def test_read_latest_version_streamable(self, url): original_resp = requests.get(url, stream=True) @@ -135,3 +109,9 @@ class TestSerializer(object): # handle. Reading it again proves we're resetting the internal # file handle with a buffer. assert original_resp.raw.read() + + def test_no_incomplete_read_on_dumps(self, url): + resp = requests.get(url + "fixed_length", stream=True) + self.serializer.dumps(resp.request, resp.raw) + + assert resp.content == b"0123456789" diff --git a/tests/test_storage_filecache.py b/tests/test_storage_filecache.py index 8c93284..f194deb 100644 --- a/tests/test_storage_filecache.py +++ b/tests/test_storage_filecache.py @@ -21,10 +21,10 @@ def randomdata(): """Plain random http data generator:""" key = "".join(sample(string.ascii_lowercase, randint(2, 4))) val = "".join(sample(string.ascii_lowercase + string.digits, randint(2, 10))) - return "&{}={}".format(key, val) + return f"&{key}={val}" -class FileCacheTestsMixin(object): +class FileCacheTestsMixin: FileCacheClass = None # Either FileCache or SeparateBodyFileCache diff --git a/tests/test_storage_redis.py b/tests/test_storage_redis.py index 9ada01c..5e794b6 100644 --- a/tests/test_storage_redis.py +++ b/tests/test_storage_redis.py @@ -2,15 +2,14 @@ # # SPDX-License-Identifier: Apache-2.0 -from datetime import datetime +from datetime import datetime, timezone +from unittest.mock import Mock -from mock import Mock from cachecontrol.caches import RedisCache -class TestRedisCache(object): - - def setup(self): +class TestRedisCache: + def setup_method(self): self.conn = Mock() self.cache = RedisCache(self.conn) @@ -18,6 +17,11 @@ class TestRedisCache(object): self.cache.set("foo", "bar", expires=datetime(2014, 2, 2)) assert self.conn.setex.called + def test_set_expiration_datetime_aware(self): + self.cache.set("foo", "bar", + expires=datetime(2014, 2, 2, tzinfo=timezone.utc)) + assert self.conn.setex.called + def test_set_expiration_int(self): self.cache.set("foo", "bar", expires=600) assert self.conn.setex.called diff --git a/tests/test_vary.py b/tests/test_vary.py index 543294b..c8e0cec 100644 --- a/tests/test_vary.py +++ b/tests/test_vary.py @@ -2,18 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 +from pprint import pprint +from urllib.parse import urljoin + import pytest import requests from cachecontrol import CacheControl from cachecontrol.cache import DictCache -from cachecontrol.compat import urljoin - -from pprint import pprint - -class TestVary(object): +class TestVary: @pytest.fixture() def sess(self, url): self.url = urljoin(url, "/vary_accept") @@ -33,7 +32,6 @@ class TestVary(object): cached.status == resp.raw.status, cached.version == resp.raw.version, cached.reason == resp.raw.reason, - cached.strict == resp.raw.strict, cached.decode_content == resp.raw.decode_content, ] diff --git a/tests/utils.py b/tests/utils.py index 8453006..99e67ec 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,8 +4,10 @@ Shared utility classes. from requests.structures import CaseInsensitiveDict +from cachecontrol.serialize import Serializer -class NullSerializer(object): + +class NullSerializer(Serializer): def dumps(self, request, response, body=None): return response @@ -3,20 +3,28 @@ ; SPDX-License-Identifier: Apache-2.0 [tox] -envlist = py27, py36, py37, py38, py39 +isolated_build = True +envlist = py{36,37,38,39,310,311}, mypy [gh-actions] python = - 2.7: py27 - 3.6: py36 3.7: py37 3.8: py38 3.9: py39 + 3.10: py310, mypy + 3.11: py311 [testenv] deps = pytest - mock cherrypy - redis - filelock + redis>=2.10.5 + filelock>=3.8.0 commands = py.test {posargs:tests/} + +[testenv:mypy] +deps = + {[testenv]deps} + mypy + types-redis + types-requests +commands = mypy {posargs:cachecontrol} |