diff options
| author | Stefano Rivera <stefanor@debian.org> | 2024-03-28 13:45:03 -0400 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2024-03-28 22:39:46 +0000 |
| commit | b253dacf777dbda760697b52dfaa05c161846f5b (patch) | |
| tree | cccd248b8187345e930a83cbdc4979b120c1a001 | |
| parent | 2fe1285c5d72d4070f77774cb69cd50bbf8006ae (diff) | |
0.3.5-1 (patches unapplied)import/0.3.5-1
Imported using git-ubuntu import.
Notes
Notes:
* New upstream release.
* Refresh patches.
* Drop patches, superseded upstream: no-subprocess32, python-3.11,
python-3.12.
81 files changed, 2362 insertions, 3236 deletions
diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index b7b094e..919b992 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -2,6 +2,7 @@ # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool # `{script: ...}` is shorthand for `{task: CmdLine@<mumble>, inputs: {script: ...}}`. +# The shell is bash. # https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines @@ -9,24 +10,85 @@ steps: - task: UsePythonVersion@0 displayName: Install python inputs: + githubToken: '$(GITHUB_PYVER_TOKEN)' versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') - script: | - type python - python --version - displayName: Show python version + set -o errexit + set -o nounset + set -o pipefail -- script: | sudo apt-get update sudo apt-get install -y python2-dev python3-pip virtualenv displayName: Install build deps condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) -- script: python -mpip install "tox<4.0" +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + displayName: Show python versions + +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi displayName: Install tooling -- script: python -mtox -e "$(tox.env)" +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 4513ae9..3630f83 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -21,25 +21,21 @@ jobs: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_310: - python.version: '3.10' - tox.env: py310-mode_mitogen + Mito_312: + python.version: '3.12' + tox.env: py312-mode_mitogen - # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_4: - tox.env: py27-mode_localhost-ansible4 + Loc_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6 - # NOTE: this hangs when ran in Ubuntu 18.04 Van_27_210: - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive + tox.env: py27-mode_localhost-ansible2.10-strategy_linear + Van_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6-strategy_linear - job: Linux pool: @@ -96,33 +92,33 @@ jobs: python.version: '3.6' tox.env: py36-mode_mitogen-distro_ubuntu2004 - Mito_310_centos6: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos6 - Mito_310_centos7: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos7 - Mito_310_centos8: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos8 - Mito_310_debian9: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian9 - Mito_310_debian10: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian10 - Mito_310_debian11: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian11 - Mito_310_ubuntu1604: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1604 - Mito_310_ubuntu1804: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1804 - Mito_310_ubuntu2004: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu2004 + Mito_312_centos6: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos6 + Mito_312_centos7: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos7 + Mito_312_centos8: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos8 + Mito_312_debian9: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian9 + Mito_312_debian10: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian10 + Mito_312_debian11: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian11 + Mito_312_ubuntu1604: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1604 + Mito_312_ubuntu1804: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1804 + Mito_312_ubuntu2004: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu2004 Ans_27_210: tox.env: py27-mode_ansible-ansible2.10 @@ -136,18 +132,18 @@ jobs: python.version: '3.6' tox.env: py36-mode_ansible-ansible4 - Ans_310_210: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible2.10 - Ans_310_3: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible3 - Ans_310_4: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible4 - Ans_310_5: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible5 - Ans_310_6: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible6 + Ans_311_210: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible2.10 + Ans_311_3: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible3 + Ans_311_4: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible4 + Ans_311_5: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible5 + Ans_312_6: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible6 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 47a8ddf..3e71638 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -10,7 +10,10 @@ import shutil import sys import tempfile -import subprocess32 as subprocess +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess try: import urlparse diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 69d67cd..c50ef22 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +from __future__ import print_function + +import getpass +import io import os import subprocess import sys @@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") + # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 + # and os.environ['USER'] is not populated on Azure macOS runners. + os.chdir(HOSTS_DIR) + with io.open('default.hosts', 'r+', encoding='utf-8') as f: + user = getpass.getuser() + content = f.read() + content = content.replace("{{ lookup('pipe', 'whoami') }}", user) + f.seek(0) + f.write(content) + f.truncate() + ci_lib.dump_file('default.hosts') + + cmd = ';'.join([ + 'from __future__ import print_function', + 'import os, sys', + 'print(sys.executable, os.path.realpath(sys.executable))', + ]) + for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: + print(interpreter) + try: + subprocess.call([interpreter, '-c', cmd]) + except OSError as exc: + print(exc) + + print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') + environ = os.environ.copy() + environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' + try: + subprocess.call([interpreter, '-c', cmd], env=environ) + except OSError as exc: + print(exc) + with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) @@ -6,6 +6,7 @@ venvs/** *.pyc *.pyd *.pyo +*.retry MANIFEST build/ dist/ diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index cec465c..a187083 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -31,15 +31,31 @@ from __future__ import unicode_literals __metaclass__ = type import collections -import imp +import logging import os +import re +import sys + +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp import mitogen.master +LOG = logging.getLogger(__name__) PREFIX = 'ansible.module_utils.' +# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`. +# name Unqualified name of the module. +# path Filesystem path of the module. +# kind One of the constants in `imp`, as returned in `imp.find_module()` +# parent `ansible_mitogen.module_finder.Module` of parent package (if any). Module = collections.namedtuple('Module', 'name path kind parent') @@ -119,14 +135,121 @@ def find_relative(parent, name, path=()): def scan_fromlist(code): + """Return an iterator of (level, name) for explicit imports in a code + object. + + Not all names identify a module. `from os import name, path` generates + `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. + + >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' + >>> code = compile(src, '<str>', 'exec') + >>> list(scan_fromlist(code)) + [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] + """ for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): for name in fromlist: - yield level, '%s.%s' % (modname_s, name) + yield level, str('%s.%s' % (modname_s, name)) if not fromlist: yield level, modname_s +def walk_imports(code, prefix=None): + """Return an iterator of names for implicit parent imports & explicit + imports in a code object. + + If a prefix is provided, then only children of that prefix are included. + Not all names identify a module. `from os import name, path` generates + `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. + + >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' + >>> code = compile(source, '<str>', 'exec') + >>> list(walk_imports(code)) + ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] + >>> list(walk_imports(code, prefix='b')) + ['b.c', 'b.d', 'b.d.e', 'b.d.f'] + """ + if prefix is None: + prefix = '' + pattern = re.compile(r'(^|\.)(\w+)') + start = len(prefix) + for _, name, fromlist in mitogen.master.scan_code_imports(code): + if not name.startswith(prefix): + continue + for match in pattern.finditer(name, start): + yield name[:match.end()] + for leaf in fromlist: + yield str('%s.%s' % (name, leaf)) + + def scan(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + """Return a list of (name, path, is_package) for ansible.module_utils + imports used by an Ansible module. + """ + log = LOG.getChild('scan') + log.debug('%r, %r, %r', module_name, module_path, search_path) + + if sys.version_info >= (3, 4): + result = _scan_importlib_find_spec( + module_name, module_path, search_path, + ) + log.debug('_scan_importlib_find_spec %r', result) + else: + result = _scan_imp_find_module(module_name, module_path, search_path) + log.debug('_scan_imp_find_module %r', result) + return result + + +def _scan_importlib_find_spec(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + module = importlib.machinery.ModuleSpec( + module_name, loader=None, origin=module_path, + ) + prefix = importlib.machinery.ModuleSpec( + PREFIX.rstrip('.'), loader=None, + ) + prefix.submodule_search_locations = search_path + queue = collections.deque([module]) + specs = {prefix.name: prefix} + while queue: + spec = queue.popleft() + if spec.origin is None: + continue + try: + with open(spec.origin, 'rb') as f: + code = compile(f.read(), spec.name, 'exec') + except Exception as exc: + raise ValueError((exc, module, spec, specs)) + + for name in walk_imports(code, prefix.name): + if name in specs: + continue + + parent_name = name.rpartition('.')[0] + parent = specs[parent_name] + if parent is None or not parent.submodule_search_locations: + specs[name] = None + continue + + child = importlib.util._find_spec( + name, parent.submodule_search_locations, + ) + if child is None or child.origin is None: + specs[name] = None + continue + + specs[name] = child + queue.append(child) + + del specs[prefix.name] + return sorted( + (spec.name, spec.origin, spec.submodule_search_locations is not None) + for spec in specs.values() if spec is not None + ) + + +def _scan_imp_find_module(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] module = Module(module_name, module_path, imp.PY_SOURCE, None) stack = [module] seen = set() diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index c4cb71f..8da1b67 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit -import imp import json import os import re @@ -65,6 +64,14 @@ except ImportError: ctypes = None try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + +try: # Cannot use cStringIO as it does not support Unicode. from StringIO import StringIO except ImportError: @@ -514,10 +521,71 @@ class ModuleUtilsImporter(object): sys.modules.pop(fullname, None) def find_module(self, fullname, path=None): + """ + Return a loader for the module with fullname, if we will load it. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. Removed in Python 3.12. + """ if fullname in self._by_fullname: return self + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`. + + Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. + """ + if fullname.endswith('.'): + return None + + try: + module_path, is_package = self._by_fullname[fullname] + except KeyError: + LOG.debug('Skipping %s: not present', fullname) + return None + + LOG.debug('Handling %s', fullname) + origin = 'master:%s' % (module_path,) + return importlib.machinery.ModuleSpec( + fullname, loader=self, origin=origin, is_package=is_package, + ) + + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + spec = module.__spec__ + path, _ = self._by_fullname[spec.name] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + exec(code, module.__dict__) + self._loaded.add(spec.name) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements PEP 302 importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) @@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner): synchronization mechanism by importing everything the module will need prior to detaching. """ + # I think "custom" means "found in custom module_utils search path", + # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) + + # I think "builtin" means "part of ansible/ansible-base/ansible-core", + # as opposed to Python builtin modules such as sys. for fullname in self.module_map['builtin']: try: mitogen.core.import_module(fullname) - except ImportError: + except ImportError as exc: # #590: Ansible 2.8 module_utils.distro is a package that # replaces itself in sys.modules with a non-package during # import. Prior to replacement, it is a real package containing @@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner): # loop progresses to the next entry and attempts to preload # 'distro._distro', the import mechanism will fail. So here we # silently ignore any failure for it. - if fullname != 'ansible.module_utils.distro._distro': - raise + if fullname == 'ansible.module_utils.distro._distro': + continue + + # ansible.module_utils.compat.selinux raises ImportError if it + # can't load libselinux.so. The importer would usually catch + # this & skip selinux operations. We don't care about selinux, + # we're using import to get a copy of the module. + if (fullname == 'ansible.module_utils.compat.selinux' + and exc.msg == 'unable to load libselinux.so'): + continue + + raise def _setup_excepthook(self): """ diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py index 25c5a69..a01b261 100644 --- a/ansible_mitogen/utils.py +++ b/ansible_mitogen/utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -import distutils.version +import re import ansible @@ -9,6 +9,21 @@ __all__ = [ 'ansible_version', ] -ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version) -del distutils + +def _parse(v_string): + # Adapted from distutils.version.LooseVersion.parse() + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + for component in component_re.split(v_string): + if not component or component == '.': + continue + try: + yield int(component) + except ValueError: + yield component + + +ansible_version = tuple(_parse(ansible.__version__)) + +del _parse +del re del ansible diff --git a/debian/changelog b/debian/changelog index 850f3ef..5d239da 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +python-mitogen (0.3.5-1) unstable; urgency=medium + + * New upstream release. + * Refresh patches. + * Drop patches, superseded upstream: no-subprocess32, python-3.11, + python-3.12. + + -- Stefano Rivera <stefanor@debian.org> Thu, 28 Mar 2024 13:45:03 -0400 + python-mitogen (0.3.4-3) unstable; urgency=medium * Patch: Support setting interpreter_python in config, again. diff --git a/debian/patches/ansible-2.16 b/debian/patches/ansible-2.16 index 3b94c37..d752943 100644 --- a/debian/patches/ansible-2.16 +++ b/debian/patches/ansible-2.16 @@ -24,10 +24,10 @@ index 754ec6c..9729b8a 100644 NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/mitogen/core.py b/mitogen/core.py -index bee722e..865f826 100644 +index cd02012..d92c47f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py -@@ -842,6 +842,9 @@ class Message(object): +@@ -874,6 +874,9 @@ class Message(object): s, n = LATIN1_CODEC.encode(s) return s @@ -37,7 +37,7 @@ index bee722e..865f826 100644 def _find_global(self, module, func): """ Return the class implementing `module_name.class_name` or raise -@@ -860,6 +863,8 @@ class Message(object): +@@ -892,6 +895,8 @@ class Message(object): return Secret elif func == 'Kwargs': return Kwargs diff --git a/debian/patches/ask-become-pass b/debian/patches/ask-become-pass index 2130532..eec9f43 100644 --- a/debian/patches/ask-become-pass +++ b/debian/patches/ask-become-pass @@ -12,13 +12,12 @@ Bug-Debian: https://bugs.debian.org/1042556 .ci/ansible_tests.py | 11 ++++++++++- ansible_mitogen/transport_config.py | 16 ++++++++++++---- tests/ansible/hosts/default.hosts | 2 ++ - tests/ansible/hosts/transport_config.hosts | 2 +- .../integration/transport_config/become_pass.yml | 5 +++-- tests/ansible/regression/all.yml | 1 + tests/ansible/regression/become_test.yml | 9 +++++++++ .../regression/issue_952__ask_become_pass.yml | 21 +++++++++++++++++++++ tests/requirements.txt | 1 + - 9 files changed, 60 insertions(+), 8 deletions(-) + 8 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 tests/ansible/regression/become_test.yml create mode 100644 tests/ansible/regression/issue_952__ask_become_pass.yml @@ -86,28 +85,15 @@ index 5fc7818..1fc1e80 100644 def password(self): diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts -index d40c3dd..7ef231b 100644 +index 1bec001..4f5ea4c 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts -@@ -7,3 +7,5 @@ target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" +@@ -10,3 +10,5 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target + +[linux_containers] -diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts -index 7d7b526..d6a15c9 100644 ---- a/tests/ansible/hosts/transport_config.hosts -+++ b/tests/ansible/hosts/transport_config.hosts -@@ -21,7 +21,7 @@ tc_transport - # That causes approx 10 seconds timeout per task - there's no host to connect to. - # This optimisation should not be relied in any test. - # Note: tc-python-path-* are intentionally not included. --ansible_python_interpreter = python3000 # Not expected to exist -+ansible_python_interpreter = python3000 - - [tc_transport] - tc-transport-unset diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml index 317e052..670faf5 100644 --- a/tests/ansible/integration/transport_config/become_pass.yml @@ -176,14 +162,14 @@ index 0000000..da269e2 + changed_when: false + check_mode: false diff --git a/tests/requirements.txt b/tests/requirements.txt -index 7301bee..eb84b9b 100644 +index 6d87d17..ae3abb8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt -@@ -3,6 +3,7 @@ coverage==5.5; python_version < '3.7' - coverage==6.4.4; python_version >= '3.7' - Django==1.6.11 # Last version supporting 2.6. - mock==2.0.0 +@@ -12,6 +12,7 @@ Django==3.2.20; python_version >= '3.6' + mock==3.0.5; python_version == '2.7' + mock==5.1.0; python_version >= '3.6' + +pexpect==4.8 - pytz==2018.5 - cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect - pycparser==2.18 # Last version supporting 2.6. + psutil==5.9.8 + + pytest==4.6.11; python_version == '2.7' diff --git a/debian/patches/no-subprocess32 b/debian/patches/no-subprocess32 deleted file mode 100644 index e347210..0000000 --- a/debian/patches/no-subprocess32 +++ /dev/null @@ -1,26 +0,0 @@ -From: Stefano Rivera <stefanor@debian.org> -Date: Tue, 18 Jul 2023 18:39:01 +0200 -Subject: Use subprocess instead of subprocess32 - -This is a compatibility module for Python 2.7, which isn't a use-case we -are about in the tests in Debian. subprocess32 isn't available in -Debian. - -Forwarded: not-needed ---- - tests/testlib.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/tests/testlib.py b/tests/testlib.py -index 83629c3..5e9d0a7 100644 ---- a/tests/testlib.py -+++ b/tests/testlib.py -@@ -18,7 +18,7 @@ except ImportError: - import ConfigParser as configparser - - import psutil --import subprocess32 as subprocess -+import subprocess - - import mitogen.core - import mitogen.fork diff --git a/debian/patches/python-3.11 b/debian/patches/python-3.11 deleted file mode 100644 index 34893dc..0000000 --- a/debian/patches/python-3.11 +++ /dev/null @@ -1,246 +0,0 @@ -From: =?utf-8?q?Nerijus_Bali=C5=ABnas?= <nerijus.baliunas@adv.bet> -Date: Tue, 21 Feb 2023 13:04:28 +0200 -Subject: Add Python 3.11 support - -Co-authored-by: Alex Willmer <alex@moreati.org.uk> -Origin: upstream, https://github.com/mitogen-hq/mitogen/pull/1019 ---- - .ci/azure-pipelines.yml | 90 ++++++++++++++++---------------- - docs/ansible_detailed.rst | 4 +- - docs/changelog.rst | 2 + - setup.py | 1 + - tests/ansible/lib/action/assert_equal.py | 7 ++- - tests/requirements.txt | 5 +- - tests/testlib.py | 2 +- - tox.ini | 9 ++-- - 8 files changed, 64 insertions(+), 56 deletions(-) - -diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml -index 4513ae9..f672240 100644 ---- a/.ci/azure-pipelines.yml -+++ b/.ci/azure-pipelines.yml -@@ -21,9 +21,9 @@ jobs: - matrix: - Mito_27: - tox.env: py27-mode_mitogen -- Mito_310: -- python.version: '3.10' -- tox.env: py310-mode_mitogen -+ Mito_311: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen - - # TODO: test python3, python3 tests are broken - Loc_27_210: -@@ -96,33 +96,33 @@ jobs: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu2004 - -- Mito_310_centos6: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_centos6 -- Mito_310_centos7: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_centos7 -- Mito_310_centos8: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_centos8 -- Mito_310_debian9: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_debian9 -- Mito_310_debian10: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_debian10 -- Mito_310_debian11: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_debian11 -- Mito_310_ubuntu1604: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_ubuntu1604 -- Mito_310_ubuntu1804: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_ubuntu1804 -- Mito_310_ubuntu2004: -- python.version: '3.10' -- tox.env: py310-mode_mitogen-distro_ubuntu2004 -+ Mito_311_centos6: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_centos6 -+ Mito_311_centos7: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_centos7 -+ Mito_311_centos8: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_centos8 -+ Mito_311_debian9: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_debian9 -+ Mito_311_debian10: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_debian10 -+ Mito_311_debian11: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_debian11 -+ Mito_311_ubuntu1604: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_ubuntu1604 -+ Mito_311_ubuntu1804: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_ubuntu1804 -+ Mito_311_ubuntu2004: -+ python.version: '3.11' -+ tox.env: py311-mode_mitogen-distro_ubuntu2004 - - Ans_27_210: - tox.env: py27-mode_ansible-ansible2.10 -@@ -136,18 +136,18 @@ jobs: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible4 - -- Ans_310_210: -- python.version: '3.10' -- tox.env: py310-mode_ansible-ansible2.10 -- Ans_310_3: -- python.version: '3.10' -- tox.env: py310-mode_ansible-ansible3 -- Ans_310_4: -- python.version: '3.10' -- tox.env: py310-mode_ansible-ansible4 -- Ans_310_5: -- python.version: '3.10' -- tox.env: py310-mode_ansible-ansible5 -- Ans_310_6: -- python.version: '3.10' -- tox.env: py310-mode_ansible-ansible6 -+ Ans_311_210: -+ python.version: '3.11' -+ tox.env: py311-mode_ansible-ansible2.10 -+ Ans_311_3: -+ python.version: '3.11' -+ tox.env: py311-mode_ansible-ansible3 -+ Ans_311_4: -+ python.version: '3.11' -+ tox.env: py311-mode_ansible-ansible4 -+ Ans_311_5: -+ python.version: '3.11' -+ tox.env: py311-mode_ansible-ansible5 -+ Ans_311_6: -+ python.version: '3.11' -+ tox.env: py311-mode_ansible-ansible6 -diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst -index 449771b..64fcd63 100644 ---- a/docs/ansible_detailed.rst -+++ b/docs/ansible_detailed.rst -@@ -147,8 +147,8 @@ Noteworthy Differences - - * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. - Mitogen 0.3.1+ supports -- - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10 -- - Ansible 5 and 6; with Python 3.8-3.10 -+ - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 -+ - Ansible 5 and 6; with Python 3.8-3.11 - Verify your installation is running one of these versions by checking - ``ansible --version`` output. - -diff --git a/docs/changelog.rst b/docs/changelog.rst -index d9a54bc..63c9ff2 100644 ---- a/docs/changelog.rst -+++ b/docs/changelog.rst -@@ -4,6 +4,8 @@ - Release Notes - ============= - -+* :gh:issue:`987` Support Python 3.11 -+ - - .. raw:: html - -diff --git a/setup.py b/setup.py -index 9d2ff36..4d7fadf 100644 ---- a/setup.py -+++ b/setup.py -@@ -77,6 +77,7 @@ setup( - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', -+ 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: System :: Distributed Computing', - 'Topic :: System :: Systems Administration', -diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py -index 5923f4b..72264cf 100644 ---- a/tests/ansible/lib/action/assert_equal.py -+++ b/tests/ansible/lib/action/assert_equal.py -@@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase - - TEMPLATE_KWARGS = {} - --_argspec = inspect.getargspec(ansible.template.Templar.template) -+try: -+ # inspect.getfullargspec() Added: 3.0 -+ _argspec = inspect.getfullargspec(ansible.template.Templar.template) -+except AttributeError: -+ # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11 -+ _argspec = inspect.getargspec(ansible.template.Templar.template) - if 'bare_deprecated' in _argspec.args: - TEMPLATE_KWARGS['bare_deprecated'] = False - -diff --git a/tests/requirements.txt b/tests/requirements.txt -index eb84b9b..9bbfe46 100644 ---- a/tests/requirements.txt -+++ b/tests/requirements.txt -@@ -1,12 +1,11 @@ --psutil==5.4.8 -+cffi==1.15.1 - coverage==5.5; python_version < '3.7' - coverage==6.4.4; python_version >= '3.7' - Django==1.6.11 # Last version supporting 2.6. - mock==2.0.0 - pexpect==4.8 - pytz==2018.5 --cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect --pycparser==2.18 # Last version supporting 2.6. -+psutil==5.9.5 - pytest-catchlog==1.2.2 - pytest==3.1.2 - subprocess32==3.5.4 -diff --git a/tests/testlib.py b/tests/testlib.py -index 5e9d0a7..da4d46f 100644 ---- a/tests/testlib.py -+++ b/tests/testlib.py -@@ -87,7 +87,7 @@ def base_executable(executable=None): - base_executable = None - - if base_executable and base_executable != executable: -- return 'be', base_executable -+ return base_executable - - # Python 2.x only has sys.base_prefix if running outside a virtualenv. - try: -diff --git a/tox.ini b/tox.ini -index da71093..3b3cb9f 100644 ---- a/tox.ini -+++ b/tox.ini -@@ -37,10 +37,10 @@ - envlist = - init, - py{27,36}-mode_ansible-ansible{2.10,3,4}, -- py{310}-mode_ansible-ansible{2.10,3,4,5,6}, -- py{27,36,310}-mode_mitogen-distro_centos{6,7,8}, -- py{27,36,310}-mode_mitogen-distro_debian{9,10,11}, -- py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004}, -+ py{311}-mode_ansible-ansible{2.10,3,4,5,6}, -+ py{27,36,311}-mode_mitogen-distro_centos{6,7,8}, -+ py{27,36,311}-mode_mitogen-distro_debian{9,10,11}, -+ py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004}, - report, - - [testenv] -@@ -53,6 +53,7 @@ basepython = - py38: python3.8 - py39: python3.9 - py310: python3.10 -+ py311: python3.11 - deps = - -r{toxinidir}/tests/requirements.txt - mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt diff --git a/debian/patches/python-3.12 b/debian/patches/python-3.12 deleted file mode 100644 index 91dc81e..0000000 --- a/debian/patches/python-3.12 +++ /dev/null @@ -1,2358 +0,0 @@ -From: Alex Willmer <alex@moreati.org.uk> -Date: Fri, 8 Mar 2024 17:39:26 -0400 -Subject: Python 3.12 support - -Origin: https://github.com/mitogen-hq/mitogen/pull/1032 -Bug-Debian: https://bugs.debian.org/1061755 -Bug-Debian: https://bugs.debian.org/1058132 -Bug-upstream: https://github.com/mitogen-hq/mitogen/issues/1033 ---- - .gitignore | 1 + - ansible_mitogen/module_finder.py | 127 +++++++++++++- - ansible_mitogen/runner.py | 91 +++++++++- - ansible_mitogen/utils.py | 21 ++- - docs/ansible_detailed.rst | 4 +- - docs/conf.py | 8 +- - mitogen/core.py | 192 +++++++++++++++++++-- - mitogen/master.py | 122 ++++++++++++- - mitogen/parent.py | 27 +-- - setup.py | 1 + - tests/ansible/files/cwd_show | 22 +++ - tests/ansible/hosts/become_same_user.hosts | 2 + - tests/ansible/hosts/connection_delegation.hosts | 1 + - tests/ansible/hosts/default.hosts | 5 +- - tests/ansible/hosts/k3.hosts | 1 + - tests/ansible/hosts/localhost.hosts | 1 + - tests/ansible/hosts/transport_config.hosts | 8 +- - tests/ansible/integration/become/su_password.yml | 41 ++++- - .../connection_delegation/delegate_to_template.yml | 4 +- - .../interpreter_discovery/ansible_2_8_tests.yml | 30 ++-- - .../interpreter_discovery/complex_args.yml | 23 ++- - ...custom_python_new_style_missing_interpreter.yml | 5 + - .../runner/custom_python_new_style_module.yml | 3 +- - .../runner/custom_python_prehistoric_module.yml | 6 + - .../modules/custom_python_detect_environment.py | 7 +- - .../lib/modules/custom_python_external_module.py | 2 +- - .../lib/modules/custom_python_external_pkg.py | 2 +- - .../lib/modules/custom_python_json_args_module.py | 2 +- - .../lib/modules/custom_python_leaky_class_vars.py | 2 +- - .../lib/modules/custom_python_modify_environ.py | 2 +- - .../custom_python_new_style_missing_interpreter.py | 49 ++++++ - .../lib/modules/custom_python_new_style_module.py | 51 +++++- - .../ansible/lib/modules/custom_python_os_getcwd.py | 2 +- - .../modules/custom_python_prehistoric_module.py | 7 +- - .../lib/modules/custom_python_run_script.py | 2 +- - .../lib/modules/custom_python_uses_distro.py | 2 +- - .../lib/modules/custom_python_want_json_module.py | 2 +- - .../lib/modules/mitogen_test_gethostbyname.py | 2 +- - tests/ansible/lib/modules/module_finder_test.py | 12 ++ - tests/ansible/lib/modules/test_echo_module.py | 53 ++++-- - .../issue_152__virtualenv_python_fails.yml | 11 +- - tests/ansible/setup/all.yml | 3 +- - tests/ansible/setup/report.yml | 20 --- - tests/ansible/setup/report_controller.yml | 17 ++ - tests/ansible/setup/report_targets.yml | 15 ++ - tests/ansible/tests/module_finder_test.py | 80 +++++++++ - tests/image_prep/_user_accounts.yml | 26 +-- - tests/importer_test.py | 126 +++++++++++++- - tests/module_finder_test.py | 14 +- - tests/requirements-tox.txt | 4 + - tests/requirements.txt | 5 +- - tests/testlib.py | 9 +- - tox.ini | 24 ++- - 53 files changed, 1151 insertions(+), 148 deletions(-) - create mode 100755 tests/ansible/files/cwd_show - create mode 100644 tests/ansible/lib/modules/module_finder_test.py - delete mode 100644 tests/ansible/setup/report.yml - create mode 100644 tests/ansible/setup/report_controller.yml - create mode 100644 tests/ansible/setup/report_targets.yml - create mode 100644 tests/ansible/tests/module_finder_test.py - create mode 100644 tests/requirements-tox.txt - -diff --git a/.gitignore b/.gitignore -index aa75f69..7297d72 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -6,6 +6,7 @@ venvs/** - *.pyc - *.pyd - *.pyo -+*.retry - MANIFEST - build/ - dist/ -diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py -index cec465c..a187083 100644 ---- a/ansible_mitogen/module_finder.py -+++ b/ansible_mitogen/module_finder.py -@@ -31,15 +31,31 @@ from __future__ import unicode_literals - __metaclass__ = type - - import collections --import imp -+import logging - import os -+import re -+import sys -+ -+try: -+ # Python >= 3.4, PEP 451 ModuleSpec API -+ import importlib.machinery -+ import importlib.util -+except ImportError: -+ # Python < 3.4, PEP 302 Import Hooks -+ import imp - - import mitogen.master - - -+LOG = logging.getLogger(__name__) - PREFIX = 'ansible.module_utils.' - - -+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`. -+# name Unqualified name of the module. -+# path Filesystem path of the module. -+# kind One of the constants in `imp`, as returned in `imp.find_module()` -+# parent `ansible_mitogen.module_finder.Module` of parent package (if any). - Module = collections.namedtuple('Module', 'name path kind parent') - - -@@ -119,14 +135,121 @@ def find_relative(parent, name, path=()): - - - def scan_fromlist(code): -+ """Return an iterator of (level, name) for explicit imports in a code -+ object. -+ -+ Not all names identify a module. `from os import name, path` generates -+ `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. -+ -+ >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' -+ >>> code = compile(src, '<str>', 'exec') -+ >>> list(scan_fromlist(code)) -+ [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] -+ """ - for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): - for name in fromlist: -- yield level, '%s.%s' % (modname_s, name) -+ yield level, str('%s.%s' % (modname_s, name)) - if not fromlist: - yield level, modname_s - - -+def walk_imports(code, prefix=None): -+ """Return an iterator of names for implicit parent imports & explicit -+ imports in a code object. -+ -+ If a prefix is provided, then only children of that prefix are included. -+ Not all names identify a module. `from os import name, path` generates -+ `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. -+ -+ >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' -+ >>> code = compile(source, '<str>', 'exec') -+ >>> list(walk_imports(code)) -+ ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] -+ >>> list(walk_imports(code, prefix='b')) -+ ['b.c', 'b.d', 'b.d.e', 'b.d.f'] -+ """ -+ if prefix is None: -+ prefix = '' -+ pattern = re.compile(r'(^|\.)(\w+)') -+ start = len(prefix) -+ for _, name, fromlist in mitogen.master.scan_code_imports(code): -+ if not name.startswith(prefix): -+ continue -+ for match in pattern.finditer(name, start): -+ yield name[:match.end()] -+ for leaf in fromlist: -+ yield str('%s.%s' % (name, leaf)) -+ -+ - def scan(module_name, module_path, search_path): -+ # type: (str, str, list[str]) -> list[(str, str, bool)] -+ """Return a list of (name, path, is_package) for ansible.module_utils -+ imports used by an Ansible module. -+ """ -+ log = LOG.getChild('scan') -+ log.debug('%r, %r, %r', module_name, module_path, search_path) -+ -+ if sys.version_info >= (3, 4): -+ result = _scan_importlib_find_spec( -+ module_name, module_path, search_path, -+ ) -+ log.debug('_scan_importlib_find_spec %r', result) -+ else: -+ result = _scan_imp_find_module(module_name, module_path, search_path) -+ log.debug('_scan_imp_find_module %r', result) -+ return result -+ -+ -+def _scan_importlib_find_spec(module_name, module_path, search_path): -+ # type: (str, str, list[str]) -> list[(str, str, bool)] -+ module = importlib.machinery.ModuleSpec( -+ module_name, loader=None, origin=module_path, -+ ) -+ prefix = importlib.machinery.ModuleSpec( -+ PREFIX.rstrip('.'), loader=None, -+ ) -+ prefix.submodule_search_locations = search_path -+ queue = collections.deque([module]) -+ specs = {prefix.name: prefix} -+ while queue: -+ spec = queue.popleft() -+ if spec.origin is None: -+ continue -+ try: -+ with open(spec.origin, 'rb') as f: -+ code = compile(f.read(), spec.name, 'exec') -+ except Exception as exc: -+ raise ValueError((exc, module, spec, specs)) -+ -+ for name in walk_imports(code, prefix.name): -+ if name in specs: -+ continue -+ -+ parent_name = name.rpartition('.')[0] -+ parent = specs[parent_name] -+ if parent is None or not parent.submodule_search_locations: -+ specs[name] = None -+ continue -+ -+ child = importlib.util._find_spec( -+ name, parent.submodule_search_locations, -+ ) -+ if child is None or child.origin is None: -+ specs[name] = None -+ continue -+ -+ specs[name] = child -+ queue.append(child) -+ -+ del specs[prefix.name] -+ return sorted( -+ (spec.name, spec.origin, spec.submodule_search_locations is not None) -+ for spec in specs.values() if spec is not None -+ ) -+ -+ -+def _scan_imp_find_module(module_name, module_path, search_path): -+ # type: (str, str, list[str]) -> list[(str, str, bool)] - module = Module(module_name, module_path, imp.PY_SOURCE, None) - stack = [module] - seen = set() -diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py -index c4cb71f..8da1b67 100644 ---- a/ansible_mitogen/runner.py -+++ b/ansible_mitogen/runner.py -@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function - __metaclass__ = type - - import atexit --import imp - import json - import os - import re -@@ -64,6 +63,14 @@ except ImportError: - # Python 2.4 - ctypes = None - -+try: -+ # Python >= 3.4, PEP 451 ModuleSpec API -+ import importlib.machinery -+ import importlib.util -+except ImportError: -+ # Python < 3.4, PEP 302 Import Hooks -+ import imp -+ - try: - # Cannot use cStringIO as it does not support Unicode. - from StringIO import StringIO -@@ -514,10 +521,71 @@ class ModuleUtilsImporter(object): - sys.modules.pop(fullname, None) - - def find_module(self, fullname, path=None): -+ """ -+ Return a loader for the module with fullname, if we will load it. -+ -+ Implements importlib.abc.MetaPathFinder.find_module(). -+ Deprecrated in Python 3.4+, replaced by find_spec(). -+ Raises ImportWarning in Python 3.10+. Removed in Python 3.12. -+ """ - if fullname in self._by_fullname: - return self - -+ def find_spec(self, fullname, path, target=None): -+ """ -+ Return a `ModuleSpec` for module with `fullname` if we will load it. -+ Otherwise return `None`. -+ -+ Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. -+ """ -+ if fullname.endswith('.'): -+ return None -+ -+ try: -+ module_path, is_package = self._by_fullname[fullname] -+ except KeyError: -+ LOG.debug('Skipping %s: not present', fullname) -+ return None -+ -+ LOG.debug('Handling %s', fullname) -+ origin = 'master:%s' % (module_path,) -+ return importlib.machinery.ModuleSpec( -+ fullname, loader=self, origin=origin, is_package=is_package, -+ ) -+ -+ def create_module(self, spec): -+ """ -+ Return a module object for the given ModuleSpec. -+ -+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. -+ Unlike Loader.load_module() this shouldn't populate sys.modules or -+ set module attributes. Both are done by Python. -+ """ -+ module = types.ModuleType(spec.name) -+ # FIXME create_module() shouldn't initialise module attributes -+ module.__file__ = spec.origin -+ return module -+ -+ def exec_module(self, module): -+ """ -+ Execute the module to initialise it. Don't return anything. -+ -+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. -+ """ -+ spec = module.__spec__ -+ path, _ = self._by_fullname[spec.name] -+ source = ansible_mitogen.target.get_small_file(self._context, path) -+ code = compile(source, path, 'exec', 0, 1) -+ exec(code, module.__dict__) -+ self._loaded.add(spec.name) -+ - def load_module(self, fullname): -+ """ -+ Return the loaded module specified by fullname. -+ -+ Implements PEP 302 importlib.abc.Loader.load_module(). -+ Deprecated in Python 3.4+, replaced by create_module() & exec_module(). -+ """ - path, is_pkg = self._by_fullname[fullname] - source = ansible_mitogen.target.get_small_file(self._context, path) - code = compile(source, path, 'exec', 0, 1) -@@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner): - synchronization mechanism by importing everything the module will need - prior to detaching. - """ -+ # I think "custom" means "found in custom module_utils search path", -+ # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. - for fullname, _, _ in self.module_map['custom']: - mitogen.core.import_module(fullname) -+ -+ # I think "builtin" means "part of ansible/ansible-base/ansible-core", -+ # as opposed to Python builtin modules such as sys. - for fullname in self.module_map['builtin']: - try: - mitogen.core.import_module(fullname) -- except ImportError: -+ except ImportError as exc: - # #590: Ansible 2.8 module_utils.distro is a package that - # replaces itself in sys.modules with a non-package during - # import. Prior to replacement, it is a real package containing -@@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner): - # loop progresses to the next entry and attempts to preload - # 'distro._distro', the import mechanism will fail. So here we - # silently ignore any failure for it. -- if fullname != 'ansible.module_utils.distro._distro': -- raise -+ if fullname == 'ansible.module_utils.distro._distro': -+ continue -+ -+ # ansible.module_utils.compat.selinux raises ImportError if it -+ # can't load libselinux.so. The importer would usually catch -+ # this & skip selinux operations. We don't care about selinux, -+ # we're using import to get a copy of the module. -+ if (fullname == 'ansible.module_utils.compat.selinux' -+ and exc.msg == 'unable to load libselinux.so'): -+ continue -+ -+ raise - - def _setup_excepthook(self): - """ -diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py -index 25c5a69..a01b261 100644 ---- a/ansible_mitogen/utils.py -+++ b/ansible_mitogen/utils.py -@@ -1,7 +1,7 @@ - from __future__ import absolute_import, division, print_function - __metaclass__ = type - --import distutils.version -+import re - - import ansible - -@@ -9,6 +9,21 @@ __all__ = [ - 'ansible_version', - ] - --ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version) --del distutils -+ -+def _parse(v_string): -+ # Adapted from distutils.version.LooseVersion.parse() -+ component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) -+ for component in component_re.split(v_string): -+ if not component or component == '.': -+ continue -+ try: -+ yield int(component) -+ except ValueError: -+ yield component -+ -+ -+ansible_version = tuple(_parse(ansible.__version__)) -+ -+del _parse -+del re - del ansible -diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst -index 64fcd63..5c930b2 100644 ---- a/docs/ansible_detailed.rst -+++ b/docs/ansible_detailed.rst -@@ -147,8 +147,8 @@ Noteworthy Differences - - * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. - Mitogen 0.3.1+ supports -- - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 -- - Ansible 5 and 6; with Python 3.8-3.11 -+ - Ansible 5; with Python 3.8-3.11 -+ - Ansible 6; with Python 3.8-3.12 - Verify your installation is running one of these versions by checking - ``ansible --version`` output. - -diff --git a/docs/conf.py b/docs/conf.py -index 54e3a5c..74e8454 100644 ---- a/docs/conf.py -+++ b/docs/conf.py -@@ -84,10 +84,16 @@ domainrefs = { - }, - } - -+# > ## Official guidance -+# > Query PyPI’s JSON API to determine where to download files from. -+# > ## Predictable URLs -+# > You can use our conveyor service to fetch this file, which exists for -+# > cases where using the API is impractical or impossible. -+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls - rst_epilog = """ - - .. |mitogen_version| replace:: %(VERSION)s - --.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__ -+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__ - - """ % locals() -diff --git a/mitogen/core.py b/mitogen/core.py -index 865f826..d92c47f 100644 ---- a/mitogen/core.py -+++ b/mitogen/core.py -@@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the - bootstrap implementation sent to every new slave context. - """ - -+import sys -+try: -+ import _frozen_importlib_external -+except ImportError: -+ pass -+else: -+ class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder): -+ """ -+ Meta path finder for sys.path and package __path__ attributes. -+ -+ Patched for https://github.com/python/cpython/issues/115911. -+ """ -+ @classmethod -+ def _path_importer_cache(cls, path): -+ if path == '': -+ try: -+ path = _frozen_importlib_external._os.getcwd() -+ except (FileNotFoundError, PermissionError): -+ return None -+ return super()._path_importer_cache(path) -+ -+ if sys.version_info[:2] <= (3, 12): -+ for i, mpf in enumerate(sys.meta_path): -+ if mpf is _frozen_importlib_external.PathFinder: -+ sys.meta_path[i] = MonkeyPatchedPathFinder -+ del i, mpf -+ -+ - import binascii - import collections - import encodings.latin_1 -@@ -49,18 +77,22 @@ import pstats - import signal - import socket - import struct --import sys - import syslog - import threading - import time - import traceback -+import types - import warnings - import weakref - import zlib - --# Python >3.7 deprecated the imp module. --warnings.filterwarnings('ignore', message='the imp module is deprecated') --import imp -+try: -+ # Python >= 3.4, PEP 451 ModuleSpec API -+ import importlib.machinery -+ import importlib.util -+except ImportError: -+ # Python < 3.4, PEP 302 Import Hooks -+ import imp - - # Absolute imports for <2.5. - select = __import__('select') -@@ -1358,6 +1390,19 @@ class Importer(object): - def __repr__(self): - return 'Importer' - -+ @staticmethod -+ def _loader_from_module(module, default=None): -+ """Return the loader for a module object.""" -+ try: -+ return module.__spec__.loader -+ except AttributeError: -+ pass -+ try: -+ return module.__loader__ -+ except AttributeError: -+ pass -+ return default -+ - def builtin_find_module(self, fullname): - # imp.find_module() will always succeed for __main__, because it is a - # built-in module. That means it exists on a special linked list deep -@@ -1365,12 +1410,19 @@ class Importer(object): - if fullname == '__main__': - raise ModuleNotFoundError() - -+ # For a module inside a package (e.g. pkg_a.mod_b) use the search path -+ # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']). - parent, _, modname = str_rpartition(fullname, '.') - if parent: - path = sys.modules[parent].__path__ - else: - path = None - -+ # For a top-level module search builtin modules, frozen modules, -+ # system specific locations (e.g. Windows registry, site-packages). -+ # Otherwise use search path of the parent package. -+ # Works for both stdlib modules & third-party modules. -+ # If the search is unsuccessful then raises ImportError. - fp, pathname, description = imp.find_module(modname, path) - if fp: - fp.close() -@@ -1382,8 +1434,9 @@ class Importer(object): - Implements importlib.abc.MetaPathFinder.find_module(). - Deprecrated in Python 3.4+, replaced by find_spec(). - Raises ImportWarning in Python 3.10+. -+ Removed in Python 3.12. - -- fullname A (fully qualified?) module name, e.g. "os.path". -+ fullname Fully qualified module name, e.g. "os.path". - path __path__ of parent packge. None for a top level module. - """ - if hasattr(_tls, 'running'): -@@ -1393,14 +1446,13 @@ class Importer(object): - try: - #_v and self._log.debug('Python requested %r', fullname) - fullname = to_text(fullname) -- pkgname, dot, _ = str_rpartition(fullname, '.') -+ pkgname, _, suffix = str_rpartition(fullname, '.') - pkg = sys.modules.get(pkgname) - if pkgname and getattr(pkg, '__loader__', None) is not self: - self._log.debug('%s is submodule of a locally loaded package', - fullname) - return None - -- suffix = fullname[len(pkgname+dot):] - if pkgname and suffix not in self._present.get(pkgname, ()): - self._log.debug('%s has no submodule %s', pkgname, suffix) - return None -@@ -1420,6 +1472,66 @@ class Importer(object): - finally: - del _tls.running - -+ def find_spec(self, fullname, path, target=None): -+ """ -+ Return a `ModuleSpec` for module with `fullname` if we will load it. -+ Otherwise return `None`, allowing other finders to try. -+ -+ fullname Fully qualified name of the module (e.g. foo.bar.baz) -+ path Path entries to search. None for a top-level module. -+ target Existing module to be reloaded (if any). -+ -+ Implements importlib.abc.MetaPathFinder.find_spec() -+ Python 3.4+. -+ """ -+ # Presence of _tls.running indicates we've re-invoked importlib. -+ # Abort early to prevent infinite recursion. See below. -+ if hasattr(_tls, 'running'): -+ return None -+ -+ log = self._log.getChild('find_spec') -+ -+ if fullname.endswith('.'): -+ return None -+ -+ pkgname, _, modname = fullname.rpartition('.') -+ if pkgname and modname not in self._present.get(pkgname, ()): -+ log.debug('Skipping %s. Parent %s has no submodule %s', -+ fullname, pkgname, modname) -+ return None -+ -+ pkg = sys.modules.get(pkgname) -+ pkg_loader = self._loader_from_module(pkg) -+ if pkgname and pkg_loader is not self: -+ log.debug('Skipping %s. Parent %s was loaded by %r', -+ fullname, pkgname, pkg_loader) -+ return None -+ -+ # #114: whitelisted prefixes override any system-installed package. -+ if self.whitelist != ['']: -+ if any(s and fullname.startswith(s) for s in self.whitelist): -+ log.debug('Handling %s. It is whitelisted', fullname) -+ return importlib.machinery.ModuleSpec(fullname, loader=self) -+ -+ if fullname == '__main__': -+ log.debug('Handling %s. A special case', fullname) -+ return importlib.machinery.ModuleSpec(fullname, loader=self) -+ -+ # Re-invoke the import machinery to allow other finders to try. -+ # Set a guard, so we don't infinitely recurse. See top of this method. -+ _tls.running = True -+ try: -+ spec = importlib.util._find_spec(fullname, path, target) -+ finally: -+ del _tls.running -+ -+ if spec: -+ log.debug('Skipping %s. Available as %r', fullname, spec) -+ return spec -+ -+ log.debug('Handling %s. Unavailable locally', fullname) -+ return importlib.machinery.ModuleSpec(fullname, loader=self) -+ - blacklisted_msg = ( - '%r is present in the Mitogen importer blacklist, therefore this ' - 'context will not attempt to request it from the master, as the ' -@@ -1506,6 +1618,64 @@ class Importer(object): - if present: - callback() - -+ def create_module(self, spec): -+ """ -+ Return a module object for the given ModuleSpec. -+ -+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. -+ Unlike Loader.load_module() this shouldn't populate sys.modules or -+ set module attributes. Both are done by Python. -+ """ -+ self._log.debug('Creating module for %r', spec) -+ -+ # FIXME Should this be done in find_spec()? Can it? -+ self._refuse_imports(spec.name) -+ -+ # FIXME "create_module() should properly handle the case where it is -+ # called more than once for the same spec/module." -- PEP-451 -+ event = threading.Event() -+ self._request_module(spec.name, callback=event.set) -+ event.wait() -+ -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ _, pkg_present, path, _, _ = self._cache[spec.name] -+ -+ if path is None: -+ raise ImportError(self.absent_msg % (spec.name)) -+ -+ spec.origin = self.get_filename(spec.name) -+ if pkg_present is not None: -+ # TODO Namespace packages -+ spec.submodule_search_locations = [] -+ self._present[spec.name] = pkg_present -+ -+ module = types.ModuleType(spec.name) -+ # FIXME create_module() shouldn't initialise module attributes -+ module.__file__ = spec.origin -+ return module -+ -+ def exec_module(self, module): -+ """ -+ Execute the module to initialise it. Don't return anything. -+ -+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. -+ """ -+ name = module.__spec__.name -+ origin = module.__spec__.origin -+ self._log.debug('Executing %s from %s', name, origin) -+ source = self.get_source(name) -+ try: -+ # Compile the source into a code object. Don't add any __future__ -+ # flags and don't inherit any from this module. -+ # FIXME Should probably be exposed as get_code() -+ code = compile(source, origin, 'exec', flags=0, dont_inherit=True) -+ except SyntaxError: -+ # FIXME Why is this LOG, rather than self._log? -+ LOG.exception('while importing %r', name) -+ raise -+ -+ exec(code, module.__dict__) -+ - def load_module(self, fullname): - """ - Return the loaded module specified by fullname. -@@ -1521,11 +1691,11 @@ class Importer(object): - self._request_module(fullname, event.set) - event.wait() - -- ret = self._cache[fullname] -- if ret[2] is None: -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ _, pkg_present, path, _, _ = self._cache[fullname] -+ if path is None: - raise ModuleNotFoundError(self.absent_msg % (fullname,)) - -- pkg_present = ret[1] - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - mod.__file__ = self.get_filename(fullname) - mod.__loader__ = self -@@ -3926,7 +4096,7 @@ class ExternalContext(object): - - def _setup_package(self): - global mitogen -- mitogen = imp.new_module('mitogen') -+ mitogen = types.ModuleType('mitogen') - mitogen.__package__ = 'mitogen' - mitogen.__path__ = [] - mitogen.__loader__ = self.importer -diff --git a/mitogen/master.py b/mitogen/master.py -index 4fb535f..b1e0a1d 100644 ---- a/mitogen/master.py -+++ b/mitogen/master.py -@@ -37,7 +37,6 @@ contexts. - - import dis - import errno --import imp - import inspect - import itertools - import logging -@@ -50,6 +49,16 @@ import threading - import types - import zlib - -+try: -+ # Python >= 3.4, PEP 451 ModuleSpec API -+ import importlib.machinery -+ import importlib.util -+ from _imp import is_builtin as _is_builtin -+except ImportError: -+ # Python < 3.4, PEP 302 Import Hooks -+ import imp -+ from imp import is_builtin as _is_builtin -+ - try: - import sysconfig - except ImportError: -@@ -122,14 +131,16 @@ def is_stdlib_name(modname): - """ - Return :data:`True` if `modname` appears to come from the standard library. - """ -- # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. -+ # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. -+ # Returns 1 if modname names a module that is "builtin" to the the Python -+ # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). - # - # """ - # Main is a little special - imp.is_builtin("__main__") will return False, - # but BuiltinImporter is still the most appropriate initial setting for - # its __loader__ attribute. - # """ -- comment in CPython pylifecycle.c:add_main_module() -- if imp.is_builtin(modname) != 0: -+ if _is_builtin(modname) != 0: - return True - - module = sys.modules.get(modname) -@@ -460,6 +471,9 @@ class FinderMethod(object): - name according to the running Python interpreter. You'd think this was a - simple task, right? Naive young fellow, welcome to the real world. - """ -+ def __init__(self): -+ self.log = LOG.getChild(self.__class__.__name__) -+ - def __repr__(self): - return '%s()' % (type(self).__name__,) - -@@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod): - return path, source, is_pkg - - --class ParentEnumerationMethod(FinderMethod): -+class ParentImpEnumerationMethod(FinderMethod): - """ - Attempt to fetch source code by examining the module's (hopefully less - insane) parent package, and if no insane parents exist, simply use -@@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod): - def _find_one_component(self, modname, search_path): - try: - #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) -+ # FIXME The imp module was removed in Python 3.12. - return imp.find_module(modname, search_path) - except ImportError: - e = sys.exc_info()[1] -@@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod): - """ - See implementation for a description of how this works. - """ -+ if sys.version_info >= (3, 4): -+ return None -+ - #if fullname not in sys.modules: - # Don't attempt this unless a module really exists in sys.modules, - # else we could return junk. -@@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod): - return self._found_module(fullname, path, fp) - - -+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): -+ def _find_parent_spec(self, fullname): -+ #history = [] -+ debug = self.log.debug -+ children = [] -+ for parent_name, child_name in self._iter_parents(fullname): -+ children.insert(0, child_name) -+ if not parent_name: -+ debug('abandoning %r, reached top-level', fullname) -+ return None, children -+ -+ try: -+ parent = sys.modules[parent_name] -+ except KeyError: -+ debug('skipping %r, not in sys.modules', parent_name) -+ continue -+ -+ try: -+ spec = parent.__spec__ -+ except AttributeError: -+ debug('skipping %r: %r.__spec__ is absent', -+ parent_name, parent) -+ continue -+ -+ if not spec: -+ debug('skipping %r: %r.__spec__=%r', -+ parent_name, parent, spec) -+ continue -+ -+ if spec.name != parent_name: -+ debug('skipping %r: %r.__spec__.name=%r does not match', -+ parent_name, parent, spec.name) -+ continue -+ -+ if not spec.submodule_search_locations: -+ debug('skipping %r: %r.__spec__.submodule_search_locations=%r', -+ parent_name, parent, spec.submodule_search_locations) -+ continue -+ -+ return spec, children -+ -+ raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' -+ % (self.__class__.__name__, fullname)) -+ -+ def find(self, fullname): -+ # Returns absolute path, ParentImpEnumerationMethod returns relative -+ # >>> spec_pem.find('six_brokenpkg._six')[::2] -+ # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) -+ -+ if sys.version_info < (3, 4): -+ return None -+ -+ fullname = to_text(fullname) -+ spec, children = self._find_parent_spec(fullname) -+ for child_name in children: -+ if spec: -+ name = '%s.%s' % (spec.name, child_name) -+ submodule_search_locations = spec.submodule_search_locations -+ else: -+ name = child_name -+ submodule_search_locations = None -+ spec = importlib.util._find_spec(name, submodule_search_locations) -+ if spec is None: -+ self.log.debug('%r spec unavailable from %s', fullname, spec) -+ return None -+ -+ is_package = spec.submodule_search_locations is not None -+ if name != fullname: -+ if not is_package: -+ self.log.debug('%r appears to be child of non-package %r', -+ fullname, spec) -+ return None -+ continue -+ -+ if not spec.has_location: -+ self.log.debug('%r.origin cannot be read as a file', spec) -+ return None -+ -+ if os.path.splitext(spec.origin)[1] != '.py': -+ self.log.debug('%r.origin does not contain Python source code', -+ spec) -+ return None -+ -+ # FIXME This should use loader.get_source() -+ with open(spec.origin, 'rb') as f: -+ source = f.read() -+ -+ return spec.origin, source, is_package -+ -+ raise ValueError('%s.find(%r) unexpectedly reached bottom' -+ % (self.__class__.__name__, fullname)) -+ -+ - class ModuleFinder(object): - """ - Given the name of a loaded module, make a best-effort attempt at finding -@@ -838,7 +949,8 @@ class ModuleFinder(object): - DefectivePython3xMainMethod(), - PkgutilMethod(), - SysModulesMethod(), -- ParentEnumerationMethod(), -+ ParentSpecEnumerationMethod(), -+ ParentImpEnumerationMethod(), - ] - - def get_module_source(self, fullname): -diff --git a/mitogen/parent.py b/mitogen/parent.py -index 32aa3cb..fe3e8d1 100644 ---- a/mitogen/parent.py -+++ b/mitogen/parent.py -@@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive - connection. - """ - --import codecs -+import binascii - import errno - import fcntl - import getpass -@@ -1405,10 +1405,14 @@ class Connection(object): - # file descriptor 0 as 100, creates a pipe, then execs a new interpreter - # with a custom argv. - # * Optimized for minimum byte count after minification & compression. -+ # The script preamble_size.py measures this. - # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with - # their respective values. - # * CONTEXT_NAME must be prefixed with the name of the Python binary in - # order to allow virtualenvs to detect their install prefix. -+ # -+ # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible -+ # module test_echo_module, used by the integration tests. - # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version - # switcher as /usr/bin/python, which introspects argv0. To workaround - # it we redirect attempts to call /usr/bin/python with an explicit -@@ -1417,7 +1421,8 @@ class Connection(object): - # do something slightly different. The Python executable is patched to - # perform an extra execvp(). I don't fully understand the details, but - # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. -- # * macOS 13.x (Darwin 22?) may remove python 2.x entirely. -+ # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python. -+ # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python - # - # Locals: - # R: read side of interpreter stdin. -@@ -1445,7 +1450,7 @@ class Connection(object): - os.environ['ARGV0']=sys.executable - os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') - os.write(1,'MITO000\n'.encode()) -- C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') -+ C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN)) - fp=os.fdopen(W,'wb',0) - fp.write(C) - fp.close() -@@ -1477,16 +1482,16 @@ class Connection(object): - source = source.replace('PREAMBLE_COMPRESSED_LEN', - str(len(preamble_compressed))) - compressed = zlib.compress(source.encode(), 9) -- encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) -- # We can't use bytes.decode() in 3.x since it was restricted to always -- # return unicode, so codecs.decode() is used instead. In 3.x -- # codecs.decode() requires a bytes object. Since we must be compatible -- # with 2.4 (no bytes literal), an extra .encode() either returns the -- # same str (2.x) or an equivalent bytes (3.x). -+ encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) -+ -+ # Just enough to decode, decompress, and exec the first stage. -+ # Priorities: wider compatibility, faster startup, shorter length. -+ # `import os` here, instead of stage 1, to save a few bytes. -+ # `sys.path=...` for https://github.com/python/cpython/issues/115911. - return self.get_python_argv() + [ - '-c', -- 'import codecs,os,sys;_=codecs.decode;' -- 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) -+ 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' -+ 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), - ] - - def get_econtext_config(self): -diff --git a/setup.py b/setup.py -index 4d7fadf..b17dab9 100644 ---- a/setup.py -+++ b/setup.py -@@ -78,6 +78,7 @@ setup( - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', -+ 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: System :: Distributed Computing', - 'Topic :: System :: Systems Administration', -diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show -new file mode 100755 -index 0000000..42ef319 ---- /dev/null -+++ b/tests/ansible/files/cwd_show -@@ -0,0 +1,22 @@ -+#!/usr/bin/env bash -+ -+# Show permissions and identities that impact the current working directory. -+# On macOS libc cwd() can return EACCES after su or sudo. -+# See also -+# - https://github.com/ansible/ansible/pull/7078 -+# - https://github.com/python/cpython/issues/115911 -+ -+set -o errexit -+set -o nounset -+set -o pipefail -+ -+whoami -+groups -+pwd -+ -+d=$(pwd) -+while [[ "$d" != "/" && -n "$d" ]]; do -+ ls -ld "$d" -+ d=$(dirname "$d") -+done -+ls -ld / -diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts -index a18b90d..ac744ed 100644 ---- a/tests/ansible/hosts/become_same_user.hosts -+++ b/tests/ansible/hosts/become_same_user.hosts -@@ -1,3 +1,5 @@ -+# code: language=ini -+# vim: syntax=dosini - - # become_same_user.yml - bsu-joe ansible_user=joe -diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts -index a22bd5d..4ae861b 100644 ---- a/tests/ansible/hosts/connection_delegation.hosts -+++ b/tests/ansible/hosts/connection_delegation.hosts -@@ -1,3 +1,4 @@ -+# code: language=ini - # vim: syntax=dosini - - # Connection delegation scenarios. It's impossible to connect to them, but their would-be -diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts -index 7ef231b..4f5ea4c 100644 ---- a/tests/ansible/hosts/default.hosts -+++ b/tests/ansible/hosts/default.hosts -@@ -1,9 +1,12 @@ -+# code: language=ini - # vim: syntax=dosini - - # When running the tests outside CI, make a single 'target' host which is the - # local machine. The ansible_user override is necessary since some tests want a - # fixed ansible.cfg remote_user setting to test against. --target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" -+# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 -+# and os.environ['USER'] is not populated on Azure macOS runners. -+target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" - - [test-targets] - target -diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts -index 34e1ff9..b210164 100644 ---- a/tests/ansible/hosts/k3.hosts -+++ b/tests/ansible/hosts/k3.hosts -@@ -1,3 +1,4 @@ -+# code: language=ini - # vim: syntax=dosini - - # Used for manual testing. -diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts -index 41af412..e42221e 100644 ---- a/tests/ansible/hosts/localhost.hosts -+++ b/tests/ansible/hosts/localhost.hosts -@@ -1,3 +1,4 @@ -+# code: language=ini - # vim: syntax=dosini - - # issue #511, #536: we must not define an explicit localhost, as some -diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts -index d6a15c9..1c1c2e1 100644 ---- a/tests/ansible/hosts/transport_config.hosts -+++ b/tests/ansible/hosts/transport_config.hosts -@@ -1,3 +1,6 @@ -+# code: language=ini -+# vim: syntax=dosini -+ - # integration/transport_config - # Hosts with twiddled configs that need to be checked somehow. - -@@ -17,8 +20,9 @@ tc_remote_user - tc_transport - - [transport_config_undiscover:vars] --# If python interpreter path is unset, Ansible tries to connect & discover it. --# That causes approx 10 seconds timeout per task - there's no host to connect to. -+# If ansible_*_interpreter isn't set Ansible tries to connect & discover it. -+# If that target doesn't exist we must wait $timeout seconds for each attempt. -+# Setting a known (invalid) interpreter skips discovery & the many timeouts. - # This optimisation should not be relied in any test. - # Note: tc-python-path-* are intentionally not included. - ansible_python_interpreter = python3000 -diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml -index bd6a0ae..52d420d 100644 ---- a/tests/ansible/integration/become/su_password.yml -+++ b/tests/ansible/integration/become/su_password.yml -@@ -1,5 +1,4 @@ - # Verify passwordful su behaviour -- # Ansible can't handle this on OS X. I don't care why. - - - name: integration/become/su_password.yml - hosts: test-targets -@@ -44,20 +43,54 @@ - fail_msg: out={{out}} - when: is_mitogen - -- - name: Ensure password su succeeds. -+ - name: Ensure password su with chdir succeeds - shell: whoami -+ args: -+ chdir: ~mitogen__user1 - become: true - become_user: mitogen__user1 - register: out - vars: - ansible_become_pass: user1_password -- when: is_mitogen -+ when: -+ # https://github.com/ansible/ansible/pull/70785 -+ - ansible_facts.distribution not in ["MacOSX"] -+ or ansible_version.full is version("2.11", ">=", strict=True) -+ or is_mitogen - - - assert: - that: - - out.stdout == 'mitogen__user1' - fail_msg: out={{out}} -- when: is_mitogen -+ when: -+ # https://github.com/ansible/ansible/pull/70785 -+ - ansible_facts.distribution not in ["MacOSX"] -+ or ansible_version.full is version("2.11", ">=", strict=True) -+ or is_mitogen -+ -+ - name: Ensure password su without chdir succeeds -+ shell: whoami -+ become: true -+ become_user: mitogen__user1 -+ register: out -+ vars: -+ ansible_become_pass: user1_password -+ when: -+ # https://github.com/ansible/ansible/pull/70785 -+ - ansible_facts.distribution not in ["MacOSX"] -+ or ansible_version.full is version("2.11", ">=", strict=True) -+ or is_mitogen -+ -+ - assert: -+ that: -+ - out.stdout == 'mitogen__user1' -+ fail_msg: out={{out}} -+ when: -+ # https://github.com/ansible/ansible/pull/70785 -+ - ansible_facts.distribution not in ["MacOSX"] -+ or ansible_version.full is version("2.11", ">=", strict=True) -+ or is_mitogen -+ - tags: - - su - - su_password -diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml -index 36de165..a2d5398 100644 ---- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml -+++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml -@@ -41,7 +41,7 @@ - 'keepalive_count': 10, - 'password': null, - 'port': null, -- 'python_path': ["/usr/bin/python"], -+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], - 'remote_name': null, - 'ssh_args': [ - '-o', -@@ -71,7 +71,7 @@ - 'keepalive_count': 10, - 'password': null, - 'port': null, -- 'python_path': ["/usr/bin/python"], -+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], - 'remote_name': null, - 'ssh_args': [ - '-o', -diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml -index ea828aa..6ac9bad 100644 ---- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml -+++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml -@@ -37,6 +37,7 @@ - vars: - ansible_python_interpreter: auto - test_echo_module: -+ facts_copy: "{{ ansible_facts }}" - register: echoout - - # can't test this assertion: -@@ -44,11 +45,24 @@ - # because Mitogen's ansible_python_interpreter is a connection-layer configurable that - # "must be extracted during each task execution to form the complete connection-layer configuration". - # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered -- - assert: -+ - name: assert discovered python matches invoked python -+ assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python is defined -- - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python -- fail_msg: auto_out={{auto_out}} echoout={{echoout}} -+ - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen -+ - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved -+ fail_msg: -+ - "auto_out: {{ auto_out }}" -+ - "echoout: {{ echoout }}" -+ when: -+ # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always -+ # reports the same path. I can't reach via symlinks. -+ # >>> sys.executable -+ # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python -+ - is_mitogen -+ or echoout.running_python.sys.version_info.major != 2 -+ or not (echoout.running_python.sys.platform == "darwin" -+ and echoout.running_python.platform.release.major == 20) - - - - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result -@@ -128,7 +142,8 @@ - - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter - block: - - test_echo_module: -- facts: -+ facts_copy: "{{ ansible_facts }}" -+ facts_to_override: - ansible_discovered_interpreter_bogus: from module - discovered_interpreter_bogus: from_module - ansible_bogus_interpreter: from_module -@@ -189,13 +204,6 @@ - - distro == 'ubuntu' - - distro_version is version('16.04', '>=', strict=True) - -- - name: mac assertions -- assert: -- that: -- - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' -- fail_msg: auto_out={{auto_out}} -- when: os_family == 'Darwin' -- - always: - - meta: clear_facts - when: -diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml -index 38d1012..6ffff5f 100644 ---- a/tests/ansible/integration/interpreter_discovery/complex_args.yml -+++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml -@@ -4,6 +4,10 @@ - - name: integration/interpreter_discovery/complex_args.yml - hosts: test-targets - gather_facts: true -+ environment: -+ http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}" -+ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" -+ no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" - tasks: - - name: create temp file to source - file: -@@ -21,28 +25,24 @@ - # special_python: source /tmp/fake && python - - name: set python using sourced file - set_fact: -- special_python: source /tmp/fake || true && python -+ # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). -+ special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" - - - name: run get_url with specially-sourced python - get_url: -- url: https://google.com -+ # Plain http for wider Ansible & Python version compatibility -+ url: http://httpbin.org/get - dest: "/tmp/" - mode: 0644 -- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs -- validate_certs: no - vars: - ansible_python_interpreter: "{{ special_python }}" -- environment: -- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" -- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - - - name: run get_url with specially-sourced python including jinja - get_url: -- url: https://google.com -+ # Plain http for wider Ansible & Python version compatibility -+ url: http://httpbin.org/get - dest: "/tmp/" - mode: 0644 -- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs -- validate_certs: no - vars: - ansible_python_interpreter: > - {% if "1" == "1" %} -@@ -50,8 +50,5 @@ - {% else %} - python - {% endif %} -- environment: -- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" -- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - tags: - - complex_args -diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml -index 0c620da..0d7cf1b 100644 ---- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml -+++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml -@@ -2,6 +2,11 @@ - - name: integration/runner/custom_python_new_style_module.yml - hosts: test-targets - tasks: -+ # FIXME Without Mitogen Ansible often reads stdin before the module. -+ # Either don't read directly from stdin, or figure out the cause. -+ - meta: end_play -+ when: not is_mitogen -+ - - custom_python_new_style_missing_interpreter: - foo: true - with_sequence: start=0 end={{end|default(1)}} -diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml -index e2384f8..8435b15 100644 ---- a/tests/ansible/integration/runner/custom_python_new_style_module.yml -+++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml -@@ -1,7 +1,8 @@ - - name: integration/runner/custom_python_new_style_module.yml - hosts: test-targets - tasks: -- # without Mitogen Ansible 2.10 hangs on this play -+ # FIXME Without Mitogen Ansible often reads stdin before the module. -+ # Either don't read directly from stdin, or figure out the cause. - - meta: end_play - when: not is_mitogen - -diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml -index f2a3eef..ebe34cc 100644 ---- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml -+++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml -@@ -1,3 +1,7 @@ -+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which -+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module -+# as it is sent to a target. There are probably very few modules in the wild -+# that still do this, if any - reload() is a Python 2.x builtin function. - # issue #555 - - - name: integration/runner/custom_python_prehistoric_module.yml -@@ -5,9 +9,11 @@ - tasks: - - custom_python_prehistoric_module: - register: out -+ when: is_mitogen - - - assert: - that: out.ok - fail_msg: out={{out}} -+ when: is_mitogen - tags: - - custom_python_prehistoric_module -diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py -index c7a222e..d2ceaf0 100644 ---- a/tests/ansible/lib/modules/custom_python_detect_environment.py -+++ b/tests/ansible/lib/modules/custom_python_detect_environment.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible new-style Python module. I return details about the Python - # interpreter I run within. - -@@ -25,6 +25,11 @@ except NameError: - def main(): - module = AnsibleModule(argument_spec={}) - module.exit_json( -+ fs={ -+ '/tmp': { -+ 'resolved': os.path.realpath('/tmp'), -+ }, -+ }, - python={ - 'version': { - 'full': '%i.%i.%i' % sys.version_info[:3], -diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py -index ae1b78c..507e53d 100644 ---- a/tests/ansible/lib/modules/custom_python_external_module.py -+++ b/tests/ansible/lib/modules/custom_python_external_module.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I expect the quote from modules2/module_utils/joker.py. - - from ansible.module_utils.basic import AnsibleModule -diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py -index be9acb2..95bd0c7 100644 ---- a/tests/ansible/lib/modules/custom_python_external_pkg.py -+++ b/tests/ansible/lib/modules/custom_python_external_pkg.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - - from ansible.module_utils.basic import AnsibleModule - from ansible.module_utils.externalpkg import extmod -diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py -index 6164057..846037e 100755 ---- a/tests/ansible/lib/modules/custom_python_json_args_module.py -+++ b/tests/ansible/lib/modules/custom_python_json_args_module.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible Python JSONARGS module. I should receive an encoding string. - - json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>""" -diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py -index 255e372..1d34232 100644 ---- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py -+++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible new-style Python module. I leak state from each invocation - # into a class variable and a global variable. - -diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py -index 51b7452..9767f85 100644 ---- a/tests/ansible/lib/modules/custom_python_modify_environ.py -+++ b/tests/ansible/lib/modules/custom_python_modify_environ.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible new-style Python module. I modify the process environment and - # don't clean up after myself. - -diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py -index 2e0ef0d..728685f 100644 ---- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py -+++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py -@@ -1,6 +1,20 @@ - # I am an Ansible new-style Python module, but I lack an interpreter. -+# See also custom_python_new_style_module, we should be updated in tandem. - -+import io -+import json -+import select -+import signal - import sys -+import warnings -+ -+# Ansible 2.7 changed how new style modules are invoked. It seems that module -+# parameters are *sometimes* read before the module runs. Modules that try -+# to read directly from stdin, such as this, are unable to. However it doesn't -+# always fail, influences seem to include Ansible & Python version. As noted -+# in ansible.module_utils.basic._load_params() we should probably use that. -+# I think (medium confidence) I narrowed the inflection (with git bisect) to -+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 - - # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980 - # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10 -@@ -8,11 +22,46 @@ import sys - # from ansible.module_utils. - # import ansible.module_utils. - -+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. -+# Previously this module has waited on stdin forever (timeoutInMinutes=120). -+SELECT_TIMEOUT = 5.0 # seconds -+SIGNAL_TIMEOUT = 10 # seconds -+ -+ -+def fail_json(msg, **kwargs): -+ kwargs.update(failed=True, msg=msg) -+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) -+ sys.exit(1) -+ -+ -+def sigalrm_handler(signum, frame): -+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) -+ - - def usage(): - sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],)) - sys.exit(1) - -+ -+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. -+signal.signal(signal.SIGALRM, sigalrm_handler) -+signal.alarm(SIGNAL_TIMEOUT) -+ -+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. -+# TODO Combine select() & read() in a loop, to handle slow trickle of data. -+# Consider buffering, line buffering, `f.read()` vs `f.read1()`. -+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. -+try: -+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) -+except (AttributeError, TypeError, io.UnsupportedOperation) as exc: -+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. -+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) -+else: -+ if not inputs_ready: -+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" -+ % (SELECT_TIMEOUT,)) -+ -+# Read all data on stdin. May block forever, if EOF is not reached. - input_json = sys.stdin.read() - - print("{") -diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py -index 1e7270c..c84d241 100755 ---- a/tests/ansible/lib/modules/custom_python_new_style_module.py -+++ b/tests/ansible/lib/modules/custom_python_new_style_module.py -@@ -1,16 +1,65 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible new-style Python module. I should receive an encoding string. -+# See also custom_python_new_style_module, we should be updated in tandem. - -+import io -+import json -+import select -+import signal - import sys -+import warnings -+ -+# Ansible 2.7 changed how new style modules are invoked. It seems that module -+# parameters are *sometimes* read before the module runs. Modules that try -+# to read directly from stdin, such as this, are unable to. However it doesn't -+# always fail, influences seem to include Ansible & Python version. As noted -+# in ansible.module_utils.basic._load_params() we should probably use that. -+# I think (medium confidence) I narrowed the inflection (with git bisect) to -+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 - - # This is the magic marker Ansible looks for: - # from ansible.module_utils. - -+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. -+# Previously this module has waited on stdin forever (timeoutInMinutes=120). -+SELECT_TIMEOUT = 5.0 # seconds -+SIGNAL_TIMEOUT = 10 # seconds -+ -+ -+def fail_json(msg, **kwargs): -+ kwargs.update(failed=True, msg=msg) -+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) -+ sys.exit(1) -+ -+ -+def sigalrm_handler(signum, frame): -+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) -+ - - def usage(): - sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],)) - sys.exit(1) - -+ -+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. -+signal.signal(signal.SIGALRM, sigalrm_handler) -+signal.alarm(SIGNAL_TIMEOUT) -+ -+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. -+# TODO Combine select() & read() in a loop, to handle slow trickle of data. -+# Consider buffering, line buffering, `f.read()` vs `f.read1()`. -+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. -+try: -+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) -+except (AttributeError, TypeError, io.UnsupportedOperation) as exc: -+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. -+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) -+else: -+ if not inputs_ready: -+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" -+ % (SELECT_TIMEOUT,)) -+ -+# Read all data on stdin. May block forever, if EOF is not reached. - input_json = sys.stdin.read() - - print("{") -diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py -index c5e264a..d465ac9 100644 ---- a/tests/ansible/lib/modules/custom_python_os_getcwd.py -+++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the - # process environment. - -diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py -index 0cf9774..6139748 100644 ---- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py -+++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py -@@ -1,4 +1,9 @@ --#!/usr/bin/env python -+#!/usr/bin/python -+ -+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which -+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module -+# as it is sent to a target. There are probably very few modules in the wild -+# that still do this, reload() is a Python 2.x builtin function. - # issue #555: I'm a module that cutpastes an old hack. - - from ansible.module_utils.basic import AnsibleModule -diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py -index d6a839a..4a6243d 100644 ---- a/tests/ansible/lib/modules/custom_python_run_script.py -+++ b/tests/ansible/lib/modules/custom_python_run_script.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible new-style Python module. I run the script provided in the - # parameter. - -diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py -index 03f3b6a..6b3a356 100644 ---- a/tests/ansible/lib/modules/custom_python_uses_distro.py -+++ b/tests/ansible/lib/modules/custom_python_uses_distro.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # issue #590: I am an Ansible new-style Python module that tries to use - # ansible.module_utils.distro. - -diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py -index f5e3386..23eeeb5 100755 ---- a/tests/ansible/lib/modules/custom_python_want_json_module.py -+++ b/tests/ansible/lib/modules/custom_python_want_json_module.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. - - import json -diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py -index 1b80a48..289e966 100644 ---- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py -+++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - - # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. - -diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py -new file mode 100644 -index 0000000..41cf1c1 ---- /dev/null -+++ b/tests/ansible/lib/modules/module_finder_test.py -@@ -0,0 +1,12 @@ -+from __future__ import absolute_import, division, print_function -+__metaclass__ = type -+ -+import os -+import sys -+ -+import ansible.module_utils.external1 -+ -+from ansible.module_utils.externalpkg.extmod import path as epem_path -+ -+def main(): -+ pass -diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py -index 1f71e87..d6a5fb9 100644 ---- a/tests/ansible/lib/modules/test_echo_module.py -+++ b/tests/ansible/lib/modules/test_echo_module.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/python - # -*- coding: utf-8 -*- - - # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> -@@ -9,28 +9,61 @@ - from __future__ import absolute_import, division, print_function - __metaclass__ = type - -+import os - import platform - import sys - from ansible.module_utils.basic import AnsibleModule - - - def main(): -- result = dict(changed=False) -- - module = AnsibleModule(argument_spec=dict( -- facts=dict(type=dict, default={}) -+ facts_copy=dict(type=dict, default={}), -+ facts_to_override=dict(type=dict, default={}) - )) - -- result['ansible_facts'] = module.params['facts'] - # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info -- if sys.platform == 'darwin' and sys.executable != '/usr/bin/python': -- if int(platform.release()[:2]) < 19: -- sys.executable = sys.executable[:-3] -- else: -+ # NB This must be synced with mitogen.parent.Connection.get_boot_command() -+ platform_release_major = int(platform.release().partition('.')[0]) -+ if sys.modules.get('mitogen') and sys.platform == 'darwin': -+ if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7': -+ sys.executable = '/usr/bin/python' -+ if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7): - # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 - # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python - sys.executable = "/usr/bin/python" -- result['running_python_interpreter'] = sys.executable -+ -+ facts_copy = module.params['facts_copy'] -+ discovered_interpreter_python = facts_copy['discovered_interpreter_python'] -+ result = { -+ 'changed': False, -+ 'ansible_facts': module.params['facts_to_override'], -+ 'discovered_and_running_samefile': os.path.samefile( -+ os.path.realpath(discovered_interpreter_python), -+ os.path.realpath(sys.executable), -+ ), -+ 'discovered_python': { -+ 'as_seen': discovered_interpreter_python, -+ 'resolved': os.path.realpath(discovered_interpreter_python), -+ }, -+ 'running_python': { -+ 'platform': { -+ 'release': { -+ 'major': platform_release_major, -+ }, -+ }, -+ 'sys': { -+ 'executable': { -+ 'as_seen': sys.executable, -+ 'resolved': os.path.realpath(sys.executable), -+ }, -+ 'platform': sys.platform, -+ 'version_info': { -+ 'major': sys.version_info[0], -+ 'minor': sys.version_info[1], -+ }, -+ }, -+ }, -+ } - - module.exit_json(**result) - -diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml -index f4c47ab..610eaf3 100644 ---- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml -+++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml -@@ -23,9 +23,16 @@ - when: - - lout.python.version.full is version('2.7', '>=', strict=True) - -- - assert: -+ - name: Check virtualenv was used -+ # On macOS runners a symlink /tmp -> /private/tmp has been seen -+ vars: -+ requested_executable: /tmp/issue_152_virtualenv/bin/python -+ expected_executables: -+ - "{{ requested_executable }}" -+ - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" -+ assert: - that: -- - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" -+ - out.sys_executable in expected_executables - fail_msg: out={{out}} - when: - - lout.python.version.full is version('2.7', '>=', strict=True) -diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml -index 2ca6b97..8903494 100644 ---- a/tests/ansible/setup/all.yml -+++ b/tests/ansible/setup/all.yml -@@ -1 +1,2 @@ --- import_playbook: report.yml -+- import_playbook: report_controller.yml -+- import_playbook: report_targets.yml -diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml -deleted file mode 100644 -index 450e4fb..0000000 ---- a/tests/ansible/setup/report.yml -+++ /dev/null -@@ -1,20 +0,0 @@ --- name: Report runtime settings -- hosts: localhost:test-targets -- gather_facts: true -- tasks: -- - debug: {var: ansible_facts.distribution} -- - debug: {var: ansible_facts.distribution_major_version} -- - debug: {var: ansible_facts.distribution_release} -- - debug: {var: ansible_facts.distribution_version} -- - debug: {var: ansible_facts.kernel} -- - debug: {var: ansible_facts.kernel_version} -- - debug: {var: ansible_facts.os_family} -- - debug: {var: ansible_facts.osrevision} -- - debug: {var: ansible_facts.osversion} -- - debug: {var: ansible_facts.python} -- - debug: {var: ansible_facts.system} -- - debug: {var: ansible_forks} -- - debug: {var: ansible_run_tags} -- - debug: {var: ansible_skip_tags} -- - debug: {var: ansible_version.full} -- - debug: {var: is_mitogen} -diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml -new file mode 100644 -index 0000000..d0d5cc1 ---- /dev/null -+++ b/tests/ansible/setup/report_controller.yml -@@ -0,0 +1,17 @@ -+- name: Report controller parameters -+ hosts: localhost -+ gather_facts: false -+ tasks: -+ - debug: -+ msg: -+ - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('<unset>') }}" -+ - ${USER}: "{{ lookup('env', 'USER') | default('<unset>') }}" -+ - $(groups): "{{ lookup('pipe', 'groups') }}" -+ - $(pwd): "{{ lookup('pipe', 'pwd') }}" -+ - $(whoami): "{{ lookup('pipe', 'whoami') }}" -+ - ansible_run_tags: "{{ ansible_run_tags | default('<unset>') }}" -+ - ansible_playbook_python: "{{ ansible_playbook_python | default('<unset>') }}" -+ - ansible_skip_tags: "{{ ansible_skip_tags | default('<unset>') }}" -+ - ansible_version.full: "{{ ansible_version.full | default('<unset>') }}" -+ - is_mitogen: "{{ is_mitogen | default('<unset>') }}" -+ - playbook_dir: "{{ playbook_dir | default('<unset>') }}" -diff --git a/tests/ansible/setup/report_targets.yml b/tests/ansible/setup/report_targets.yml -new file mode 100644 -index 0000000..5aa6712 ---- /dev/null -+++ b/tests/ansible/setup/report_targets.yml -@@ -0,0 +1,15 @@ -+- name: Report target facts -+ hosts: localhost:test-targets -+ gather_facts: true -+ tasks: -+ - debug: {var: ansible_facts.distribution} -+ - debug: {var: ansible_facts.distribution_major_version} -+ - debug: {var: ansible_facts.distribution_release} -+ - debug: {var: ansible_facts.distribution_version} -+ - debug: {var: ansible_facts.kernel} -+ - debug: {var: ansible_facts.kernel_version} -+ - debug: {var: ansible_facts.os_family} -+ - debug: {var: ansible_facts.osrevision} -+ - debug: {var: ansible_facts.osversion} -+ - debug: {var: ansible_facts.python} -+ - debug: {var: ansible_facts.system} -diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py -new file mode 100644 -index 0000000..79e8fdb ---- /dev/null -+++ b/tests/ansible/tests/module_finder_test.py -@@ -0,0 +1,80 @@ -+import os.path -+import sys -+import textwrap -+import unittest -+ -+import ansible_mitogen.module_finder -+ -+import testlib -+ -+ -+class ScanFromListTest(testlib.TestCase): -+ def test_absolute_imports(self): -+ source = textwrap.dedent('''\ -+ from __future__ import absolute_import -+ import a; import b.c; from d.e import f; from g import h, i -+ ''') -+ code = compile(source, '<str>', 'exec') -+ self.assertEqual( -+ list(ansible_mitogen.module_finder.scan_fromlist(code)), -+ [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], -+ ) -+ -+ -+class WalkImportsTest(testlib.TestCase): -+ def test_absolute_imports(self): -+ source = textwrap.dedent('''\ -+ from __future__ import absolute_import -+ import a; import b; import b.c; from b.d import e, f -+ ''') -+ code = compile(source, '<str>', 'exec') -+ -+ self.assertEqual( -+ list(ansible_mitogen.module_finder.walk_imports(code)), -+ ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], -+ ) -+ self.assertEqual( -+ list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), -+ ['b.c', 'b.d', 'b.d.e', 'b.d.f'], -+ ) -+ -+ -+class ScanTest(testlib.TestCase): -+ module_name = 'ansible_module_module_finder_test__this_should_not_matter' -+ module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') -+ search_path = ( -+ 'does_not_exist/module_utils', -+ testlib.ANSIBLE_MODULE_UTILS_DIR, -+ ) -+ -+ @staticmethod -+ def relpath(path): -+ return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) -+ -+ @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') -+ def test_importlib_find_spec(self): -+ scan = ansible_mitogen.module_finder._scan_importlib_find_spec -+ actual = scan(self.module_name, self.module_path, self.search_path) -+ self.assertEqual( -+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], -+ [ -+ ('ansible.module_utils.external1', 'external1.py', False), -+ ('ansible.module_utils.external2', 'external2.py', False), -+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), -+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), -+ ], -+ ) -+ -+ @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') -+ def test_imp_find_module(self): -+ scan = ansible_mitogen.module_finder._scan_imp_find_module -+ actual = scan(self.module_name, self.module_path, self.search_path) -+ self.assertEqual( -+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], -+ [ -+ ('ansible.module_utils.external1', 'external1.py', False), -+ ('ansible.module_utils.external2', 'external2.py', False), -+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), -+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), -+ ], -+ ) -diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml -index 6224b61..0b6d5e6 100644 ---- a/tests/image_prep/_user_accounts.yml -+++ b/tests/image_prep/_user_accounts.yml -@@ -4,6 +4,8 @@ - # WARNING: this creates non-privilged accounts with pre-set passwords! - # - -+- import_playbook: ../ansible/setup/report_controller.yml -+ - - hosts: all - gather_facts: true - strategy: mitogen_free -@@ -37,12 +39,12 @@ - - normal_users: "{{ - lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) -- }}" -+ }}" - - all_users: "{{ - special_users + - normal_users -- }}" -+ }}" - tasks: - - name: Disable non-localhost SSH for Mitogen users - when: false -@@ -100,6 +102,7 @@ - with_items: "{{all_users}}" - copy: - dest: /var/lib/AccountsService/users/mitogen__{{item}} -+ mode: u=rw,go= - content: | - [User] - SystemAccount=true -@@ -108,7 +111,7 @@ - when: ansible_system == 'Linux' and out.stat.exists - service: - name: accounts-daemon -- restarted: true -+ state: restarted - - - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" -@@ -117,6 +120,9 @@ - copy: - dest: ~mitogen__slow_user/.{{item}} - src: ../data/docker/mitogen__slow_user.profile -+ owner: mitogen__slow_user -+ group: mitogen__group -+ mode: u=rw,go=r - with_items: - - bashrc - - profile -@@ -125,6 +131,9 @@ - copy: - dest: ~mitogen__permdenied/.{{item}} - src: ../data/docker/mitogen__permdenied.profile -+ owner: mitogen__permdenied -+ group: mitogen__group -+ mode: u=rw,go=r - with_items: - - bashrc - - profile -@@ -136,20 +145,13 @@ - state: directory - mode: go= - owner: mitogen__has_sudo_pubkey -+ group: mitogen__group - - copy: - dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys - src: ../data/docker/mitogen__has_sudo_pubkey.key.pub - mode: go= - owner: mitogen__has_sudo_pubkey -- -- - name: Install slow profile for one account -- block: -- - copy: -- dest: ~mitogen__slow_user/.profile -- src: ../data/docker/mitogen__slow_user.profile -- - copy: -- dest: ~mitogen__slow_user/.bashrc -- src: ../data/docker/mitogen__slow_user.profile -+ group: mitogen__group - - - name: Require a TTY for two accounts - lineinfile: -diff --git a/tests/importer_test.py b/tests/importer_test.py -index e48c02a..e86af8a 100644 ---- a/tests/importer_test.py -+++ b/tests/importer_test.py -@@ -2,6 +2,7 @@ import sys - import threading - import types - import zlib -+import unittest - - import mock - -@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin): - super(ImporterMixin, self).tearDown() - - -+class InvalidNameTest(ImporterMixin, testlib.TestCase): -+ modname = 'trailingdot.' -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ response = (modname, None, None, None, None) -+ -+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+ def test_find_spec_invalid(self): -+ self.set_get_module_response(self.response) -+ self.assertEqual(self.importer.find_spec(self.modname, path=None), None) -+ -+ -+class MissingModuleTest(ImporterMixin, testlib.TestCase): -+ modname = 'missing' -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ response = (modname, None, None, None, None) -+ -+ @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') -+ def test_load_module_missing(self): -+ self.set_get_module_response(self.response) -+ self.assertRaises(ImportError, self.importer.load_module, self.modname) -+ -+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+ def test_find_spec_missing(self): -+ """ -+ Importer should optimistically offer itself as a module loader -+ when there are no disqualifying criteria. -+ """ -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = self.importer.find_spec(self.modname, path=None) -+ self.assertIsInstance(spec, importlib.machinery.ModuleSpec) -+ self.assertEqual(spec.name, self.modname) -+ self.assertEqual(spec.loader, self.importer) -+ -+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+ def test_create_module_missing(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ self.assertRaises(ImportError, self.importer.create_module, spec) -+ -+ -+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') - class LoadModuleTest(ImporterMixin, testlib.TestCase): - data = zlib.compress(b("data = 1\n\n")) - path = 'fake_module.py' -@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - response = (modname, None, path, data, []) - -- def test_no_such_module(self): -- self.set_get_module_response( -- # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -- (self.modname, None, None, None, None) -- ) -- self.assertRaises(ImportError, -- lambda: self.importer.load_module(self.modname)) -- - def test_module_added_to_sys_modules(self): - self.set_get_module_response(self.response) - mod = self.importer.load_module(self.modname) -@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): - self.assertIsNone(mod.__package__) - - -+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+class ModuleSpecTest(ImporterMixin, testlib.TestCase): -+ data = zlib.compress(b("data = 1\n\n")) -+ path = 'fake_module.py' -+ modname = 'fake_module' -+ -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ response = (modname, None, path, data, []) -+ -+ def test_module_attributes(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ mod = self.importer.create_module(spec) -+ self.assertIsInstance(mod, types.ModuleType) -+ self.assertEqual(mod.__name__, 'fake_module') -+ #self.assertFalse(hasattr(mod, '__file__')) -+ -+ -+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') - class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): - data = zlib.compress(b("data = 1\n\n")) - path = 'fake_module.py' -@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): - self.assertEqual(mod.__package__, 'mypkg') - - -+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): -+ data = zlib.compress(b("data = 1\n\n")) -+ path = 'fake_module.py' -+ modname = 'mypkg.fake_module' -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ response = (modname, None, path, data, []) -+ -+ def test_module_attributes(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ mod = self.importer.create_module(spec) -+ self.assertIsInstance(mod, types.ModuleType) -+ self.assertEqual(mod.__name__, 'mypkg.fake_module') -+ #self.assertFalse(hasattr(mod, '__file__')) -+ -+ -+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') - class LoadModulePackageTest(ImporterMixin, testlib.TestCase): - data = zlib.compress(b("func = lambda: 1\n\n")) - path = 'fake_pkg/__init__.py' -@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): - self.assertEqual(mod.func.__module__, self.modname) - - -+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+class PackageSpecTest(ImporterMixin, testlib.TestCase): -+ data = zlib.compress(b("func = lambda: 1\n\n")) -+ path = 'fake_pkg/__init__.py' -+ modname = 'fake_pkg' -+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related -+ response = (modname, [], path, data, []) -+ -+ def test_module_attributes(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ mod = self.importer.create_module(spec) -+ self.assertIsInstance(mod, types.ModuleType) -+ self.assertEqual(mod.__name__, 'fake_pkg') -+ #self.assertFalse(hasattr(mod, '__file__')) -+ -+ def test_get_filename(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ _ = self.importer.create_module(spec) -+ filename = self.importer.get_filename(self.modname) -+ self.assertEqual('master:fake_pkg/__init__.py', filename) -+ -+ def test_get_source(self): -+ import importlib.machinery -+ self.set_get_module_response(self.response) -+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer) -+ _ = self.importer.create_module(spec) -+ source = self.importer.get_source(self.modname) -+ self.assertEqual(source, -+ mitogen.core.to_text(zlib.decompress(self.data))) -+ -+ - class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): - def initdir(self, caplog): - self.caplog = caplog -diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py -index 68ceeb4..072968f 100644 ---- a/tests/module_finder_test.py -+++ b/tests/module_finder_test.py -@@ -139,9 +139,7 @@ class SysModulesMethodTest(testlib.TestCase): - self.assertIsNone(tup) - - --class GetModuleViaParentEnumerationTest(testlib.TestCase): -- klass = mitogen.master.ParentEnumerationMethod -- -+class ParentEnumerationMixin(object): - def call(self, fullname): - return self.klass().find(fullname) - -@@ -231,6 +229,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): - self.assertEqual(is_pkg, False) - - -+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') -+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): -+ klass = mitogen.master.ParentImpEnumerationMethod -+ -+ -+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') -+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): -+ klass = mitogen.master.ParentSpecEnumerationMethod -+ -+ - class ResolveRelPathTest(testlib.TestCase): - klass = mitogen.master.ModuleFinder - -diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt -new file mode 100644 -index 0000000..bc7f7c2 ---- /dev/null -+++ b/tests/requirements-tox.txt -@@ -0,0 +1,4 @@ -+tox==3.28; python_version == '2.7' -+tox==3.28; python_version == '3.6' -+tox==4.8.0; python_version == '3.7' -+tox>=4.13.0,~=4.0; python_version >= '3.8' -diff --git a/tests/requirements.txt b/tests/requirements.txt -index 9bbfe46..12a1379 100644 ---- a/tests/requirements.txt -+++ b/tests/requirements.txt -@@ -16,4 +16,7 @@ urllib3[secure]==1.23; python_version < '2.7' - urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' - # Last idna compatible with Python 2.6 was idna 2.7. - idna==2.7; python_version < '2.7' --virtualenv==20.10.0 -+ -+virtualenv==20.15.1; python_version == '2.7' -+virtualenv==20.17.1; python_version == '3.6' -+virtualenv==20.25.1; python_version >= '3.7' -diff --git a/tests/testlib.py b/tests/testlib.py -index da4d46f..8f020b3 100644 ---- a/tests/testlib.py -+++ b/tests/testlib.py -@@ -47,8 +47,13 @@ except NameError: - - - LOG = logging.getLogger(__name__) --DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') --MODS_DIR = os.path.join(DATA_DIR, 'importer') -+ -+TESTS_DIR = os.path.join(os.path.dirname(__file__)) -+ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') -+ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') -+ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') -+DATA_DIR = os.path.join(TESTS_DIR, 'data') -+MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') - - sys.path.append(DATA_DIR) - sys.path.append(MODS_DIR) -diff --git a/tox.ini b/tox.ini -index 3b3cb9f..b3e0eb1 100644 ---- a/tox.ini -+++ b/tox.ini -@@ -27,6 +27,9 @@ - # ansible == 4.* ansible-core ~= 2.11.0 - # ansible == 5.* ansible-core ~= 2.12.0 - # ansible == 6.* ansible-core ~= 2.13.0 -+# ansible == 7.x ansible-core ~= 2.14.0 -+# ansible == 8.x ansible-core ~= 2.15.0 -+# ansible == 9.x ansible-core ~= 2.16.0 - - # pip --no-python-version-warning - # pip --disable-pip-version-check -@@ -60,8 +63,8 @@ deps = - ansible2.10: ansible==2.10.7 - ansible3: ansible==3.4.0 - ansible4: ansible==4.10.0 -- ansible5: ansible==5.8.0 -- ansible6: ansible==6.0.0 -+ ansible5: ansible~=5.0 -+ ansible6: ansible~=6.0 - install_command = - python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} - commands_pre = -@@ -82,7 +85,6 @@ passenv = - HOME - setenv = - # See also azure-pipelines.yml -- ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive - ANSIBLE_STRATEGY = mitogen_linear - NOCOVERAGE_ERASE = 1 - NOCOVERAGE_REPORT = 1 -@@ -114,10 +116,26 @@ setenv = - distros_ubuntu1804: DISTROS=ubuntu1804 - distros_ubuntu2004: DISTROS=ubuntu2004 - mode_ansible: MODE=ansible -+ mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive - mode_debops_common: MODE=debops_common -+ mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive - mode_mitogen: MODE=mitogen - strategy_linear: ANSIBLE_STRATEGY=linear -+allowlist_externals = -+ # Added: Tox 3.18: Tox 4.0+ -+ *_install.py -+ *_tests.py -+ aws -+ docker -+ docker-credential-secretservice -+ echo -+ gpg2 -+ pass - whitelist_externals = -+ # Deprecated: Tox 3.18+; Removed: Tox 4.0 -+ *_install.py -+ *_tests.py -+ aws - docker - docker-credential-secretservice - echo diff --git a/debian/patches/python3 b/debian/patches/python3 index 91a6c3c..3a04255 100644 --- a/debian/patches/python3 +++ b/debian/patches/python3 @@ -187,7 +187,7 @@ index 6573e1b..98e93ec 100644 argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) diff --git a/tests/ssh_test.py b/tests/ssh_test.py -index 429fdb5..34e3e03 100644 +index 3149fcb..92cc703 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -18,6 +18,7 @@ class StubSshMixin(testlib.RouterMixin): diff --git a/debian/patches/series b/debian/patches/series index 1c1ad92..9a29acd 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -4,10 +4,7 @@ skip-failing-tests skip-django-tests skip-python2.7-test poll-poller -no-subprocess32 ansible-2.14 ask-become-pass interpreter_python ansible-2.16 -python-3.11 -python-3.12 diff --git a/debian/patches/skip-django-tests b/debian/patches/skip-django-tests index e024d01..a6054e6 100644 --- a/debian/patches/skip-django-tests +++ b/debian/patches/skip-django-tests @@ -4,19 +4,139 @@ Subject: Skip Django tests They require Django, and tend to fail for us. --- - tests/module_finder_test.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) + tests/module_finder_test.py | 125 ++++++++++++++++++++++---------------------- + 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py -index ff18bbc..68ceeb4 100644 +index 67e937e..f5012a8 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py -@@ -296,7 +296,7 @@ class FindRelatedTest(testlib.TestCase): +@@ -305,65 +305,66 @@ class FindRelatedTest(testlib.TestCase): self.assertEqual(set(related), self.SIMPLE_EXPECT) --if sys.version_info > (2, 6): +-class DjangoMixin(object): +- WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') +- +- @classmethod +- def modules_expected_path(cls): +- if sys.version_info[0:2] < (3, 0): +- modules_expected_filename = 'modules_expected_py2x.json' +- elif sys.version_info[0:2] <= (3, 6): +- modules_expected_filename = 'modules_expected_py3x-legacy.json' +- elif sys.version_info[0:2] >= (3, 10): +- modules_expected_filename = 'modules_expected_py3x-new.json' +- return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) +- +- @classmethod +- def setUpClass(cls): +- super(DjangoMixin, cls).setUpClass() +- sys.path.append(cls.WEBPROJECT_PATH) +- os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' +- with open(cls.modules_expected_path(), 'rb') as f: +- cls.MODULES_EXPECTED = json.load(f) +- +- @classmethod +- def tearDownClass(cls): +- sys.path.remove(cls.WEBPROJECT_PATH) +- del os.environ['DJANGO_SETTINGS_MODULE'] +- super(DjangoMixin, cls).tearDownClass() +- +- +-class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): +- maxDiff = None +- +- def test_django_db(self): +- import django.db +- module_finder = mitogen.master.ModuleFinder() +- related = module_finder.find_related('django.db') +- expected = self.MODULES_EXPECTED['find_related']['django.db'] +- self.assertEqual(related, expected) +- +- def test_django_db_models(self): +- import django.db.models +- module_finder = mitogen.master.ModuleFinder() +- related = module_finder.find_related('django.db.models') +- expected = self.MODULES_EXPECTED['find_related']['django.db.models'] +- self.assertEqual(related, expected) +- +- +-class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): +- maxDiff = None +- +- def test_django_db(self): +- import django.db +- module_finder = mitogen.master.ModuleFinder() +- related = module_finder.find_related_imports('django.db') +- expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] +- self.assertEqual(related, expected) +- +- def test_django_db_models(self): +- import django.db.models +- module_finder = mitogen.master.ModuleFinder() +- related = module_finder.find_related_imports('django.db.models') +- expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] +- self.assertEqual(related, expected) +if False: # sys.version_info > (2, 6): - class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') - ++ class DjangoMixin(object): ++ WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') ++ ++ @classmethod ++ def modules_expected_path(cls): ++ if sys.version_info[0:2] < (3, 0): ++ modules_expected_filename = 'modules_expected_py2x.json' ++ elif sys.version_info[0:2] <= (3, 6): ++ modules_expected_filename = 'modules_expected_py3x-legacy.json' ++ elif sys.version_info[0:2] >= (3, 10): ++ modules_expected_filename = 'modules_expected_py3x-new.json' ++ return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) ++ ++ @classmethod ++ def setUpClass(cls): ++ super(DjangoMixin, cls).setUpClass() ++ sys.path.append(cls.WEBPROJECT_PATH) ++ os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' ++ with open(cls.modules_expected_path(), 'rb') as f: ++ cls.MODULES_EXPECTED = json.load(f) ++ ++ @classmethod ++ def tearDownClass(cls): ++ sys.path.remove(cls.WEBPROJECT_PATH) ++ del os.environ['DJANGO_SETTINGS_MODULE'] ++ super(DjangoMixin, cls).tearDownClass() ++ ++ ++ class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): ++ maxDiff = None ++ ++ def test_django_db(self): ++ import django.db ++ module_finder = mitogen.master.ModuleFinder() ++ related = module_finder.find_related('django.db') ++ expected = self.MODULES_EXPECTED['find_related']['django.db'] ++ self.assertEqual(related, expected) ++ ++ def test_django_db_models(self): ++ import django.db.models ++ module_finder = mitogen.master.ModuleFinder() ++ related = module_finder.find_related('django.db.models') ++ expected = self.MODULES_EXPECTED['find_related']['django.db.models'] ++ self.assertEqual(related, expected) ++ ++ ++ class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): ++ maxDiff = None ++ ++ def test_django_db(self): ++ import django.db ++ module_finder = mitogen.master.ModuleFinder() ++ related = module_finder.find_related_imports('django.db') ++ expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] ++ self.assertEqual(related, expected) ++ ++ def test_django_db_models(self): ++ import django.db.models ++ module_finder = mitogen.master.ModuleFinder() ++ related = module_finder.find_related_imports('django.db.models') ++ expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] ++ self.assertEqual(related, expected) diff --git a/docs/_templates/github.html b/docs/_templates/github.html index bb2b5ee..e6ed304 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,4 +1,4 @@ <p> <br> -<a class="github-button" href="https://github.com/dw/mitogen/" data-size="large" data-show-count="true" aria-label="Star dw/mitogen on GitHub">Star</a> +<a class="github-button" href="https://github.com/mitogen-hq/mitogen/" data-size="large" data-show-count="true" aria-label="Star mitogen on GitHub">Star</a> </p> diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 449771b..cb83aa7 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged. .. _Ansible: https://www.ansible.com/ -.. _Bug reports: https://goo.gl/yLKZiJ +.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose Overview @@ -147,8 +147,11 @@ Noteworthy Differences * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. Mitogen 0.3.1+ supports - - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10 - - Ansible 5 and 6; with Python 3.8-3.10 + + - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 + - Ansible 5; with Python 3.8-3.11 + - Ansible 6; with Python 3.8-3.12 + Verify your installation is running one of these versions by checking ``ansible --version`` output. @@ -1291,7 +1294,7 @@ Sample Profiles --------------- The summaries below may be reproduced using data and scripts maintained in the -`pcaps branch <https://github.com/dw/mitogen/tree/pcaps/>`_. Traces were +`pcaps branch <https://github.com/mitogen-hq/mitogen/tree/pcaps/>`_. Traces were recorded using Ansible 2.5.14. @@ -1300,7 +1303,7 @@ Trivial Loop: Local Host This demonstrates Mitogen vs. SSH pipelining to the local machine running `bench/loop-100-items.yml -<https://github.com/dw/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_, +<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_, executing a simple command 100 times. Most Ansible controller overhead is isolated, characterizing just module executor and connection layer performance. Mitogen requires **63x less bandwidth and 5.9x less time**. @@ -1328,7 +1331,7 @@ File Transfer: UK to France ~~~~~~~~~~~~~~~~~~~~~~~~~~~ `This playbook -<https://github.com/dw/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_ +<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_ was used to compare file transfer performance over a ~26 ms link. It uses the ``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the target. diff --git a/docs/changelog.rst b/docs/changelog.rst index d9a54bc..1c7441c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,18 @@ Release Notes </style> To avail of fixes in an unreleased version, please download a ZIP file -`directly from GitHub <https://github.com/dw/mitogen/>`_. +`directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_. + +v0.3.5 (2024-03-17) +------------------- + +* :gh:issue:`987` Support Python 3.11 +* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when + becoming an unprivileged user with Python 3.x +* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>`_, + required by Python 3.12 +* :gh:issue:`1033` Support Python 3.12 + v0.3.4 (2023-07-02) ------------------- @@ -65,7 +76,7 @@ v0.3.0 (2021-10-28) ------------------- This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. -`See here for details <https://github.com/dw/mitogen/pull/715#issuecomment-750697248>`_. +`See here for details <https://github.com/mitogen-hq/mitogen/pull/715#issuecomment-750697248>`_. * :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports * :gh:issue:`770` better check for supported Ansible version @@ -86,7 +97,7 @@ v0.2.10 (2021-10-28) * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in - :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` + :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 * :gh:issue:`775` Add msvcrt to the default module deny list * :gh:issue:`847` Removed historic Continuous Integration reverse shell @@ -173,7 +184,7 @@ Mitogen for Ansible :linux:man7:`unix` sockets across privilege domains. * :gh:issue:`467`: an incompatibility running Mitogen under `Molecule - <https://molecule.readthedocs.io/en/stable/>`_ was resolved. + <https://ansible.readthedocs.io/projects/molecule/>`_ was resolved. * :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of connections, ``async`` tasks, tasks using custom :mod:`module_utils`, @@ -1225,9 +1236,8 @@ Core Library parameter may specify an argument vector prefix rather than a string program path. -* :gh:issue:`300`: the broker could crash on - OS X during shutdown due to scheduled `kqueue - <https://www.freebsd.org/cgi/man.cgi?query=kqueue>`_ filter changes for +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to + scheduled :freebsd:man2:`kqueue` filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. diff --git a/docs/conf.py b/docs/conf.py index 54e3a5c..fb9974c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,8 @@ -import os import sys -sys.path.append('..') sys.path.append('.') -import mitogen -VERSION = '%s.%s.%s' % mitogen.__version__ + +VERSION = '0.3.5' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' @@ -44,15 +42,15 @@ version = VERSION domainrefs = { 'gh:commit': { 'text': '%s', - 'url': 'https://github.com/dw/mitogen/commit/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/commit/%s', }, 'gh:issue': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/issues/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/issues/%s', }, 'gh:pull': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/pull/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/pull/%s', }, 'ans:mod': { 'text': '%s module', @@ -64,30 +62,36 @@ domainrefs = { }, 'freebsd:man2': { 'text': '%s(2)', - 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + 'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s', }, 'linux:man1': { 'text': '%s(1)', - 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + 'url': 'https://man7.org/linux/man-pages/man1/%s.1.html', }, 'linux:man2': { 'text': '%s(2)', - 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + 'url': 'https://man7.org/linux/man-pages/man2/%s.2.html', }, 'linux:man3': { 'text': '%s(3)', - 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + 'url': 'https://man7.org/linux/man-pages/man3/%s.3.html', }, 'linux:man7': { 'text': '%s(7)', - 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + 'url': 'https://man7.org/linux/man-pages/man7/%s.7.html', }, } +# > ## Official guidance +# > Query PyPI’s JSON API to determine where to download files from. +# > ## Predictable URLs +# > You can use our conveyor service to fetch this file, which exists for +# > cases where using the API is impractical or impossible. +# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s -.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__ +.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__ """ % locals() diff --git a/docs/contributors.rst b/docs/contributors.rst index 584c4cd..61a9eb1 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters. <li>randy — <em>desperate for automation</em></li> <li>Michael & Vicky Twomey-Lee</li> <li><a href="http://www.wezm.net/">Wesley Moore</a></li> + <li><a href="https://github.com/baryluk">Witold Baryluk</a></li> </ul> diff --git a/docs/index.rst b/docs/index.rst index d33cf29..32083db 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_ can be built, and while the API is quite friendly and comparable to `Fabric`_, ultimately it is not intended for direct use by consumer software. -.. _Salt: https://docs.saltstack.com/en/latest/ +.. _Salt: https://docs.saltproject.io/en/latest/ .. _Ansible: https://docs.ansible.com/ .. _Fabric: https://www.fabfile.org/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c4674f..a2894c8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ +docutils<0.18 +Jinja2<3 +MarkupSafe<2.1 Sphinx==2.1.2; python_version > '3.0' sphinxcontrib-programoutput==0.14; python_version > '3.0' alabaster==0.7.10; python_version > '3.0' diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 8b32cbc..d030b96 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 4) +__version__ = (0, 3, 5) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index bee722e..cd02012 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the bootstrap implementation sent to every new slave context. """ +import sys +try: + import _frozen_importlib_external +except ImportError: + pass +else: + class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder): + """ + Meta path finder for sys.path and package __path__ attributes. + + Patched for https://github.com/python/cpython/issues/115911. + """ + @classmethod + def _path_importer_cache(cls, path): + if path == '': + try: + path = _frozen_importlib_external._os.getcwd() + except (FileNotFoundError, PermissionError): + return None + return super()._path_importer_cache(path) + + if sys.version_info[:2] <= (3, 12): + for i, mpf in enumerate(sys.meta_path): + if mpf is _frozen_importlib_external.PathFinder: + sys.meta_path[i] = MonkeyPatchedPathFinder + del i, mpf + + import binascii import collections import encodings.latin_1 @@ -49,18 +77,22 @@ import pstats import signal import socket import struct -import sys import syslog import threading import time import traceback +import types import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp # Absolute imports for <2.5. select = __import__('select') @@ -1353,6 +1385,19 @@ class Importer(object): def __repr__(self): return 'Importer' + @staticmethod + def _loader_from_module(module, default=None): + """Return the loader for a module object.""" + try: + return module.__spec__.loader + except AttributeError: + pass + try: + return module.__loader__ + except AttributeError: + pass + return default + def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a # built-in module. That means it exists on a special linked list deep @@ -1360,12 +1405,19 @@ class Importer(object): if fullname == '__main__': raise ModuleNotFoundError() + # For a module inside a package (e.g. pkg_a.mod_b) use the search path + # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']). parent, _, modname = str_rpartition(fullname, '.') if parent: path = sys.modules[parent].__path__ else: path = None + # For a top-level module search builtin modules, frozen modules, + # system specific locations (e.g. Windows registry, site-packages). + # Otherwise use search path of the parent package. + # Works for both stdlib modules & third-party modules. + # If the search is unsuccessful then raises ImportError. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() @@ -1377,8 +1429,9 @@ class Importer(object): Implements importlib.abc.MetaPathFinder.find_module(). Deprecrated in Python 3.4+, replaced by find_spec(). Raises ImportWarning in Python 3.10+. + Removed in Python 3.12. - fullname A (fully qualified?) module name, e.g. "os.path". + fullname Fully qualified module name, e.g. "os.path". path __path__ of parent packge. None for a top level module. """ if hasattr(_tls, 'running'): @@ -1388,14 +1441,13 @@ class Importer(object): try: #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) - pkgname, dot, _ = str_rpartition(fullname, '.') + pkgname, _, suffix = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: self._log.debug('%s is submodule of a locally loaded package', fullname) return None - suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): self._log.debug('%s has no submodule %s', pkgname, suffix) return None @@ -1415,6 +1467,66 @@ class Importer(object): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`, allowing other finders to try. + + fullname Fully qualified name of the module (e.g. foo.bar.baz) + path Path entries to search. None for a top-level module. + target Existing module to be reloaded (if any). + + Implements importlib.abc.MetaPathFinder.find_spec() + Python 3.4+. + """ + # Presence of _tls.running indicates we've re-invoked importlib. + # Abort early to prevent infinite recursion. See below. + if hasattr(_tls, 'running'): + return None + + log = self._log.getChild('find_spec') + + if fullname.endswith('.'): + return None + + pkgname, _, modname = fullname.rpartition('.') + if pkgname and modname not in self._present.get(pkgname, ()): + log.debug('Skipping %s. Parent %s has no submodule %s', + fullname, pkgname, modname) + return None + + pkg = sys.modules.get(pkgname) + pkg_loader = self._loader_from_module(pkg) + if pkgname and pkg_loader is not self: + log.debug('Skipping %s. Parent %s was loaded by %r', + fullname, pkgname, pkg_loader) + return None + + # #114: whitelisted prefixes override any system-installed package. + if self.whitelist != ['']: + if any(s and fullname.startswith(s) for s in self.whitelist): + log.debug('Handling %s. It is whitelisted', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + if fullname == '__main__': + log.debug('Handling %s. A special case', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + # Re-invoke the import machinery to allow other finders to try. + # Set a guard, so we don't infinitely recurse. See top of this method. + _tls.running = True + try: + spec = importlib.util._find_spec(fullname, path, target) + finally: + del _tls.running + + if spec: + log.debug('Skipping %s. Available as %r', fullname, spec) + return spec + + log.debug('Handling %s. Unavailable locally', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1501,6 +1613,64 @@ class Importer(object): if present: callback() + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + self._log.debug('Creating module for %r', spec) + + # FIXME Should this be done in find_spec()? Can it? + self._refuse_imports(spec.name) + + # FIXME "create_module() should properly handle the case where it is + # called more than once for the same spec/module." -- PEP-451 + event = threading.Event() + self._request_module(spec.name, callback=event.set) + event.wait() + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[spec.name] + + if path is None: + raise ImportError(self.absent_msg % (spec.name)) + + spec.origin = self.get_filename(spec.name) + if pkg_present is not None: + # TODO Namespace packages + spec.submodule_search_locations = [] + self._present[spec.name] = pkg_present + + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + name = module.__spec__.name + origin = module.__spec__.origin + self._log.debug('Executing %s from %s', name, origin) + source = self.get_source(name) + try: + # Compile the source into a code object. Don't add any __future__ + # flags and don't inherit any from this module. + # FIXME Should probably be exposed as get_code() + code = compile(source, origin, 'exec', flags=0, dont_inherit=True) + except SyntaxError: + # FIXME Why is this LOG, rather than self._log? + LOG.exception('while importing %r', name) + raise + + exec(code, module.__dict__) + def load_module(self, fullname): """ Return the loaded module specified by fullname. @@ -1516,11 +1686,11 @@ class Importer(object): self._request_module(fullname, event.set) event.wait() - ret = self._cache[fullname] - if ret[2] is None: + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[fullname] + if path is None: raise ModuleNotFoundError(self.absent_msg % (fullname,)) - pkg_present = ret[1] mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -3921,7 +4091,7 @@ class ExternalContext(object): def _setup_package(self): global mitogen - mitogen = imp.new_module('mitogen') + mitogen = types.ModuleType('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f..b1e0a1d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -37,7 +37,6 @@ contexts. import dis import errno -import imp import inspect import itertools import logging @@ -51,6 +50,16 @@ import types import zlib try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util + from _imp import is_builtin as _is_builtin +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + from imp import is_builtin as _is_builtin + +try: import sysconfig except ImportError: sysconfig = None @@ -122,14 +131,16 @@ def is_stdlib_name(modname): """ Return :data:`True` if `modname` appears to come from the standard library. """ - # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. + # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. + # Returns 1 if modname names a module that is "builtin" to the the Python + # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). # # """ # Main is a little special - imp.is_builtin("__main__") will return False, # but BuiltinImporter is still the most appropriate initial setting for # its __loader__ attribute. # """ -- comment in CPython pylifecycle.c:add_main_module() - if imp.is_builtin(modname) != 0: + if _is_builtin(modname) != 0: return True module = sys.modules.get(modname) @@ -460,6 +471,9 @@ class FinderMethod(object): name according to the running Python interpreter. You'd think this was a simple task, right? Naive young fellow, welcome to the real world. """ + def __init__(self): + self.log = LOG.getChild(self.__class__.__name__) + def __repr__(self): return '%s()' % (type(self).__name__,) @@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod): return path, source, is_pkg -class ParentEnumerationMethod(FinderMethod): +class ParentImpEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package, and if no insane parents exist, simply use @@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod): def _find_one_component(self, modname, search_path): try: #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + # FIXME The imp module was removed in Python 3.12. return imp.find_module(modname, search_path) except ImportError: e = sys.exc_info()[1] @@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod): """ See implementation for a description of how this works. """ + if sys.version_info >= (3, 4): + return None + #if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. @@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod): return self._found_module(fullname, path, fp) +class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): + def _find_parent_spec(self, fullname): + #history = [] + debug = self.log.debug + children = [] + for parent_name, child_name in self._iter_parents(fullname): + children.insert(0, child_name) + if not parent_name: + debug('abandoning %r, reached top-level', fullname) + return None, children + + try: + parent = sys.modules[parent_name] + except KeyError: + debug('skipping %r, not in sys.modules', parent_name) + continue + + try: + spec = parent.__spec__ + except AttributeError: + debug('skipping %r: %r.__spec__ is absent', + parent_name, parent) + continue + + if not spec: + debug('skipping %r: %r.__spec__=%r', + parent_name, parent, spec) + continue + + if spec.name != parent_name: + debug('skipping %r: %r.__spec__.name=%r does not match', + parent_name, parent, spec.name) + continue + + if not spec.submodule_search_locations: + debug('skipping %r: %r.__spec__.submodule_search_locations=%r', + parent_name, parent, spec.submodule_search_locations) + continue + + return spec, children + + raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + def find(self, fullname): + # Returns absolute path, ParentImpEnumerationMethod returns relative + # >>> spec_pem.find('six_brokenpkg._six')[::2] + # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) + + if sys.version_info < (3, 4): + return None + + fullname = to_text(fullname) + spec, children = self._find_parent_spec(fullname) + for child_name in children: + if spec: + name = '%s.%s' % (spec.name, child_name) + submodule_search_locations = spec.submodule_search_locations + else: + name = child_name + submodule_search_locations = None + spec = importlib.util._find_spec(name, submodule_search_locations) + if spec is None: + self.log.debug('%r spec unavailable from %s', fullname, spec) + return None + + is_package = spec.submodule_search_locations is not None + if name != fullname: + if not is_package: + self.log.debug('%r appears to be child of non-package %r', + fullname, spec) + return None + continue + + if not spec.has_location: + self.log.debug('%r.origin cannot be read as a file', spec) + return None + + if os.path.splitext(spec.origin)[1] != '.py': + self.log.debug('%r.origin does not contain Python source code', + spec) + return None + + # FIXME This should use loader.get_source() + with open(spec.origin, 'rb') as f: + source = f.read() + + return spec.origin, source, is_package + + raise ValueError('%s.find(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + class ModuleFinder(object): """ Given the name of a loaded module, make a best-effort attempt at finding @@ -838,7 +949,8 @@ class ModuleFinder(object): DefectivePython3xMainMethod(), PkgutilMethod(), SysModulesMethod(), - ParentEnumerationMethod(), + ParentSpecEnumerationMethod(), + ParentImpEnumerationMethod(), ] def get_module_source(self, fullname): diff --git a/mitogen/parent.py b/mitogen/parent.py index 32aa3cb..29bcf66 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive connection. """ -import codecs +import binascii import errno import fcntl import getpass @@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux :linux:man2:`epoll` interface. + Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' @@ -1405,10 +1405,14 @@ class Connection(object): # file descriptor 0 as 100, creates a pipe, then execs a new interpreter # with a custom argv. # * Optimized for minimum byte count after minification & compression. + # The script preamble_size.py measures this. # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in # order to allow virtualenvs to detect their install prefix. + # + # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible + # module test_echo_module, used by the integration tests. # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version # switcher as /usr/bin/python, which introspects argv0. To workaround # it we redirect attempts to call /usr/bin/python with an explicit @@ -1417,7 +1421,8 @@ class Connection(object): # do something slightly different. The Python executable is patched to # perform an extra execvp(). I don't fully understand the details, but # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. - # * macOS 13.x (Darwin 22?) may remove python 2.x entirely. + # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python. + # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python # # Locals: # R: read side of interpreter stdin. @@ -1445,7 +1450,7 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) - C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') + C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN)) fp=os.fdopen(W,'wb',0) fp.write(C) fp.close() @@ -1477,16 +1482,16 @@ class Connection(object): source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) compressed = zlib.compress(source.encode(), 9) - encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) - # We can't use bytes.decode() in 3.x since it was restricted to always - # return unicode, so codecs.decode() is used instead. In 3.x - # codecs.decode() requires a bytes object. Since we must be compatible - # with 2.4 (no bytes literal), an extra .encode() either returns the - # same str (2.x) or an equivalent bytes (3.x). + encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) + + # Just enough to decode, decompress, and exec the first stage. + # Priorities: wider compatibility, faster startup, shorter length. + # `import os` here, instead of stage 1, to save a few bytes. + # `sys.path=...` for https://github.com/python/cpython/issues/115911. return self.get_python_argv() + [ '-c', - 'import codecs,os,sys;_=codecs.decode;' - 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) + 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' + 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), ] def get_econtext_config(self): diff --git a/mitogen/service.py b/mitogen/service.py index 0e5f641..7fde901 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -752,10 +752,12 @@ class PushFileService(Service): One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. - overridden_sources: optional dict containing source code to override path's source code - extra_sys_paths: loads additional sys paths for use in finding modules; beneficial - in situations like loading Ansible Collections because source code - dependencies come from different file paths than where the source lives + :param dict overridden_sources: + Optional dict containing source code to override path's source code + :param extra_sys_paths: + Loads additional sys paths for use in finding modules; beneficial + in situations like loading Ansible Collections because source code + dependencies come from different file paths than where the source lives """ for path in paths: overridden_source = None @@ -77,6 +77,8 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index ce4511f..c34dd21 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -48,5 +48,21 @@ host_pattern_mismatch = error task_output_limit = 10 [ssh_connection] -ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s +# https://www.openssh.com/legacy.html +# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. +# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). +# Duplicated cases in +# - tests/ansible/ansible.cfg +# - tests/ansible/integration/connection_delegation/delegate_to_template.yml +# - tests/ansible/integration/connection_delegation/stack_construction.yml +# - tests/ansible/integration/process/unix_socket_cleanup.yml +# - tests/ansible/integration/ssh/variables.yml +# - tests/testlib.py +ssh_args = + -o ControlMaster=auto + -o ControlPersist=60s + -o ForwardAgent=yes + -o HostKeyAlgorithms=+ssh-rsa + -o PubkeyAcceptedKeyTypes=+ssh-rsa + -o UserKnownHostsFile=/dev/null pipelining = True diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show new file mode 100755 index 0000000..42ef319 --- /dev/null +++ b/tests/ansible/files/cwd_show @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Show permissions and identities that impact the current working directory. +# On macOS libc cwd() can return EACCES after su or sudo. +# See also +# - https://github.com/ansible/ansible/pull/7078 +# - https://github.com/python/cpython/issues/115911 + +set -o errexit +set -o nounset +set -o pipefail + +whoami +groups +pwd + +d=$(pwd) +while [[ "$d" != "/" && -n "$d" ]]; do + ls -ld "$d" + d=$(dirname "$d") +done +ls -ld / diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index a18b90d..ac744ed 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,3 +1,5 @@ +# code: language=ini +# vim: syntax=dosini # become_same_user.yml bsu-joe ansible_user=joe diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index a22bd5d..4ae861b 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Connection delegation scenarios. It's impossible to connect to them, but their would-be diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index d40c3dd..1bec001 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,9 +1,12 @@ +# code: language=ini # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" +# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 +# and os.environ['USER'] is not populated on Azure macOS runners. +target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts index 34e1ff9..b210164 100644 --- a/tests/ansible/hosts/k3.hosts +++ b/tests/ansible/hosts/k3.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Used for manual testing. diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts index 41af412..e42221e 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # issue #511, #536: we must not define an explicit localhost, as some diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 7d7b526..1c1c2e1 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,3 +1,6 @@ +# code: language=ini +# vim: syntax=dosini + # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. @@ -17,11 +20,12 @@ tc_remote_user tc_transport [transport_config_undiscover:vars] -# If python interpreter path is unset, Ansible tries to connect & discover it. -# That causes approx 10 seconds timeout per task - there's no host to connect to. +# If ansible_*_interpreter isn't set Ansible tries to connect & discover it. +# If that target doesn't exist we must wait $timeout seconds for each attempt. +# Setting a known (invalid) interpreter skips discovery & the many timeouts. # This optimisation should not be relied in any test. # Note: tc-python-path-* are intentionally not included. -ansible_python_interpreter = python3000 # Not expected to exist +ansible_python_interpreter = python3000 [tc_transport] tc-transport-unset diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index bd6a0ae..52d420d 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -1,5 +1,4 @@ # Verify passwordful su behaviour - # Ansible can't handle this on OS X. I don't care why. - name: integration/become/su_password.yml hosts: test-targets @@ -44,20 +43,54 @@ fail_msg: out={{out}} when: is_mitogen - - name: Ensure password su succeeds. + - name: Ensure password su with chdir succeeds shell: whoami + args: + chdir: ~mitogen__user1 become: true become_user: mitogen__user1 register: out vars: ansible_become_pass: user1_password - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - name: Ensure password su without chdir succeeds + shell: whoami + become: true + become_user: mitogen__user1 + register: out + vars: + ansible_become_pass: user1_password + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - assert: + that: + - out.stdout == 'mitogen__user1' + fail_msg: out={{out}} + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + tags: - su - su_password diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 36de165..7d33a16 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -41,17 +41,15 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -71,17 +69,15 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 380b319..279a7b2 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -81,14 +81,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -126,14 +124,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -182,14 +178,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -227,14 +221,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -257,14 +249,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -313,14 +303,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -359,14 +347,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index ea828aa..6ac9bad 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -37,6 +37,7 @@ vars: ansible_python_interpreter: auto test_echo_module: + facts_copy: "{{ ansible_facts }}" register: echoout # can't test this assertion: @@ -44,11 +45,24 @@ # because Mitogen's ansible_python_interpreter is a connection-layer configurable that # "must be extracted during each task execution to form the complete connection-layer configuration". # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered - - assert: + - name: assert discovered python matches invoked python + assert: that: - auto_out.ansible_facts.discovered_interpreter_python is defined - - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python - fail_msg: auto_out={{auto_out}} echoout={{echoout}} + - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen + - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + fail_msg: + - "auto_out: {{ auto_out }}" + - "echoout: {{ echoout }}" + when: + # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always + # reports the same path. I can't reach via symlinks. + # >>> sys.executable + # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python + - is_mitogen + or echoout.running_python.sys.version_info.major != 2 + or not (echoout.running_python.sys.platform == "darwin" + and echoout.running_python.platform.release.major == 20) - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result @@ -128,7 +142,8 @@ - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter block: - test_echo_module: - facts: + facts_copy: "{{ ansible_facts }}" + facts_to_override: ansible_discovered_interpreter_bogus: from module discovered_interpreter_bogus: from_module ansible_bogus_interpreter: from_module @@ -189,13 +204,6 @@ - distro == 'ubuntu' - distro_version is version('16.04', '>=', strict=True) - - name: mac assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: os_family == 'Darwin' - always: - meta: clear_facts when: diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index 38d1012..6ffff5f 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -4,6 +4,10 @@ - name: integration/interpreter_discovery/complex_args.yml hosts: test-targets gather_facts: true + environment: + http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}" + https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" + no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: - name: create temp file to source file: @@ -21,28 +25,24 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - special_python: source /tmp/fake || true && python + # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). + special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" - name: run get_url with specially-sourced python get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: "{{ special_python }}" - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - name: run get_url with specially-sourced python including jinja get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: > {% if "1" == "1" %} @@ -50,8 +50,5 @@ {% else %} python {% endif %} - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" tags: - complex_args diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index 625ad7b..b5d40b7 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -9,7 +9,7 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -c local -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 0c620da..0d7cf1b 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -2,6 +2,11 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. + - meta: end_play + when: not is_mitogen + - custom_python_new_style_missing_interpreter: foo: true with_sequence: start=0 end={{end|default(1)}} diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index e2384f8..8435b15 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,7 +1,8 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: - # without Mitogen Ansible 2.10 hangs on this play + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml index f2a3eef..ebe34cc 100644 --- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml +++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml @@ -1,3 +1,7 @@ +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, if any - reload() is a Python 2.x builtin function. # issue #555 - name: integration/runner/custom_python_prehistoric_module.yml @@ -5,9 +9,11 @@ tasks: - custom_python_prehistoric_module: register: out + when: is_mitogen - assert: that: out.ok fail_msg: out={{out}} + when: is_mitogen tags: - custom_python_prehistoric_module diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 8e912b2..6041d30 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -17,7 +17,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -34,7 +34,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -59,7 +59,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -76,7 +76,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -101,7 +101,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -118,7 +118,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -148,7 +148,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -165,7 +165,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py index 5923f4b..72264cf 100644 --- a/tests/ansible/lib/action/assert_equal.py +++ b/tests/ansible/lib/action/assert_equal.py @@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase TEMPLATE_KWARGS = {} -_argspec = inspect.getargspec(ansible.template.Templar.template) +try: + # inspect.getfullargspec() Added: 3.0 + _argspec = inspect.getfullargspec(ansible.template.Templar.template) +except AttributeError: + # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11 + _argspec = inspect.getargspec(ansible.template.Templar.template) if 'bare_deprecated' in _argspec.args: TEMPLATE_KWARGS['bare_deprecated'] = False diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index c7a222e..d2ceaf0 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I return details about the Python # interpreter I run within. @@ -25,6 +25,11 @@ except NameError: def main(): module = AnsibleModule(argument_spec={}) module.exit_json( + fs={ + '/tmp': { + 'resolved': os.path.realpath('/tmp'), + }, + }, python={ 'version': { 'full': '%i.%i.%i' % sys.version_info[:3], diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py index ae1b78c..507e53d 100644 --- a/tests/ansible/lib/modules/custom_python_external_module.py +++ b/tests/ansible/lib/modules/custom_python_external_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I expect the quote from modules2/module_utils/joker.py. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py index be9acb2..95bd0c7 100644 --- a/tests/ansible/lib/modules/custom_python_external_pkg.py +++ b/tests/ansible/lib/modules/custom_python_external_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.externalpkg import extmod diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 6164057..846037e 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>""" diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py index 255e372..1d34232 100644 --- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py +++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I leak state from each invocation # into a class variable and a global variable. diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 51b7452..9767f85 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I modify the process environment and # don't clean up after myself. diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 2e0ef0d..728685f 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -1,6 +1,20 @@ # I am an Ansible new-style Python module, but I lack an interpreter. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980 # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10 @@ -8,11 +22,46 @@ import sys # from ansible.module_utils. # import ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 1e7270c..c84d241 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,16 +1,65 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I should receive an encoding string. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # This is the magic marker Ansible looks for: # from ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index c5e264a..d465ac9 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 0cf9774..6139748 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/python + +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, reload() is a Python 2.x builtin function. # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index d6a839a..4a6243d 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I run the script provided in the # parameter. diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 03f3b6a..6b3a356 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # issue #590: I am an Ansible new-style Python module that tries to use # ansible.module_utils.distro. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index f5e3386..23eeeb5 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 1b80a48..289e966 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py new file mode 100644 index 0000000..41cf1c1 --- /dev/null +++ b/tests/ansible/lib/modules/module_finder_test.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os +import sys + +import ansible.module_utils.external1 + +from ansible.module_utils.externalpkg.extmod import path as epem_path + +def main(): + pass diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 1f71e87..d6a5fb9 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # -*- coding: utf-8 -*- # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> @@ -9,28 +9,61 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +import os import platform import sys from ansible.module_utils.basic import AnsibleModule def main(): - result = dict(changed=False) - module = AnsibleModule(argument_spec=dict( - facts=dict(type=dict, default={}) + facts_copy=dict(type=dict, default={}), + facts_to_override=dict(type=dict, default={}) )) - result['ansible_facts'] = module.params['facts'] # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info - if sys.platform == 'darwin' and sys.executable != '/usr/bin/python': - if int(platform.release()[:2]) < 19: - sys.executable = sys.executable[:-3] - else: + # NB This must be synced with mitogen.parent.Connection.get_boot_command() + platform_release_major = int(platform.release().partition('.')[0]) + if sys.modules.get('mitogen') and sys.platform == 'darwin': + if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7': + sys.executable = '/usr/bin/python' + if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7): # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python sys.executable = "/usr/bin/python" - result['running_python_interpreter'] = sys.executable + + facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + result = { + 'changed': False, + 'ansible_facts': module.params['facts_to_override'], + 'discovered_and_running_samefile': os.path.samefile( + os.path.realpath(discovered_interpreter_python), + os.path.realpath(sys.executable), + ), + 'discovered_python': { + 'as_seen': discovered_interpreter_python, + 'resolved': os.path.realpath(discovered_interpreter_python), + }, + 'running_python': { + 'platform': { + 'release': { + 'major': platform_release_major, + }, + }, + 'sys': { + 'executable': { + 'as_seen': sys.executable, + 'resolved': os.path.realpath(sys.executable), + }, + 'platform': sys.platform, + 'version_info': { + 'major': sys.version_info[0], + 'minor': sys.version_info[1], + }, + }, + }, + } module.exit_json(**result) diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index f4c47ab..610eaf3 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -23,9 +23,16 @@ when: - lout.python.version.full is version('2.7', '>=', strict=True) - - assert: + - name: Check virtualenv was used + # On macOS runners a symlink /tmp -> /private/tmp has been seen + vars: + requested_executable: /tmp/issue_152_virtualenv/bin/python + expected_executables: + - "{{ requested_executable }}" + - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" + assert: that: - - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" + - out.sys_executable in expected_executables fail_msg: out={{out}} when: - lout.python.version.full is version('2.7', '>=', strict=True) diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml index 2ca6b97..8903494 100644 --- a/tests/ansible/setup/all.yml +++ b/tests/ansible/setup/all.yml @@ -1 +1,2 @@ -- import_playbook: report.yml +- import_playbook: report_controller.yml +- import_playbook: report_targets.yml diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml new file mode 100644 index 0000000..d0d5cc1 --- /dev/null +++ b/tests/ansible/setup/report_controller.yml @@ -0,0 +1,17 @@ +- name: Report controller parameters + hosts: localhost + gather_facts: false + tasks: + - debug: + msg: + - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('<unset>') }}" + - ${USER}: "{{ lookup('env', 'USER') | default('<unset>') }}" + - $(groups): "{{ lookup('pipe', 'groups') }}" + - $(pwd): "{{ lookup('pipe', 'pwd') }}" + - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_run_tags: "{{ ansible_run_tags | default('<unset>') }}" + - ansible_playbook_python: "{{ ansible_playbook_python | default('<unset>') }}" + - ansible_skip_tags: "{{ ansible_skip_tags | default('<unset>') }}" + - ansible_version.full: "{{ ansible_version.full | default('<unset>') }}" + - is_mitogen: "{{ is_mitogen | default('<unset>') }}" + - playbook_dir: "{{ playbook_dir | default('<unset>') }}" diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report_targets.yml index 450e4fb..5aa6712 100644 --- a/tests/ansible/setup/report.yml +++ b/tests/ansible/setup/report_targets.yml @@ -1,4 +1,4 @@ -- name: Report runtime settings +- name: Report target facts hosts: localhost:test-targets gather_facts: true tasks: @@ -13,8 +13,3 @@ - debug: {var: ansible_facts.osversion} - debug: {var: ansible_facts.python} - debug: {var: ansible_facts.system} - - debug: {var: ansible_forks} - - debug: {var: ansible_run_tags} - - debug: {var: ansible_skip_tags} - - debug: {var: ansible_version.full} - - debug: {var: is_mitogen} diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py new file mode 100644 index 0000000..79e8fdb --- /dev/null +++ b/tests/ansible/tests/module_finder_test.py @@ -0,0 +1,80 @@ +import os.path +import sys +import textwrap +import unittest + +import ansible_mitogen.module_finder + +import testlib + + +class ScanFromListTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b.c; from d.e import f; from g import h, i + ''') + code = compile(source, '<str>', 'exec') + self.assertEqual( + list(ansible_mitogen.module_finder.scan_fromlist(code)), + [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], + ) + + +class WalkImportsTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b; import b.c; from b.d import e, f + ''') + code = compile(source, '<str>', 'exec') + + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code)), + ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], + ) + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), + ['b.c', 'b.d', 'b.d.e', 'b.d.f'], + ) + + +class ScanTest(testlib.TestCase): + module_name = 'ansible_module_module_finder_test__this_should_not_matter' + module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') + search_path = ( + 'does_not_exist/module_utils', + testlib.ANSIBLE_MODULE_UTILS_DIR, + ) + + @staticmethod + def relpath(path): + return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) + + @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') + def test_importlib_find_spec(self): + scan = ansible_mitogen.module_finder._scan_importlib_find_spec + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) + + @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') + def test_imp_find_module(self): + scan = ansible_mitogen.module_finder._scan_imp_find_module + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json new file mode 100644 index 0000000..4bf0bd3 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py2x.json @@ -0,0 +1,186 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.utils", + "django.utils._os", + "django.utils.deprecation", + "django.utils.encoding", + "django.utils.functional", + "django.utils.inspect", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.six", + "django.utils.version" + ], + "django.db.models": [ + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.checks", + "django.core.checks.caches", + "django.core.checks.compatibility", + "django.core.checks.compatibility.django_1_10", + "django.core.checks.compatibility.django_1_8_0", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.urls", + "django.core.checks.utils", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.base", + "django.db.models.functions.datetime", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.decorators", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.html", + "django.utils.html_parser", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.safestring", + "django.utils.six", + "django.utils.text", + "django.utils.timezone", + "django.utils.translation", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json new file mode 100644 index 0000000..06e94e6 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-legacy.json @@ -0,0 +1,209 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json new file mode 100644 index 0000000..dcbcc78 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -0,0 +1,208 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" +] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} +
\ No newline at end of file diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 6224b61..0b6d5e6 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -4,6 +4,8 @@ # WARNING: this creates non-privilged accounts with pre-set passwords! # +- import_playbook: ../ansible/setup/report_controller.yml + - hosts: all gather_facts: true strategy: mitogen_free @@ -37,12 +39,12 @@ normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + }}" all_users: "{{ special_users + normal_users - }}" + }}" tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -100,6 +102,7 @@ with_items: "{{all_users}}" copy: dest: /var/lib/AccountsService/users/mitogen__{{item}} + mode: u=rw,go= content: | [User] SystemAccount=true @@ -108,7 +111,7 @@ when: ansible_system == 'Linux' and out.stat.exists service: name: accounts-daemon - restarted: true + state: restarted - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" @@ -117,6 +120,9 @@ copy: dest: ~mitogen__slow_user/.{{item}} src: ../data/docker/mitogen__slow_user.profile + owner: mitogen__slow_user + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -125,6 +131,9 @@ copy: dest: ~mitogen__permdenied/.{{item}} src: ../data/docker/mitogen__permdenied.profile + owner: mitogen__permdenied + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -136,20 +145,13 @@ state: directory mode: go= owner: mitogen__has_sudo_pubkey + group: mitogen__group - copy: dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys src: ../data/docker/mitogen__has_sudo_pubkey.key.pub mode: go= owner: mitogen__has_sudo_pubkey - - - name: Install slow profile for one account - block: - - copy: - dest: ~mitogen__slow_user/.profile - src: ../data/docker/mitogen__slow_user.profile - - copy: - dest: ~mitogen__slow_user/.bashrc - src: ../data/docker/mitogen__slow_user.profile + group: mitogen__group - name: Require a TTY for two accounts lineinfile: diff --git a/tests/importer_test.py b/tests/importer_test.py index e48c02a..e86af8a 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -2,6 +2,7 @@ import sys import threading import types import zlib +import unittest import mock @@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin): super(ImporterMixin, self).tearDown() +class InvalidNameTest(ImporterMixin, testlib.TestCase): + modname = 'trailingdot.' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_invalid(self): + self.set_get_module_response(self.response) + self.assertEqual(self.importer.find_spec(self.modname, path=None), None) + + +class MissingModuleTest(ImporterMixin, testlib.TestCase): + modname = 'missing' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') + def test_load_module_missing(self): + self.set_get_module_response(self.response) + self.assertRaises(ImportError, self.importer.load_module, self.modname) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_missing(self): + """ + Importer should optimistically offer itself as a module loader + when there are no disqualifying criteria. + """ + import importlib.machinery + self.set_get_module_response(self.response) + spec = self.importer.find_spec(self.modname, path=None) + self.assertIsInstance(spec, importlib.machinery.ModuleSpec) + self.assertEqual(spec.name, self.modname) + self.assertEqual(spec.loader, self.importer) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_create_module_missing(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + self.assertRaises(ImportError, self.importer.create_module, spec) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModuleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): # 0:fullname 1:pkg_present 2:path 3:compressed 4:related response = (modname, None, path, data, []) - def test_no_such_module(self): - self.set_get_module_response( - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - (self.modname, None, None, None, None) - ) - self.assertRaises(ImportError, - lambda: self.importer.load_module(self.modname)) - def test_module_added_to_sys_modules(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) @@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): self.assertIsNone(mod.__package__) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ModuleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'fake_module' + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.__package__, 'mypkg') +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'mypkg.fake_module' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'mypkg.fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModulePackageTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' @@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.func.__module__, self.modname) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class PackageSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("func = lambda: 1\n\n")) + path = 'fake_pkg/__init__.py' + modname = 'fake_pkg' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, [], path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_pkg') + #self.assertFalse(hasattr(mod, '__file__')) + + def test_get_filename(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + filename = self.importer.get_filename(self.modname) + self.assertEqual('master:fake_pkg/__init__.py', filename) + + def test_get_source(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + source = self.importer.get_source(self.modname) + self.assertEqual(source, + mitogen.core.to_text(zlib.decompress(self.data))) + + class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): def initdir(self, caplog): self.caplog = caplog diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index ff18bbc..67e937e 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,4 +1,5 @@ import inspect +import json import os import sys import unittest @@ -139,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase): self.assertIsNone(tup) -class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ParentEnumerationMethod - +class ParentEnumerationMixin(object): def call(self, fullname): return self.klass().find(fullname) @@ -231,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): self.assertEqual(is_pkg, False) +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') +class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentImpEnumerationMethod + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentSpecEnumerationMethod + + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder @@ -296,199 +305,65 @@ class FindRelatedTest(testlib.TestCase): self.assertEqual(set(related), self.SIMPLE_EXPECT) -if sys.version_info > (2, 6): - class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') - - # TODO: rip out Django and replace with a static tree of weird imports - # that don't depend on .. Django! The hack below is because the version - # of Django we need to test against 2.6 doesn't actually run on 3.6. - # But we don't care, we just need to be able to import it. - # - # File "django/utils/html_parser.py", line 12, in <module> - # AttributeError: module 'html.parser' has no attribute - # 'HTMLParseError' - # - from django.utils.six.moves import html_parser as _html_parser - _html_parser.HTMLParseError = Exception - - @classmethod - def setUpClass(cls): - super(DjangoMixin, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoMixin, cls).tearDownClass() - - - class FindRelatedImportsTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related_imports(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - 'django', - 'django.core', - 'django.core.signals', - 'django.db.utils', - 'django.utils.functional', - ]) - - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.maxDiff=None - self.assertEqual(related, [ - u'django', - u'django.core.exceptions', - u'django.db', - u'django.db.models', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.query', - u'django.db.models.signals', - ]) - - - class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - maxDiff = None - - def call(self, fullname): - return self.klass().find_related(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.signals', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.utils', - u'django.utils._os', - u'django.utils.encoding', - u'django.utils.functional', - u'django.utils.importlib', - u'django.utils.module_loading', - u'django.utils.six', - ]) - - @unittest.skipIf( - condition=(sys.version_info >= (3, 0)), - reason='broken due to ancient vendored six.py' - ) - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.files', - u'django.core.files.base', - u'django.core.files.images', - u'django.core.files.locks', - u'django.core.files.move', - u'django.core.files.storage', - u'django.core.files.utils', - u'django.core.signals', - u'django.core.validators', - u'django.db', - u'django.db.backends', - u'django.db.backends.signals', - u'django.db.backends.util', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.constants', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.proxy', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.options', - u'django.db.models.query', - u'django.db.models.query_utils', - u'django.db.models.related', - u'django.db.models.signals', - u'django.db.models.sql', - u'django.db.models.sql.aggregates', - u'django.db.models.sql.constants', - u'django.db.models.sql.datastructures', - u'django.db.models.sql.expressions', - u'django.db.models.sql.query', - u'django.db.models.sql.subqueries', - u'django.db.models.sql.where', - u'django.db.transaction', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.forms', - u'django.forms.fields', - u'django.forms.forms', - u'django.forms.formsets', - u'django.forms.models', - u'django.forms.util', - u'django.forms.widgets', - u'django.utils', - u'django.utils._os', - u'django.utils.crypto', - u'django.utils.datastructures', - u'django.utils.dateformat', - u'django.utils.dateparse', - u'django.utils.dates', - u'django.utils.datetime_safe', - u'django.utils.decorators', - u'django.utils.deprecation', - u'django.utils.encoding', - u'django.utils.formats', - u'django.utils.functional', - u'django.utils.html', - u'django.utils.html_parser', - u'django.utils.importlib', - u'django.utils.ipv6', - u'django.utils.itercompat', - u'django.utils.module_loading', - u'django.utils.numberformat', - u'django.utils.safestring', - u'django.utils.six', - u'django.utils.text', - u'django.utils.timezone', - u'django.utils.translation', - u'django.utils.tree', - u'django.utils.tzinfo', - u'pytz', - u'pytz.exceptions', - u'pytz.lazy', - u'pytz.tzfile', - u'pytz.tzinfo', - ]) +class DjangoMixin(object): + WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') + + @classmethod + def modules_expected_path(cls): + if sys.version_info[0:2] < (3, 0): + modules_expected_filename = 'modules_expected_py2x.json' + elif sys.version_info[0:2] <= (3, 6): + modules_expected_filename = 'modules_expected_py3x-legacy.json' + elif sys.version_info[0:2] >= (3, 10): + modules_expected_filename = 'modules_expected_py3x-new.json' + return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) + + @classmethod + def setUpClass(cls): + super(DjangoMixin, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + with open(cls.modules_expected_path(), 'rb') as f: + cls.MODULES_EXPECTED = json.load(f) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoMixin, cls).tearDownClass() + + +class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db') + expected = self.MODULES_EXPECTED['find_related']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db.models') + expected = self.MODULES_EXPECTED['find_related']['django.db.models'] + self.assertEqual(related, expected) + + +class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db.models') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] + self.assertEqual(related, expected) diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt new file mode 100644 index 0000000..bc7f7c2 --- /dev/null +++ b/tests/requirements-tox.txt @@ -0,0 +1,4 @@ +tox==3.28; python_version == '2.7' +tox==3.28; python_version == '3.6' +tox==4.8.0; python_version == '3.7' +tox>=4.13.0,~=4.0; python_version >= '3.8' diff --git a/tests/requirements.txt b/tests/requirements.txt index 7301bee..6d87d17 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,14 +1,25 @@ -psutil==5.4.8 -coverage==5.5; python_version < '3.7' -coverage==6.4.4; python_version >= '3.7' -Django==1.6.11 # Last version supporting 2.6. -mock==2.0.0 -pytz==2018.5 -cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect -pycparser==2.18 # Last version supporting 2.6. -pytest-catchlog==1.2.2 -pytest==3.1.2 -subprocess32==3.5.4 +cffi==1.15.1; python_version < '3.8' +cffi==1.16; python_version >= '3.8' + +coverage==5.5; python_version == '2.7' +coverage==6.2; python_version == '3.6' +coverage==7.2.7; python_version == '3.7' +coverage==7.4.3; python_version >= '3.8' + +Django==1.11.29; python_version < '3.0' +Django==3.2.20; python_version >= '3.6' + +mock==3.0.5; python_version == '2.7' +mock==5.1.0; python_version >= '3.6' + +psutil==5.9.8 + +pytest==4.6.11; python_version == '2.7' +pytest==7.0.1; python_version == '3.6' +pytest==7.4.4; python_version == '3.7' +pytest==8.0.2; python_version >= '3.8' + +subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings @@ -16,4 +27,7 @@ urllib3[secure]==1.23; python_version < '2.7' urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' # Last idna compatible with Python 2.6 was idna 2.7. idna==2.7; python_version < '2.7' -virtualenv==20.10.0 + +virtualenv==20.15.1; python_version == '2.7' +virtualenv==20.17.1; python_version == '3.6' +virtualenv==20.25.1; python_version >= '3.7' diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 429fdb5..3149fcb 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -134,12 +134,13 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_enforce_unknown_host_key(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: e = self.assertRaises(mitogen.ssh.HostKeyError, lambda: self.docker_ssh( username='mitogen__has_sudo_pubkey', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) ) @@ -149,11 +150,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_accept_enforce_host_keys(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='accept', ) context.shutdown(wait=True) @@ -166,7 +168,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) context.shutdown(wait=True) diff --git a/tests/testlib.py b/tests/testlib.py index 83629c3..8c40e7f 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -18,7 +18,10 @@ except ImportError: import ConfigParser as configparser import psutil -import subprocess32 as subprocess +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess import mitogen.core import mitogen.fork @@ -47,8 +50,13 @@ except NameError: LOG = logging.getLogger(__name__) -DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -MODS_DIR = os.path.join(DATA_DIR, 'importer') + +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') +ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') +ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') +DATA_DIR = os.path.join(TESTS_DIR, 'data') +MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') sys.path.append(DATA_DIR) sys.path.append(MODS_DIR) @@ -87,7 +95,7 @@ def base_executable(executable=None): base_executable = None if base_executable and base_executable != executable: - return 'be', base_executable + return base_executable # Python 2.x only has sys.base_prefix if running outside a virtualenv. try: @@ -499,19 +507,18 @@ def get_docker_host(): class DockerizedSshDaemon(object): - def _get_container_port(self): - s = subprocess.check_output(['docker', 'port', self.container_name]) - for line in s.decode().splitlines(): - m = self.PORT_RE.match(line) - if not m: - continue - dport, proto, _, bport = m.groups() - if dport == '22' and proto == 'tcp': - self.port = int(bport) + PORT_RE = re.compile( + # e.g. 0.0.0.0:32771, :::32771, [::]:32771' + r'(?P<addr>[0-9.]+|::|\[[a-f0-9:.]+\]):(?P<port>[0-9]+)', + ) - self.host = self.get_host() - if self.port is None: + @classmethod + def get_port(cls, container): + s = subprocess.check_output(['docker', 'port', container, '22/tcp']) + m = cls.PORT_RE.search(s.decode()) + if not m: raise ValueError('could not find SSH port in: %r' % (s,)) + return int(m.group('port')) def start_container(self): try: @@ -530,7 +537,6 @@ class DockerizedSshDaemon(object): self.image, ] subprocess.check_output(args) - self._get_container_port() def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): if '-' in mitogen_test_distro: @@ -545,12 +551,9 @@ class DockerizedSshDaemon(object): self.python_path = '/usr/bin/python' self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - - # 22/tcp -> 0.0.0.0:32771 - self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') - self.port = None - self.start_container() + self.host = self.get_host() + self.port = self.get_port(self.container_name) def get_host(self): return get_docker_host() @@ -631,12 +634,33 @@ class DockerMixin(RouterMixin): cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() + @property + def docker_ssh_default_kwargs(self): + return { + 'hostname': self.dockerized_ssh.host, + 'port': self.dockerized_ssh.port, + 'check_host_keys': 'ignore', + 'ssh_debug_level': 3, + # https://www.openssh.com/legacy.html + # ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. + # Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). + # Duplicated cases in + # - tests/ansible/ansible.cfg + # - tests/ansible/integration/connection_delegation/delegate_to_template.yml + # - tests/ansible/integration/connection_delegation/stack_construction.yml + # - tests/ansible/integration/process/unix_socket_cleanup.yml + # - tests/ansible/integration/ssh/variables.yml + # - tests/testlib.py + 'ssh_args': [ + '-o', 'HostKeyAlgorithms +ssh-rsa', + '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa', + ], + 'python_path': self.dockerized_ssh.python_path, + } + def docker_ssh(self, **kwargs): - kwargs.setdefault('hostname', self.dockerized_ssh.host) - kwargs.setdefault('port', self.dockerized_ssh.port) - kwargs.setdefault('check_host_keys', 'ignore') - kwargs.setdefault('ssh_debug_level', 3) - kwargs.setdefault('python_path', self.dockerized_ssh.python_path) + for k, v in self.docker_ssh_default_kwargs.items(): + kwargs.setdefault(k, v) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): @@ -1,23 +1,39 @@ -# This file is a local convenience. It is not a substitute for the full CI -# suite, and does not cover the full range of Python versions for Mitogen. - -# I use this on Ubuntu 20.04, with the following additions +# This configuration drives both CI and local development. +# I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox -# Last version to support each python version +# Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv +# ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== +# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 +# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 +# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 3.5 <= 2.11 <= 2.13 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² +# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 +# 3.8 <= 2.12 +# 3.9 <= 2.15 +# 3.10 +# 3.11 +# 3.12 >= 2.13¹ +# +# Notes +# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13). +# Python 3.12 removed support for find_module(), replaced by find_spec(). +# In Ansible <= 4.x ansible.module_utils.six lacks find_spec(). +# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3 +# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder +# lacks find_spec(). https://github.com/ansible/ansible/pull/76225 +# +# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). +# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. +# https://github.com/ansible/ansible/pull/80751 # -# Python tox virt'env pip A cntllr A target coverage -# ========== ======== ======== ======== ======== ======== ======== -# python2.4 1.4 1.8 1.1 2.3? -# python2.5 1.6.1 1.9.1 1.3.1 ??? -# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 2.13 4.5.4 -# python2.7 20.3 2.11 -# python3.5 2.11 -# python3.6 2.11 -# python3.7 2.11 +# 2. Higher virtualenv versions cannot run under this Python version. They can +# still generate virtual environments for it. # Ansible Dependency # ================== ====================== @@ -27,6 +43,9 @@ # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 # ansible == 6.* ansible-core ~= 2.13.0 +# ansible == 7.x ansible-core ~= 2.14.0 +# ansible == 8.x ansible-core ~= 2.15.0 +# ansible == 9.x ansible-core ~= 2.16.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -37,10 +56,11 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{310}-mode_ansible-ansible{2.10,3,4,5,6}, - py{27,36,310}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,310}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{311}-mode_ansible-ansible{2.10,3,4,5}, + py{312}-mode_ansible-ansible{6}, + py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -53,14 +73,16 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 + py312: python3.12 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt ansible2.10: ansible==2.10.7 ansible3: ansible==3.4.0 ansible4: ansible==4.10.0 - ansible5: ansible==5.8.0 - ansible6: ansible==6.0.0 + ansible5: ansible~=5.0 + ansible6: ansible~=6.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -81,7 +103,6 @@ passenv = HOME setenv = # See also azure-pipelines.yml - ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 @@ -113,10 +134,26 @@ setenv = distros_ubuntu1804: DISTROS=ubuntu1804 distros_ubuntu2004: DISTROS=ubuntu2004 mode_ansible: MODE=ansible + mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_debops_common: MODE=debops_common + mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen strategy_linear: ANSIBLE_STRATEGY=linear +allowlist_externals = + # Added: Tox 3.18: Tox 4.0+ + *_install.py + *_tests.py + aws + docker + docker-credential-secretservice + echo + gpg2 + pass whitelist_externals = + # Deprecated: Tox 3.18+; Removed: Tox 4.0 + *_install.py + *_tests.py + aws docker docker-credential-secretservice echo @@ -141,7 +178,9 @@ whitelist_externals = echo [testenv:docs] -basepython = python3 +basepython = python3.8 changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +deps = + -r docs/requirements.txt |
