summaryrefslogtreecommitdiff
diff options
authorStefano Rivera <stefanor@debian.org>2024-03-28 13:45:03 -0400
committergit-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com>2024-03-28 22:39:46 +0000
commitb253dacf777dbda760697b52dfaa05c161846f5b (patch)
treecccd248b8187345e930a83cbdc4979b120c1a001
parent2fe1285c5d72d4070f77774cb69cd50bbf8006ae (diff)
0.3.5-1 (patches unapplied)import/0.3.5-1
Imported using git-ubuntu import.
Notes
Notes: * New upstream release. * Refresh patches. * Drop patches, superseded upstream: no-subprocess32, python-3.11, python-3.12.
-rw-r--r--.ci/azure-pipelines-steps.yml74
-rw-r--r--.ci/azure-pipelines.yml108
-rw-r--r--.ci/ci_lib.py5
-rwxr-xr-x.ci/localhost_ansible_tests.py36
-rw-r--r--.gitignore1
-rw-r--r--ansible_mitogen/module_finder.py127
-rw-r--r--ansible_mitogen/runner.py91
-rw-r--r--ansible_mitogen/utils.py21
-rw-r--r--debian/changelog9
-rw-r--r--debian/patches/ansible-2.166
-rw-r--r--debian/patches/ask-become-pass36
-rw-r--r--debian/patches/no-subprocess3226
-rw-r--r--debian/patches/python-3.11246
-rw-r--r--debian/patches/python-3.122358
-rw-r--r--debian/patches/python32
-rw-r--r--debian/patches/series3
-rw-r--r--debian/patches/skip-django-tests136
-rw-r--r--docs/_templates/github.html2
-rw-r--r--docs/ansible_detailed.rst15
-rw-r--r--docs/changelog.rst24
-rw-r--r--docs/conf.py30
-rw-r--r--docs/contributors.rst1
-rw-r--r--docs/index.rst2
-rw-r--r--docs/requirements.txt3
-rw-r--r--mitogen/__init__.py2
-rw-r--r--mitogen/core.py192
-rw-r--r--mitogen/master.py122
-rw-r--r--mitogen/parent.py29
-rw-r--r--mitogen/service.py10
-rw-r--r--setup.py2
-rw-r--r--tests/ansible/ansible.cfg18
-rwxr-xr-xtests/ansible/files/cwd_show22
-rw-r--r--tests/ansible/hosts/become_same_user.hosts2
-rw-r--r--tests/ansible/hosts/connection_delegation.hosts1
-rw-r--r--tests/ansible/hosts/default.hosts5
-rw-r--r--tests/ansible/hosts/k3.hosts1
-rw-r--r--tests/ansible/hosts/localhost.hosts1
-rw-r--r--tests/ansible/hosts/transport_config.hosts10
-rw-r--r--tests/ansible/integration/become/su_password.yml41
-rw-r--r--tests/ansible/integration/connection_delegation/delegate_to_template.yml32
-rw-r--r--tests/ansible/integration/connection_delegation/stack_construction.yml98
-rw-r--r--tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml30
-rw-r--r--tests/ansible/integration/interpreter_discovery/complex_args.yml23
-rw-r--r--tests/ansible/integration/process/unix_socket_cleanup.yml2
-rw-r--r--tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml5
-rw-r--r--tests/ansible/integration/runner/custom_python_new_style_module.yml3
-rw-r--r--tests/ansible/integration/runner/custom_python_prehistoric_module.yml6
-rw-r--r--tests/ansible/integration/ssh/variables.yml16
-rw-r--r--tests/ansible/lib/action/assert_equal.py7
-rw-r--r--tests/ansible/lib/modules/custom_python_detect_environment.py7
-rw-r--r--tests/ansible/lib/modules/custom_python_external_module.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_external_pkg.py2
-rwxr-xr-xtests/ansible/lib/modules/custom_python_json_args_module.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_leaky_class_vars.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_modify_environ.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py49
-rwxr-xr-xtests/ansible/lib/modules/custom_python_new_style_module.py51
-rw-r--r--tests/ansible/lib/modules/custom_python_os_getcwd.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_prehistoric_module.py7
-rw-r--r--tests/ansible/lib/modules/custom_python_run_script.py2
-rw-r--r--tests/ansible/lib/modules/custom_python_uses_distro.py2
-rwxr-xr-xtests/ansible/lib/modules/custom_python_want_json_module.py2
-rw-r--r--tests/ansible/lib/modules/mitogen_test_gethostbyname.py2
-rw-r--r--tests/ansible/lib/modules/module_finder_test.py12
-rw-r--r--tests/ansible/lib/modules/test_echo_module.py53
-rw-r--r--tests/ansible/regression/issue_152__virtualenv_python_fails.yml11
-rw-r--r--tests/ansible/setup/all.yml3
-rw-r--r--tests/ansible/setup/report_controller.yml17
-rw-r--r--tests/ansible/setup/report_targets.yml (renamed from tests/ansible/setup/report.yml)7
-rw-r--r--tests/ansible/tests/module_finder_test.py80
-rw-r--r--tests/data/importer/webproject/modules_expected_py2x.json186
-rw-r--r--tests/data/importer/webproject/modules_expected_py3x-legacy.json209
-rw-r--r--tests/data/importer/webproject/modules_expected_py3x-new.json208
-rw-r--r--tests/image_prep/_user_accounts.yml26
-rw-r--r--tests/importer_test.py126
-rw-r--r--tests/module_finder_test.py273
-rw-r--r--tests/requirements-tox.txt4
-rw-r--r--tests/requirements.txt38
-rw-r--r--tests/ssh_test.py8
-rw-r--r--tests/testlib.py76
-rw-r--r--tox.ini85
81 files changed, 2362 insertions, 3236 deletions
diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml
index b7b094e..919b992 100644
--- a/.ci/azure-pipelines-steps.yml
+++ b/.ci/azure-pipelines-steps.yml
@@ -2,6 +2,7 @@
# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool
# `{script: ...}` is shorthand for `{task: CmdLine@<mumble>, inputs: {script: ...}}`.
+# The shell is bash.
# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines
# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines
@@ -9,24 +10,85 @@ steps:
- task: UsePythonVersion@0
displayName: Install python
inputs:
+ githubToken: '$(GITHUB_PYVER_TOKEN)'
versionSpec: '$(python.version)'
condition: ne(variables['python.version'], '')
- script: |
- type python
- python --version
- displayName: Show python version
+ set -o errexit
+ set -o nounset
+ set -o pipefail
-- script: |
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
displayName: Install build deps
condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux'))
-- script: python -mpip install "tox<4.0"
+- script: |
+ set -o errexit
+ set -o nounset
+ set -o pipefail
+
+ # macOS builders lack a realpath command
+ type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
+ type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
+ type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
+ echo
+
+ if [ -e /usr/bin/python ]; then
+ echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
+ fi
+
+ if [ -e /usr/bin/python2 ]; then
+ echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
+ fi
+
+ if [ -e /usr/bin/python2.7 ]; then
+ echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
+ fi
+ displayName: Show python versions
+
+- script: |
+ set -o errexit
+ set -o nounset
+ set -o pipefail
+
+ # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+ PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+ if [[ -z $PYTHON ]]; then
+ echo 1>&2 "Python interpreter could not be determined"
+ exit 1
+ fi
+
+ if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
+ "$PYTHON" -m ensurepip --user --altinstall --no-default-pip
+ "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+ elif [[ $PYTHON == "python2.7" ]]; then
+ curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+ "$PYTHON" get-pip.py --user --no-python-version-warning
+ # Avoid Python 2.x pip masking system pip
+ rm -f ~/.local/bin/{easy_install,pip,wheel}
+ "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+ else
+ "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+ fi
displayName: Install tooling
-- script: python -mtox -e "$(tox.env)"
+- script: |
+ set -o errexit
+ set -o nounset
+ set -o pipefail
+
+ # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+ PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+ if [[ -z $PYTHON ]]; then
+ echo 1>&2 "Python interpreter could not be determined"
+ exit 1
+ fi
+
+ "$PYTHON" -m tox -e "$(tox.env)"
displayName: "Run tests"
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml
index 4513ae9..3630f83 100644
--- a/.ci/azure-pipelines.yml
+++ b/.ci/azure-pipelines.yml
@@ -21,25 +21,21 @@ jobs:
matrix:
Mito_27:
tox.env: py27-mode_mitogen
- Mito_310:
- python.version: '3.10'
- tox.env: py310-mode_mitogen
+ Mito_312:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen
- # TODO: test python3, python3 tests are broken
Loc_27_210:
tox.env: py27-mode_localhost-ansible2.10
- Loc_27_4:
- tox.env: py27-mode_localhost-ansible4
+ Loc_312_6:
+ python.version: '3.12'
+ tox.env: py312-mode_localhost-ansible6
- # NOTE: this hangs when ran in Ubuntu 18.04
Van_27_210:
- tox.env: py27-mode_localhost-ansible2.10
- STRATEGY: linear
- ANSIBLE_SKIP_TAGS: resource_intensive
- Van_27_4:
- tox.env: py27-mode_localhost-ansible4
- STRATEGY: linear
- ANSIBLE_SKIP_TAGS: resource_intensive
+ tox.env: py27-mode_localhost-ansible2.10-strategy_linear
+ Van_312_6:
+ python.version: '3.12'
+ tox.env: py312-mode_localhost-ansible6-strategy_linear
- job: Linux
pool:
@@ -96,33 +92,33 @@ jobs:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_ubuntu2004
- Mito_310_centos6:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_centos6
- Mito_310_centos7:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_centos7
- Mito_310_centos8:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_centos8
- Mito_310_debian9:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_debian9
- Mito_310_debian10:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_debian10
- Mito_310_debian11:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_debian11
- Mito_310_ubuntu1604:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_ubuntu1604
- Mito_310_ubuntu1804:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_ubuntu1804
- Mito_310_ubuntu2004:
- python.version: '3.10'
- tox.env: py310-mode_mitogen-distro_ubuntu2004
+ Mito_312_centos6:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_centos6
+ Mito_312_centos7:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_centos7
+ Mito_312_centos8:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_centos8
+ Mito_312_debian9:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_debian9
+ Mito_312_debian10:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_debian10
+ Mito_312_debian11:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_debian11
+ Mito_312_ubuntu1604:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_ubuntu1604
+ Mito_312_ubuntu1804:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_ubuntu1804
+ Mito_312_ubuntu2004:
+ python.version: '3.12'
+ tox.env: py312-mode_mitogen-distro_ubuntu2004
Ans_27_210:
tox.env: py27-mode_ansible-ansible2.10
@@ -136,18 +132,18 @@ jobs:
python.version: '3.6'
tox.env: py36-mode_ansible-ansible4
- Ans_310_210:
- python.version: '3.10'
- tox.env: py310-mode_ansible-ansible2.10
- Ans_310_3:
- python.version: '3.10'
- tox.env: py310-mode_ansible-ansible3
- Ans_310_4:
- python.version: '3.10'
- tox.env: py310-mode_ansible-ansible4
- Ans_310_5:
- python.version: '3.10'
- tox.env: py310-mode_ansible-ansible5
- Ans_310_6:
- python.version: '3.10'
- tox.env: py310-mode_ansible-ansible6
+ Ans_311_210:
+ python.version: '3.11'
+ tox.env: py311-mode_ansible-ansible2.10
+ Ans_311_3:
+ python.version: '3.11'
+ tox.env: py311-mode_ansible-ansible3
+ Ans_311_4:
+ python.version: '3.11'
+ tox.env: py311-mode_ansible-ansible4
+ Ans_311_5:
+ python.version: '3.11'
+ tox.env: py311-mode_ansible-ansible5
+ Ans_312_6:
+ python.version: '3.12'
+ tox.env: py312-mode_ansible-ansible6
diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py
index 47a8ddf..3e71638 100644
--- a/.ci/ci_lib.py
+++ b/.ci/ci_lib.py
@@ -10,7 +10,10 @@ import shutil
import sys
import tempfile
-import subprocess32 as subprocess
+if sys.version_info < (3, 0):
+ import subprocess32 as subprocess
+else:
+ import subprocess
try:
import urlparse
diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py
index 69d67cd..c50ef22 100755
--- a/.ci/localhost_ansible_tests.py
+++ b/.ci/localhost_ansible_tests.py
@@ -1,6 +1,10 @@
#!/usr/bin/env python
# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
+from __future__ import print_function
+
+import getpass
+import io
import os
import subprocess
import sys
@@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'):
os.chdir(IMAGE_PREP_DIR)
ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+ # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
+ # and os.environ['USER'] is not populated on Azure macOS runners.
+ os.chdir(HOSTS_DIR)
+ with io.open('default.hosts', 'r+', encoding='utf-8') as f:
+ user = getpass.getuser()
+ content = f.read()
+ content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
+ f.seek(0)
+ f.write(content)
+ f.truncate()
+ ci_lib.dump_file('default.hosts')
+
+ cmd = ';'.join([
+ 'from __future__ import print_function',
+ 'import os, sys',
+ 'print(sys.executable, os.path.realpath(sys.executable))',
+ ])
+ for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+ print(interpreter)
+ try:
+ subprocess.call([interpreter, '-c', cmd])
+ except OSError as exc:
+ print(exc)
+
+ print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+ environ = os.environ.copy()
+ environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+ try:
+ subprocess.call([interpreter, '-c', cmd], env=environ)
+ except OSError as exc:
+ print(exc)
+
with ci_lib.Fold('ansible'):
os.chdir(TESTS_DIR)
diff --git a/.gitignore b/.gitignore
index aa75f69..7297d72 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ venvs/**
*.pyc
*.pyd
*.pyo
+*.retry
MANIFEST
build/
dist/
diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py
index cec465c..a187083 100644
--- a/ansible_mitogen/module_finder.py
+++ b/ansible_mitogen/module_finder.py
@@ -31,15 +31,31 @@ from __future__ import unicode_literals
__metaclass__ = type
import collections
-import imp
+import logging
import os
+import re
+import sys
+
+try:
+ # Python >= 3.4, PEP 451 ModuleSpec API
+ import importlib.machinery
+ import importlib.util
+except ImportError:
+ # Python < 3.4, PEP 302 Import Hooks
+ import imp
import mitogen.master
+LOG = logging.getLogger(__name__)
PREFIX = 'ansible.module_utils.'
+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
+# name Unqualified name of the module.
+# path Filesystem path of the module.
+# kind One of the constants in `imp`, as returned in `imp.find_module()`
+# parent `ansible_mitogen.module_finder.Module` of parent package (if any).
Module = collections.namedtuple('Module', 'name path kind parent')
@@ -119,14 +135,121 @@ def find_relative(parent, name, path=()):
def scan_fromlist(code):
+ """Return an iterator of (level, name) for explicit imports in a code
+ object.
+
+ Not all names identify a module. `from os import name, path` generates
+ `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+ >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+ >>> code = compile(src, '<str>', 'exec')
+ >>> list(scan_fromlist(code))
+ [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+ """
for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
for name in fromlist:
- yield level, '%s.%s' % (modname_s, name)
+ yield level, str('%s.%s' % (modname_s, name))
if not fromlist:
yield level, modname_s
+def walk_imports(code, prefix=None):
+ """Return an iterator of names for implicit parent imports & explicit
+ imports in a code object.
+
+ If a prefix is provided, then only children of that prefix are included.
+ Not all names identify a module. `from os import name, path` generates
+ `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+ >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+ >>> code = compile(source, '<str>', 'exec')
+ >>> list(walk_imports(code))
+ ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+ >>> list(walk_imports(code, prefix='b'))
+ ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+ """
+ if prefix is None:
+ prefix = ''
+ pattern = re.compile(r'(^|\.)(\w+)')
+ start = len(prefix)
+ for _, name, fromlist in mitogen.master.scan_code_imports(code):
+ if not name.startswith(prefix):
+ continue
+ for match in pattern.finditer(name, start):
+ yield name[:match.end()]
+ for leaf in fromlist:
+ yield str('%s.%s' % (name, leaf))
+
+
def scan(module_name, module_path, search_path):
+ # type: (str, str, list[str]) -> list[(str, str, bool)]
+ """Return a list of (name, path, is_package) for ansible.module_utils
+ imports used by an Ansible module.
+ """
+ log = LOG.getChild('scan')
+ log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+ if sys.version_info >= (3, 4):
+ result = _scan_importlib_find_spec(
+ module_name, module_path, search_path,
+ )
+ log.debug('_scan_importlib_find_spec %r', result)
+ else:
+ result = _scan_imp_find_module(module_name, module_path, search_path)
+ log.debug('_scan_imp_find_module %r', result)
+ return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+ # type: (str, str, list[str]) -> list[(str, str, bool)]
+ module = importlib.machinery.ModuleSpec(
+ module_name, loader=None, origin=module_path,
+ )
+ prefix = importlib.machinery.ModuleSpec(
+ PREFIX.rstrip('.'), loader=None,
+ )
+ prefix.submodule_search_locations = search_path
+ queue = collections.deque([module])
+ specs = {prefix.name: prefix}
+ while queue:
+ spec = queue.popleft()
+ if spec.origin is None:
+ continue
+ try:
+ with open(spec.origin, 'rb') as f:
+ code = compile(f.read(), spec.name, 'exec')
+ except Exception as exc:
+ raise ValueError((exc, module, spec, specs))
+
+ for name in walk_imports(code, prefix.name):
+ if name in specs:
+ continue
+
+ parent_name = name.rpartition('.')[0]
+ parent = specs[parent_name]
+ if parent is None or not parent.submodule_search_locations:
+ specs[name] = None
+ continue
+
+ child = importlib.util._find_spec(
+ name, parent.submodule_search_locations,
+ )
+ if child is None or child.origin is None:
+ specs[name] = None
+ continue
+
+ specs[name] = child
+ queue.append(child)
+
+ del specs[prefix.name]
+ return sorted(
+ (spec.name, spec.origin, spec.submodule_search_locations is not None)
+ for spec in specs.values() if spec is not None
+ )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+ # type: (str, str, list[str]) -> list[(str, str, bool)]
module = Module(module_name, module_path, imp.PY_SOURCE, None)
stack = [module]
seen = set()
diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py
index c4cb71f..8da1b67 100644
--- a/ansible_mitogen/runner.py
+++ b/ansible_mitogen/runner.py
@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import atexit
-import imp
import json
import os
import re
@@ -65,6 +64,14 @@ except ImportError:
ctypes = None
try:
+ # Python >= 3.4, PEP 451 ModuleSpec API
+ import importlib.machinery
+ import importlib.util
+except ImportError:
+ # Python < 3.4, PEP 302 Import Hooks
+ import imp
+
+try:
# Cannot use cStringIO as it does not support Unicode.
from StringIO import StringIO
except ImportError:
@@ -514,10 +521,71 @@ class ModuleUtilsImporter(object):
sys.modules.pop(fullname, None)
def find_module(self, fullname, path=None):
+ """
+ Return a loader for the module with fullname, if we will load it.
+
+ Implements importlib.abc.MetaPathFinder.find_module().
+ Deprecrated in Python 3.4+, replaced by find_spec().
+ Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+ """
if fullname in self._by_fullname:
return self
+ def find_spec(self, fullname, path, target=None):
+ """
+ Return a `ModuleSpec` for module with `fullname` if we will load it.
+ Otherwise return `None`.
+
+ Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+ """
+ if fullname.endswith('.'):
+ return None
+
+ try:
+ module_path, is_package = self._by_fullname[fullname]
+ except KeyError:
+ LOG.debug('Skipping %s: not present', fullname)
+ return None
+
+ LOG.debug('Handling %s', fullname)
+ origin = 'master:%s' % (module_path,)
+ return importlib.machinery.ModuleSpec(
+ fullname, loader=self, origin=origin, is_package=is_package,
+ )
+
+ def create_module(self, spec):
+ """
+ Return a module object for the given ModuleSpec.
+
+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+ Unlike Loader.load_module() this shouldn't populate sys.modules or
+ set module attributes. Both are done by Python.
+ """
+ module = types.ModuleType(spec.name)
+ # FIXME create_module() shouldn't initialise module attributes
+ module.__file__ = spec.origin
+ return module
+
+ def exec_module(self, module):
+ """
+ Execute the module to initialise it. Don't return anything.
+
+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+ """
+ spec = module.__spec__
+ path, _ = self._by_fullname[spec.name]
+ source = ansible_mitogen.target.get_small_file(self._context, path)
+ code = compile(source, path, 'exec', 0, 1)
+ exec(code, module.__dict__)
+ self._loaded.add(spec.name)
+
def load_module(self, fullname):
+ """
+ Return the loaded module specified by fullname.
+
+ Implements PEP 302 importlib.abc.Loader.load_module().
+ Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+ """
path, is_pkg = self._by_fullname[fullname]
source = ansible_mitogen.target.get_small_file(self._context, path)
code = compile(source, path, 'exec', 0, 1)
@@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner):
synchronization mechanism by importing everything the module will need
prior to detaching.
"""
+ # I think "custom" means "found in custom module_utils search path",
+ # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
for fullname, _, _ in self.module_map['custom']:
mitogen.core.import_module(fullname)
+
+ # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+ # as opposed to Python builtin modules such as sys.
for fullname in self.module_map['builtin']:
try:
mitogen.core.import_module(fullname)
- except ImportError:
+ except ImportError as exc:
# #590: Ansible 2.8 module_utils.distro is a package that
# replaces itself in sys.modules with a non-package during
# import. Prior to replacement, it is a real package containing
@@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner):
# loop progresses to the next entry and attempts to preload
# 'distro._distro', the import mechanism will fail. So here we
# silently ignore any failure for it.
- if fullname != 'ansible.module_utils.distro._distro':
- raise
+ if fullname == 'ansible.module_utils.distro._distro':
+ continue
+
+ # ansible.module_utils.compat.selinux raises ImportError if it
+ # can't load libselinux.so. The importer would usually catch
+ # this & skip selinux operations. We don't care about selinux,
+ # we're using import to get a copy of the module.
+ if (fullname == 'ansible.module_utils.compat.selinux'
+ and exc.msg == 'unable to load libselinux.so'):
+ continue
+
+ raise
def _setup_excepthook(self):
"""
diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py
index 25c5a69..a01b261 100644
--- a/ansible_mitogen/utils.py
+++ b/ansible_mitogen/utils.py
@@ -1,7 +1,7 @@
from __future__ import absolute_import, division, print_function
__metaclass__ = type
-import distutils.version
+import re
import ansible
@@ -9,6 +9,21 @@ __all__ = [
'ansible_version',
]
-ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version)
-del distutils
+
+def _parse(v_string):
+ # Adapted from distutils.version.LooseVersion.parse()
+ component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
+ for component in component_re.split(v_string):
+ if not component or component == '.':
+ continue
+ try:
+ yield int(component)
+ except ValueError:
+ yield component
+
+
+ansible_version = tuple(_parse(ansible.__version__))
+
+del _parse
+del re
del ansible
diff --git a/debian/changelog b/debian/changelog
index 850f3ef..5d239da 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+python-mitogen (0.3.5-1) unstable; urgency=medium
+
+ * New upstream release.
+ * Refresh patches.
+ * Drop patches, superseded upstream: no-subprocess32, python-3.11,
+ python-3.12.
+
+ -- Stefano Rivera <stefanor@debian.org> Thu, 28 Mar 2024 13:45:03 -0400
+
python-mitogen (0.3.4-3) unstable; urgency=medium
* Patch: Support setting interpreter_python in config, again.
diff --git a/debian/patches/ansible-2.16 b/debian/patches/ansible-2.16
index 3b94c37..d752943 100644
--- a/debian/patches/ansible-2.16
+++ b/debian/patches/ansible-2.16
@@ -24,10 +24,10 @@ index 754ec6c..9729b8a 100644
NEW_VERSION_MSG = (
"Your Ansible version (%s) is too recent. The most recent version\n"
diff --git a/mitogen/core.py b/mitogen/core.py
-index bee722e..865f826 100644
+index cd02012..d92c47f 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
-@@ -842,6 +842,9 @@ class Message(object):
+@@ -874,6 +874,9 @@ class Message(object):
s, n = LATIN1_CODEC.encode(s)
return s
@@ -37,7 +37,7 @@ index bee722e..865f826 100644
def _find_global(self, module, func):
"""
Return the class implementing `module_name.class_name` or raise
-@@ -860,6 +863,8 @@ class Message(object):
+@@ -892,6 +895,8 @@ class Message(object):
return Secret
elif func == 'Kwargs':
return Kwargs
diff --git a/debian/patches/ask-become-pass b/debian/patches/ask-become-pass
index 2130532..eec9f43 100644
--- a/debian/patches/ask-become-pass
+++ b/debian/patches/ask-become-pass
@@ -12,13 +12,12 @@ Bug-Debian: https://bugs.debian.org/1042556
.ci/ansible_tests.py | 11 ++++++++++-
ansible_mitogen/transport_config.py | 16 ++++++++++++----
tests/ansible/hosts/default.hosts | 2 ++
- tests/ansible/hosts/transport_config.hosts | 2 +-
.../integration/transport_config/become_pass.yml | 5 +++--
tests/ansible/regression/all.yml | 1 +
tests/ansible/regression/become_test.yml | 9 +++++++++
.../regression/issue_952__ask_become_pass.yml | 21 +++++++++++++++++++++
tests/requirements.txt | 1 +
- 9 files changed, 60 insertions(+), 8 deletions(-)
+ 8 files changed, 59 insertions(+), 7 deletions(-)
create mode 100644 tests/ansible/regression/become_test.yml
create mode 100644 tests/ansible/regression/issue_952__ask_become_pass.yml
@@ -86,28 +85,15 @@ index 5fc7818..1fc1e80 100644
def password(self):
diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts
-index d40c3dd..7ef231b 100644
+index 1bec001..4f5ea4c 100644
--- a/tests/ansible/hosts/default.hosts
+++ b/tests/ansible/hosts/default.hosts
-@@ -7,3 +7,5 @@ target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}"
+@@ -10,3 +10,5 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
[test-targets]
target
+
+[linux_containers]
-diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts
-index 7d7b526..d6a15c9 100644
---- a/tests/ansible/hosts/transport_config.hosts
-+++ b/tests/ansible/hosts/transport_config.hosts
-@@ -21,7 +21,7 @@ tc_transport
- # That causes approx 10 seconds timeout per task - there's no host to connect to.
- # This optimisation should not be relied in any test.
- # Note: tc-python-path-* are intentionally not included.
--ansible_python_interpreter = python3000 # Not expected to exist
-+ansible_python_interpreter = python3000
-
- [tc_transport]
- tc-transport-unset
diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml
index 317e052..670faf5 100644
--- a/tests/ansible/integration/transport_config/become_pass.yml
@@ -176,14 +162,14 @@ index 0000000..da269e2
+ changed_when: false
+ check_mode: false
diff --git a/tests/requirements.txt b/tests/requirements.txt
-index 7301bee..eb84b9b 100644
+index 6d87d17..ae3abb8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
-@@ -3,6 +3,7 @@ coverage==5.5; python_version < '3.7'
- coverage==6.4.4; python_version >= '3.7'
- Django==1.6.11 # Last version supporting 2.6.
- mock==2.0.0
+@@ -12,6 +12,7 @@ Django==3.2.20; python_version >= '3.6'
+ mock==3.0.5; python_version == '2.7'
+ mock==5.1.0; python_version >= '3.6'
+
+pexpect==4.8
- pytz==2018.5
- cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect
- pycparser==2.18 # Last version supporting 2.6.
+ psutil==5.9.8
+
+ pytest==4.6.11; python_version == '2.7'
diff --git a/debian/patches/no-subprocess32 b/debian/patches/no-subprocess32
deleted file mode 100644
index e347210..0000000
--- a/debian/patches/no-subprocess32
+++ /dev/null
@@ -1,26 +0,0 @@
-From: Stefano Rivera <stefanor@debian.org>
-Date: Tue, 18 Jul 2023 18:39:01 +0200
-Subject: Use subprocess instead of subprocess32
-
-This is a compatibility module for Python 2.7, which isn't a use-case we
-are about in the tests in Debian. subprocess32 isn't available in
-Debian.
-
-Forwarded: not-needed
----
- tests/testlib.py | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/tests/testlib.py b/tests/testlib.py
-index 83629c3..5e9d0a7 100644
---- a/tests/testlib.py
-+++ b/tests/testlib.py
-@@ -18,7 +18,7 @@ except ImportError:
- import ConfigParser as configparser
-
- import psutil
--import subprocess32 as subprocess
-+import subprocess
-
- import mitogen.core
- import mitogen.fork
diff --git a/debian/patches/python-3.11 b/debian/patches/python-3.11
deleted file mode 100644
index 34893dc..0000000
--- a/debian/patches/python-3.11
+++ /dev/null
@@ -1,246 +0,0 @@
-From: =?utf-8?q?Nerijus_Bali=C5=ABnas?= <nerijus.baliunas@adv.bet>
-Date: Tue, 21 Feb 2023 13:04:28 +0200
-Subject: Add Python 3.11 support
-
-Co-authored-by: Alex Willmer <alex@moreati.org.uk>
-Origin: upstream, https://github.com/mitogen-hq/mitogen/pull/1019
----
- .ci/azure-pipelines.yml | 90 ++++++++++++++++----------------
- docs/ansible_detailed.rst | 4 +-
- docs/changelog.rst | 2 +
- setup.py | 1 +
- tests/ansible/lib/action/assert_equal.py | 7 ++-
- tests/requirements.txt | 5 +-
- tests/testlib.py | 2 +-
- tox.ini | 9 ++--
- 8 files changed, 64 insertions(+), 56 deletions(-)
-
-diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml
-index 4513ae9..f672240 100644
---- a/.ci/azure-pipelines.yml
-+++ b/.ci/azure-pipelines.yml
-@@ -21,9 +21,9 @@ jobs:
- matrix:
- Mito_27:
- tox.env: py27-mode_mitogen
-- Mito_310:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen
-+ Mito_311:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen
-
- # TODO: test python3, python3 tests are broken
- Loc_27_210:
-@@ -96,33 +96,33 @@ jobs:
- python.version: '3.6'
- tox.env: py36-mode_mitogen-distro_ubuntu2004
-
-- Mito_310_centos6:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_centos6
-- Mito_310_centos7:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_centos7
-- Mito_310_centos8:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_centos8
-- Mito_310_debian9:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_debian9
-- Mito_310_debian10:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_debian10
-- Mito_310_debian11:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_debian11
-- Mito_310_ubuntu1604:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_ubuntu1604
-- Mito_310_ubuntu1804:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_ubuntu1804
-- Mito_310_ubuntu2004:
-- python.version: '3.10'
-- tox.env: py310-mode_mitogen-distro_ubuntu2004
-+ Mito_311_centos6:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_centos6
-+ Mito_311_centos7:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_centos7
-+ Mito_311_centos8:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_centos8
-+ Mito_311_debian9:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_debian9
-+ Mito_311_debian10:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_debian10
-+ Mito_311_debian11:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_debian11
-+ Mito_311_ubuntu1604:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_ubuntu1604
-+ Mito_311_ubuntu1804:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_ubuntu1804
-+ Mito_311_ubuntu2004:
-+ python.version: '3.11'
-+ tox.env: py311-mode_mitogen-distro_ubuntu2004
-
- Ans_27_210:
- tox.env: py27-mode_ansible-ansible2.10
-@@ -136,18 +136,18 @@ jobs:
- python.version: '3.6'
- tox.env: py36-mode_ansible-ansible4
-
-- Ans_310_210:
-- python.version: '3.10'
-- tox.env: py310-mode_ansible-ansible2.10
-- Ans_310_3:
-- python.version: '3.10'
-- tox.env: py310-mode_ansible-ansible3
-- Ans_310_4:
-- python.version: '3.10'
-- tox.env: py310-mode_ansible-ansible4
-- Ans_310_5:
-- python.version: '3.10'
-- tox.env: py310-mode_ansible-ansible5
-- Ans_310_6:
-- python.version: '3.10'
-- tox.env: py310-mode_ansible-ansible6
-+ Ans_311_210:
-+ python.version: '3.11'
-+ tox.env: py311-mode_ansible-ansible2.10
-+ Ans_311_3:
-+ python.version: '3.11'
-+ tox.env: py311-mode_ansible-ansible3
-+ Ans_311_4:
-+ python.version: '3.11'
-+ tox.env: py311-mode_ansible-ansible4
-+ Ans_311_5:
-+ python.version: '3.11'
-+ tox.env: py311-mode_ansible-ansible5
-+ Ans_311_6:
-+ python.version: '3.11'
-+ tox.env: py311-mode_ansible-ansible6
-diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
-index 449771b..64fcd63 100644
---- a/docs/ansible_detailed.rst
-+++ b/docs/ansible_detailed.rst
-@@ -147,8 +147,8 @@ Noteworthy Differences
-
- * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
- Mitogen 0.3.1+ supports
-- - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10
-- - Ansible 5 and 6; with Python 3.8-3.10
-+ - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11
-+ - Ansible 5 and 6; with Python 3.8-3.11
- Verify your installation is running one of these versions by checking
- ``ansible --version`` output.
-
-diff --git a/docs/changelog.rst b/docs/changelog.rst
-index d9a54bc..63c9ff2 100644
---- a/docs/changelog.rst
-+++ b/docs/changelog.rst
-@@ -4,6 +4,8 @@
- Release Notes
- =============
-
-+* :gh:issue:`987` Support Python 3.11
-+
-
- .. raw:: html
-
-diff --git a/setup.py b/setup.py
-index 9d2ff36..4d7fadf 100644
---- a/setup.py
-+++ b/setup.py
-@@ -77,6 +77,7 @@ setup(
- 'Programming Language :: Python :: 3.8',
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
-+ 'Programming Language :: Python :: 3.11',
- 'Programming Language :: Python :: Implementation :: CPython',
- 'Topic :: System :: Distributed Computing',
- 'Topic :: System :: Systems Administration',
-diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py
-index 5923f4b..72264cf 100644
---- a/tests/ansible/lib/action/assert_equal.py
-+++ b/tests/ansible/lib/action/assert_equal.py
-@@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase
-
- TEMPLATE_KWARGS = {}
-
--_argspec = inspect.getargspec(ansible.template.Templar.template)
-+try:
-+ # inspect.getfullargspec() Added: 3.0
-+ _argspec = inspect.getfullargspec(ansible.template.Templar.template)
-+except AttributeError:
-+ # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11
-+ _argspec = inspect.getargspec(ansible.template.Templar.template)
- if 'bare_deprecated' in _argspec.args:
- TEMPLATE_KWARGS['bare_deprecated'] = False
-
-diff --git a/tests/requirements.txt b/tests/requirements.txt
-index eb84b9b..9bbfe46 100644
---- a/tests/requirements.txt
-+++ b/tests/requirements.txt
-@@ -1,12 +1,11 @@
--psutil==5.4.8
-+cffi==1.15.1
- coverage==5.5; python_version < '3.7'
- coverage==6.4.4; python_version >= '3.7'
- Django==1.6.11 # Last version supporting 2.6.
- mock==2.0.0
- pexpect==4.8
- pytz==2018.5
--cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect
--pycparser==2.18 # Last version supporting 2.6.
-+psutil==5.9.5
- pytest-catchlog==1.2.2
- pytest==3.1.2
- subprocess32==3.5.4
-diff --git a/tests/testlib.py b/tests/testlib.py
-index 5e9d0a7..da4d46f 100644
---- a/tests/testlib.py
-+++ b/tests/testlib.py
-@@ -87,7 +87,7 @@ def base_executable(executable=None):
- base_executable = None
-
- if base_executable and base_executable != executable:
-- return 'be', base_executable
-+ return base_executable
-
- # Python 2.x only has sys.base_prefix if running outside a virtualenv.
- try:
-diff --git a/tox.ini b/tox.ini
-index da71093..3b3cb9f 100644
---- a/tox.ini
-+++ b/tox.ini
-@@ -37,10 +37,10 @@
- envlist =
- init,
- py{27,36}-mode_ansible-ansible{2.10,3,4},
-- py{310}-mode_ansible-ansible{2.10,3,4,5,6},
-- py{27,36,310}-mode_mitogen-distro_centos{6,7,8},
-- py{27,36,310}-mode_mitogen-distro_debian{9,10,11},
-- py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004},
-+ py{311}-mode_ansible-ansible{2.10,3,4,5,6},
-+ py{27,36,311}-mode_mitogen-distro_centos{6,7,8},
-+ py{27,36,311}-mode_mitogen-distro_debian{9,10,11},
-+ py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004},
- report,
-
- [testenv]
-@@ -53,6 +53,7 @@ basepython =
- py38: python3.8
- py39: python3.9
- py310: python3.10
-+ py311: python3.11
- deps =
- -r{toxinidir}/tests/requirements.txt
- mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
diff --git a/debian/patches/python-3.12 b/debian/patches/python-3.12
deleted file mode 100644
index 91dc81e..0000000
--- a/debian/patches/python-3.12
+++ /dev/null
@@ -1,2358 +0,0 @@
-From: Alex Willmer <alex@moreati.org.uk>
-Date: Fri, 8 Mar 2024 17:39:26 -0400
-Subject: Python 3.12 support
-
-Origin: https://github.com/mitogen-hq/mitogen/pull/1032
-Bug-Debian: https://bugs.debian.org/1061755
-Bug-Debian: https://bugs.debian.org/1058132
-Bug-upstream: https://github.com/mitogen-hq/mitogen/issues/1033
----
- .gitignore | 1 +
- ansible_mitogen/module_finder.py | 127 +++++++++++++-
- ansible_mitogen/runner.py | 91 +++++++++-
- ansible_mitogen/utils.py | 21 ++-
- docs/ansible_detailed.rst | 4 +-
- docs/conf.py | 8 +-
- mitogen/core.py | 192 +++++++++++++++++++--
- mitogen/master.py | 122 ++++++++++++-
- mitogen/parent.py | 27 +--
- setup.py | 1 +
- tests/ansible/files/cwd_show | 22 +++
- tests/ansible/hosts/become_same_user.hosts | 2 +
- tests/ansible/hosts/connection_delegation.hosts | 1 +
- tests/ansible/hosts/default.hosts | 5 +-
- tests/ansible/hosts/k3.hosts | 1 +
- tests/ansible/hosts/localhost.hosts | 1 +
- tests/ansible/hosts/transport_config.hosts | 8 +-
- tests/ansible/integration/become/su_password.yml | 41 ++++-
- .../connection_delegation/delegate_to_template.yml | 4 +-
- .../interpreter_discovery/ansible_2_8_tests.yml | 30 ++--
- .../interpreter_discovery/complex_args.yml | 23 ++-
- ...custom_python_new_style_missing_interpreter.yml | 5 +
- .../runner/custom_python_new_style_module.yml | 3 +-
- .../runner/custom_python_prehistoric_module.yml | 6 +
- .../modules/custom_python_detect_environment.py | 7 +-
- .../lib/modules/custom_python_external_module.py | 2 +-
- .../lib/modules/custom_python_external_pkg.py | 2 +-
- .../lib/modules/custom_python_json_args_module.py | 2 +-
- .../lib/modules/custom_python_leaky_class_vars.py | 2 +-
- .../lib/modules/custom_python_modify_environ.py | 2 +-
- .../custom_python_new_style_missing_interpreter.py | 49 ++++++
- .../lib/modules/custom_python_new_style_module.py | 51 +++++-
- .../ansible/lib/modules/custom_python_os_getcwd.py | 2 +-
- .../modules/custom_python_prehistoric_module.py | 7 +-
- .../lib/modules/custom_python_run_script.py | 2 +-
- .../lib/modules/custom_python_uses_distro.py | 2 +-
- .../lib/modules/custom_python_want_json_module.py | 2 +-
- .../lib/modules/mitogen_test_gethostbyname.py | 2 +-
- tests/ansible/lib/modules/module_finder_test.py | 12 ++
- tests/ansible/lib/modules/test_echo_module.py | 53 ++++--
- .../issue_152__virtualenv_python_fails.yml | 11 +-
- tests/ansible/setup/all.yml | 3 +-
- tests/ansible/setup/report.yml | 20 ---
- tests/ansible/setup/report_controller.yml | 17 ++
- tests/ansible/setup/report_targets.yml | 15 ++
- tests/ansible/tests/module_finder_test.py | 80 +++++++++
- tests/image_prep/_user_accounts.yml | 26 +--
- tests/importer_test.py | 126 +++++++++++++-
- tests/module_finder_test.py | 14 +-
- tests/requirements-tox.txt | 4 +
- tests/requirements.txt | 5 +-
- tests/testlib.py | 9 +-
- tox.ini | 24 ++-
- 53 files changed, 1151 insertions(+), 148 deletions(-)
- create mode 100755 tests/ansible/files/cwd_show
- create mode 100644 tests/ansible/lib/modules/module_finder_test.py
- delete mode 100644 tests/ansible/setup/report.yml
- create mode 100644 tests/ansible/setup/report_controller.yml
- create mode 100644 tests/ansible/setup/report_targets.yml
- create mode 100644 tests/ansible/tests/module_finder_test.py
- create mode 100644 tests/requirements-tox.txt
-
-diff --git a/.gitignore b/.gitignore
-index aa75f69..7297d72 100644
---- a/.gitignore
-+++ b/.gitignore
-@@ -6,6 +6,7 @@ venvs/**
- *.pyc
- *.pyd
- *.pyo
-+*.retry
- MANIFEST
- build/
- dist/
-diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py
-index cec465c..a187083 100644
---- a/ansible_mitogen/module_finder.py
-+++ b/ansible_mitogen/module_finder.py
-@@ -31,15 +31,31 @@ from __future__ import unicode_literals
- __metaclass__ = type
-
- import collections
--import imp
-+import logging
- import os
-+import re
-+import sys
-+
-+try:
-+ # Python >= 3.4, PEP 451 ModuleSpec API
-+ import importlib.machinery
-+ import importlib.util
-+except ImportError:
-+ # Python < 3.4, PEP 302 Import Hooks
-+ import imp
-
- import mitogen.master
-
-
-+LOG = logging.getLogger(__name__)
- PREFIX = 'ansible.module_utils.'
-
-
-+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
-+# name Unqualified name of the module.
-+# path Filesystem path of the module.
-+# kind One of the constants in `imp`, as returned in `imp.find_module()`
-+# parent `ansible_mitogen.module_finder.Module` of parent package (if any).
- Module = collections.namedtuple('Module', 'name path kind parent')
-
-
-@@ -119,14 +135,121 @@ def find_relative(parent, name, path=()):
-
-
- def scan_fromlist(code):
-+ """Return an iterator of (level, name) for explicit imports in a code
-+ object.
-+
-+ Not all names identify a module. `from os import name, path` generates
-+ `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
-+
-+ >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
-+ >>> code = compile(src, '<str>', 'exec')
-+ >>> list(scan_fromlist(code))
-+ [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
-+ """
- for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
- for name in fromlist:
-- yield level, '%s.%s' % (modname_s, name)
-+ yield level, str('%s.%s' % (modname_s, name))
- if not fromlist:
- yield level, modname_s
-
-
-+def walk_imports(code, prefix=None):
-+ """Return an iterator of names for implicit parent imports & explicit
-+ imports in a code object.
-+
-+ If a prefix is provided, then only children of that prefix are included.
-+ Not all names identify a module. `from os import name, path` generates
-+ `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
-+
-+ >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
-+ >>> code = compile(source, '<str>', 'exec')
-+ >>> list(walk_imports(code))
-+ ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
-+ >>> list(walk_imports(code, prefix='b'))
-+ ['b.c', 'b.d', 'b.d.e', 'b.d.f']
-+ """
-+ if prefix is None:
-+ prefix = ''
-+ pattern = re.compile(r'(^|\.)(\w+)')
-+ start = len(prefix)
-+ for _, name, fromlist in mitogen.master.scan_code_imports(code):
-+ if not name.startswith(prefix):
-+ continue
-+ for match in pattern.finditer(name, start):
-+ yield name[:match.end()]
-+ for leaf in fromlist:
-+ yield str('%s.%s' % (name, leaf))
-+
-+
- def scan(module_name, module_path, search_path):
-+ # type: (str, str, list[str]) -> list[(str, str, bool)]
-+ """Return a list of (name, path, is_package) for ansible.module_utils
-+ imports used by an Ansible module.
-+ """
-+ log = LOG.getChild('scan')
-+ log.debug('%r, %r, %r', module_name, module_path, search_path)
-+
-+ if sys.version_info >= (3, 4):
-+ result = _scan_importlib_find_spec(
-+ module_name, module_path, search_path,
-+ )
-+ log.debug('_scan_importlib_find_spec %r', result)
-+ else:
-+ result = _scan_imp_find_module(module_name, module_path, search_path)
-+ log.debug('_scan_imp_find_module %r', result)
-+ return result
-+
-+
-+def _scan_importlib_find_spec(module_name, module_path, search_path):
-+ # type: (str, str, list[str]) -> list[(str, str, bool)]
-+ module = importlib.machinery.ModuleSpec(
-+ module_name, loader=None, origin=module_path,
-+ )
-+ prefix = importlib.machinery.ModuleSpec(
-+ PREFIX.rstrip('.'), loader=None,
-+ )
-+ prefix.submodule_search_locations = search_path
-+ queue = collections.deque([module])
-+ specs = {prefix.name: prefix}
-+ while queue:
-+ spec = queue.popleft()
-+ if spec.origin is None:
-+ continue
-+ try:
-+ with open(spec.origin, 'rb') as f:
-+ code = compile(f.read(), spec.name, 'exec')
-+ except Exception as exc:
-+ raise ValueError((exc, module, spec, specs))
-+
-+ for name in walk_imports(code, prefix.name):
-+ if name in specs:
-+ continue
-+
-+ parent_name = name.rpartition('.')[0]
-+ parent = specs[parent_name]
-+ if parent is None or not parent.submodule_search_locations:
-+ specs[name] = None
-+ continue
-+
-+ child = importlib.util._find_spec(
-+ name, parent.submodule_search_locations,
-+ )
-+ if child is None or child.origin is None:
-+ specs[name] = None
-+ continue
-+
-+ specs[name] = child
-+ queue.append(child)
-+
-+ del specs[prefix.name]
-+ return sorted(
-+ (spec.name, spec.origin, spec.submodule_search_locations is not None)
-+ for spec in specs.values() if spec is not None
-+ )
-+
-+
-+def _scan_imp_find_module(module_name, module_path, search_path):
-+ # type: (str, str, list[str]) -> list[(str, str, bool)]
- module = Module(module_name, module_path, imp.PY_SOURCE, None)
- stack = [module]
- seen = set()
-diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py
-index c4cb71f..8da1b67 100644
---- a/ansible_mitogen/runner.py
-+++ b/ansible_mitogen/runner.py
-@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function
- __metaclass__ = type
-
- import atexit
--import imp
- import json
- import os
- import re
-@@ -64,6 +63,14 @@ except ImportError:
- # Python 2.4
- ctypes = None
-
-+try:
-+ # Python >= 3.4, PEP 451 ModuleSpec API
-+ import importlib.machinery
-+ import importlib.util
-+except ImportError:
-+ # Python < 3.4, PEP 302 Import Hooks
-+ import imp
-+
- try:
- # Cannot use cStringIO as it does not support Unicode.
- from StringIO import StringIO
-@@ -514,10 +521,71 @@ class ModuleUtilsImporter(object):
- sys.modules.pop(fullname, None)
-
- def find_module(self, fullname, path=None):
-+ """
-+ Return a loader for the module with fullname, if we will load it.
-+
-+ Implements importlib.abc.MetaPathFinder.find_module().
-+ Deprecrated in Python 3.4+, replaced by find_spec().
-+ Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
-+ """
- if fullname in self._by_fullname:
- return self
-
-+ def find_spec(self, fullname, path, target=None):
-+ """
-+ Return a `ModuleSpec` for module with `fullname` if we will load it.
-+ Otherwise return `None`.
-+
-+ Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
-+ """
-+ if fullname.endswith('.'):
-+ return None
-+
-+ try:
-+ module_path, is_package = self._by_fullname[fullname]
-+ except KeyError:
-+ LOG.debug('Skipping %s: not present', fullname)
-+ return None
-+
-+ LOG.debug('Handling %s', fullname)
-+ origin = 'master:%s' % (module_path,)
-+ return importlib.machinery.ModuleSpec(
-+ fullname, loader=self, origin=origin, is_package=is_package,
-+ )
-+
-+ def create_module(self, spec):
-+ """
-+ Return a module object for the given ModuleSpec.
-+
-+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
-+ Unlike Loader.load_module() this shouldn't populate sys.modules or
-+ set module attributes. Both are done by Python.
-+ """
-+ module = types.ModuleType(spec.name)
-+ # FIXME create_module() shouldn't initialise module attributes
-+ module.__file__ = spec.origin
-+ return module
-+
-+ def exec_module(self, module):
-+ """
-+ Execute the module to initialise it. Don't return anything.
-+
-+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
-+ """
-+ spec = module.__spec__
-+ path, _ = self._by_fullname[spec.name]
-+ source = ansible_mitogen.target.get_small_file(self._context, path)
-+ code = compile(source, path, 'exec', 0, 1)
-+ exec(code, module.__dict__)
-+ self._loaded.add(spec.name)
-+
- def load_module(self, fullname):
-+ """
-+ Return the loaded module specified by fullname.
-+
-+ Implements PEP 302 importlib.abc.Loader.load_module().
-+ Deprecated in Python 3.4+, replaced by create_module() & exec_module().
-+ """
- path, is_pkg = self._by_fullname[fullname]
- source = ansible_mitogen.target.get_small_file(self._context, path)
- code = compile(source, path, 'exec', 0, 1)
-@@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner):
- synchronization mechanism by importing everything the module will need
- prior to detaching.
- """
-+ # I think "custom" means "found in custom module_utils search path",
-+ # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
- for fullname, _, _ in self.module_map['custom']:
- mitogen.core.import_module(fullname)
-+
-+ # I think "builtin" means "part of ansible/ansible-base/ansible-core",
-+ # as opposed to Python builtin modules such as sys.
- for fullname in self.module_map['builtin']:
- try:
- mitogen.core.import_module(fullname)
-- except ImportError:
-+ except ImportError as exc:
- # #590: Ansible 2.8 module_utils.distro is a package that
- # replaces itself in sys.modules with a non-package during
- # import. Prior to replacement, it is a real package containing
-@@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner):
- # loop progresses to the next entry and attempts to preload
- # 'distro._distro', the import mechanism will fail. So here we
- # silently ignore any failure for it.
-- if fullname != 'ansible.module_utils.distro._distro':
-- raise
-+ if fullname == 'ansible.module_utils.distro._distro':
-+ continue
-+
-+ # ansible.module_utils.compat.selinux raises ImportError if it
-+ # can't load libselinux.so. The importer would usually catch
-+ # this & skip selinux operations. We don't care about selinux,
-+ # we're using import to get a copy of the module.
-+ if (fullname == 'ansible.module_utils.compat.selinux'
-+ and exc.msg == 'unable to load libselinux.so'):
-+ continue
-+
-+ raise
-
- def _setup_excepthook(self):
- """
-diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py
-index 25c5a69..a01b261 100644
---- a/ansible_mitogen/utils.py
-+++ b/ansible_mitogen/utils.py
-@@ -1,7 +1,7 @@
- from __future__ import absolute_import, division, print_function
- __metaclass__ = type
-
--import distutils.version
-+import re
-
- import ansible
-
-@@ -9,6 +9,21 @@ __all__ = [
- 'ansible_version',
- ]
-
--ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version)
--del distutils
-+
-+def _parse(v_string):
-+ # Adapted from distutils.version.LooseVersion.parse()
-+ component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
-+ for component in component_re.split(v_string):
-+ if not component or component == '.':
-+ continue
-+ try:
-+ yield int(component)
-+ except ValueError:
-+ yield component
-+
-+
-+ansible_version = tuple(_parse(ansible.__version__))
-+
-+del _parse
-+del re
- del ansible
-diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
-index 64fcd63..5c930b2 100644
---- a/docs/ansible_detailed.rst
-+++ b/docs/ansible_detailed.rst
-@@ -147,8 +147,8 @@ Noteworthy Differences
-
- * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
- Mitogen 0.3.1+ supports
-- - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11
-- - Ansible 5 and 6; with Python 3.8-3.11
-+ - Ansible 5; with Python 3.8-3.11
-+ - Ansible 6; with Python 3.8-3.12
- Verify your installation is running one of these versions by checking
- ``ansible --version`` output.
-
-diff --git a/docs/conf.py b/docs/conf.py
-index 54e3a5c..74e8454 100644
---- a/docs/conf.py
-+++ b/docs/conf.py
-@@ -84,10 +84,16 @@ domainrefs = {
- },
- }
-
-+# > ## Official guidance
-+# > Query PyPI’s JSON API to determine where to download files from.
-+# > ## Predictable URLs
-+# > You can use our conveyor service to fetch this file, which exists for
-+# > cases where using the API is impractical or impossible.
-+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
- rst_epilog = """
-
- .. |mitogen_version| replace:: %(VERSION)s
-
--.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__
-+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
-
- """ % locals()
-diff --git a/mitogen/core.py b/mitogen/core.py
-index 865f826..d92c47f 100644
---- a/mitogen/core.py
-+++ b/mitogen/core.py
-@@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the
- bootstrap implementation sent to every new slave context.
- """
-
-+import sys
-+try:
-+ import _frozen_importlib_external
-+except ImportError:
-+ pass
-+else:
-+ class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder):
-+ """
-+ Meta path finder for sys.path and package __path__ attributes.
-+
-+ Patched for https://github.com/python/cpython/issues/115911.
-+ """
-+ @classmethod
-+ def _path_importer_cache(cls, path):
-+ if path == '':
-+ try:
-+ path = _frozen_importlib_external._os.getcwd()
-+ except (FileNotFoundError, PermissionError):
-+ return None
-+ return super()._path_importer_cache(path)
-+
-+ if sys.version_info[:2] <= (3, 12):
-+ for i, mpf in enumerate(sys.meta_path):
-+ if mpf is _frozen_importlib_external.PathFinder:
-+ sys.meta_path[i] = MonkeyPatchedPathFinder
-+ del i, mpf
-+
-+
- import binascii
- import collections
- import encodings.latin_1
-@@ -49,18 +77,22 @@ import pstats
- import signal
- import socket
- import struct
--import sys
- import syslog
- import threading
- import time
- import traceback
-+import types
- import warnings
- import weakref
- import zlib
-
--# Python >3.7 deprecated the imp module.
--warnings.filterwarnings('ignore', message='the imp module is deprecated')
--import imp
-+try:
-+ # Python >= 3.4, PEP 451 ModuleSpec API
-+ import importlib.machinery
-+ import importlib.util
-+except ImportError:
-+ # Python < 3.4, PEP 302 Import Hooks
-+ import imp
-
- # Absolute imports for <2.5.
- select = __import__('select')
-@@ -1358,6 +1390,19 @@ class Importer(object):
- def __repr__(self):
- return 'Importer'
-
-+ @staticmethod
-+ def _loader_from_module(module, default=None):
-+ """Return the loader for a module object."""
-+ try:
-+ return module.__spec__.loader
-+ except AttributeError:
-+ pass
-+ try:
-+ return module.__loader__
-+ except AttributeError:
-+ pass
-+ return default
-+
- def builtin_find_module(self, fullname):
- # imp.find_module() will always succeed for __main__, because it is a
- # built-in module. That means it exists on a special linked list deep
-@@ -1365,12 +1410,19 @@ class Importer(object):
- if fullname == '__main__':
- raise ModuleNotFoundError()
-
-+ # For a module inside a package (e.g. pkg_a.mod_b) use the search path
-+ # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']).
- parent, _, modname = str_rpartition(fullname, '.')
- if parent:
- path = sys.modules[parent].__path__
- else:
- path = None
-
-+ # For a top-level module search builtin modules, frozen modules,
-+ # system specific locations (e.g. Windows registry, site-packages).
-+ # Otherwise use search path of the parent package.
-+ # Works for both stdlib modules & third-party modules.
-+ # If the search is unsuccessful then raises ImportError.
- fp, pathname, description = imp.find_module(modname, path)
- if fp:
- fp.close()
-@@ -1382,8 +1434,9 @@ class Importer(object):
- Implements importlib.abc.MetaPathFinder.find_module().
- Deprecrated in Python 3.4+, replaced by find_spec().
- Raises ImportWarning in Python 3.10+.
-+ Removed in Python 3.12.
-
-- fullname A (fully qualified?) module name, e.g. "os.path".
-+ fullname Fully qualified module name, e.g. "os.path".
- path __path__ of parent packge. None for a top level module.
- """
- if hasattr(_tls, 'running'):
-@@ -1393,14 +1446,13 @@ class Importer(object):
- try:
- #_v and self._log.debug('Python requested %r', fullname)
- fullname = to_text(fullname)
-- pkgname, dot, _ = str_rpartition(fullname, '.')
-+ pkgname, _, suffix = str_rpartition(fullname, '.')
- pkg = sys.modules.get(pkgname)
- if pkgname and getattr(pkg, '__loader__', None) is not self:
- self._log.debug('%s is submodule of a locally loaded package',
- fullname)
- return None
-
-- suffix = fullname[len(pkgname+dot):]
- if pkgname and suffix not in self._present.get(pkgname, ()):
- self._log.debug('%s has no submodule %s', pkgname, suffix)
- return None
-@@ -1420,6 +1472,66 @@ class Importer(object):
- finally:
- del _tls.running
-
-+ def find_spec(self, fullname, path, target=None):
-+ """
-+ Return a `ModuleSpec` for module with `fullname` if we will load it.
-+ Otherwise return `None`, allowing other finders to try.
-+
-+ fullname Fully qualified name of the module (e.g. foo.bar.baz)
-+ path Path entries to search. None for a top-level module.
-+ target Existing module to be reloaded (if any).
-+
-+ Implements importlib.abc.MetaPathFinder.find_spec()
-+ Python 3.4+.
-+ """
-+ # Presence of _tls.running indicates we've re-invoked importlib.
-+ # Abort early to prevent infinite recursion. See below.
-+ if hasattr(_tls, 'running'):
-+ return None
-+
-+ log = self._log.getChild('find_spec')
-+
-+ if fullname.endswith('.'):
-+ return None
-+
-+ pkgname, _, modname = fullname.rpartition('.')
-+ if pkgname and modname not in self._present.get(pkgname, ()):
-+ log.debug('Skipping %s. Parent %s has no submodule %s',
-+ fullname, pkgname, modname)
-+ return None
-+
-+ pkg = sys.modules.get(pkgname)
-+ pkg_loader = self._loader_from_module(pkg)
-+ if pkgname and pkg_loader is not self:
-+ log.debug('Skipping %s. Parent %s was loaded by %r',
-+ fullname, pkgname, pkg_loader)
-+ return None
-+
-+ # #114: whitelisted prefixes override any system-installed package.
-+ if self.whitelist != ['']:
-+ if any(s and fullname.startswith(s) for s in self.whitelist):
-+ log.debug('Handling %s. It is whitelisted', fullname)
-+ return importlib.machinery.ModuleSpec(fullname, loader=self)
-+
-+ if fullname == '__main__':
-+ log.debug('Handling %s. A special case', fullname)
-+ return importlib.machinery.ModuleSpec(fullname, loader=self)
-+
-+ # Re-invoke the import machinery to allow other finders to try.
-+ # Set a guard, so we don't infinitely recurse. See top of this method.
-+ _tls.running = True
-+ try:
-+ spec = importlib.util._find_spec(fullname, path, target)
-+ finally:
-+ del _tls.running
-+
-+ if spec:
-+ log.debug('Skipping %s. Available as %r', fullname, spec)
-+ return spec
-+
-+ log.debug('Handling %s. Unavailable locally', fullname)
-+ return importlib.machinery.ModuleSpec(fullname, loader=self)
-+
- blacklisted_msg = (
- '%r is present in the Mitogen importer blacklist, therefore this '
- 'context will not attempt to request it from the master, as the '
-@@ -1506,6 +1618,64 @@ class Importer(object):
- if present:
- callback()
-
-+ def create_module(self, spec):
-+ """
-+ Return a module object for the given ModuleSpec.
-+
-+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
-+ Unlike Loader.load_module() this shouldn't populate sys.modules or
-+ set module attributes. Both are done by Python.
-+ """
-+ self._log.debug('Creating module for %r', spec)
-+
-+ # FIXME Should this be done in find_spec()? Can it?
-+ self._refuse_imports(spec.name)
-+
-+ # FIXME "create_module() should properly handle the case where it is
-+ # called more than once for the same spec/module." -- PEP-451
-+ event = threading.Event()
-+ self._request_module(spec.name, callback=event.set)
-+ event.wait()
-+
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ _, pkg_present, path, _, _ = self._cache[spec.name]
-+
-+ if path is None:
-+ raise ImportError(self.absent_msg % (spec.name))
-+
-+ spec.origin = self.get_filename(spec.name)
-+ if pkg_present is not None:
-+ # TODO Namespace packages
-+ spec.submodule_search_locations = []
-+ self._present[spec.name] = pkg_present
-+
-+ module = types.ModuleType(spec.name)
-+ # FIXME create_module() shouldn't initialise module attributes
-+ module.__file__ = spec.origin
-+ return module
-+
-+ def exec_module(self, module):
-+ """
-+ Execute the module to initialise it. Don't return anything.
-+
-+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
-+ """
-+ name = module.__spec__.name
-+ origin = module.__spec__.origin
-+ self._log.debug('Executing %s from %s', name, origin)
-+ source = self.get_source(name)
-+ try:
-+ # Compile the source into a code object. Don't add any __future__
-+ # flags and don't inherit any from this module.
-+ # FIXME Should probably be exposed as get_code()
-+ code = compile(source, origin, 'exec', flags=0, dont_inherit=True)
-+ except SyntaxError:
-+ # FIXME Why is this LOG, rather than self._log?
-+ LOG.exception('while importing %r', name)
-+ raise
-+
-+ exec(code, module.__dict__)
-+
- def load_module(self, fullname):
- """
- Return the loaded module specified by fullname.
-@@ -1521,11 +1691,11 @@ class Importer(object):
- self._request_module(fullname, event.set)
- event.wait()
-
-- ret = self._cache[fullname]
-- if ret[2] is None:
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ _, pkg_present, path, _, _ = self._cache[fullname]
-+ if path is None:
- raise ModuleNotFoundError(self.absent_msg % (fullname,))
-
-- pkg_present = ret[1]
- mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
- mod.__file__ = self.get_filename(fullname)
- mod.__loader__ = self
-@@ -3926,7 +4096,7 @@ class ExternalContext(object):
-
- def _setup_package(self):
- global mitogen
-- mitogen = imp.new_module('mitogen')
-+ mitogen = types.ModuleType('mitogen')
- mitogen.__package__ = 'mitogen'
- mitogen.__path__ = []
- mitogen.__loader__ = self.importer
-diff --git a/mitogen/master.py b/mitogen/master.py
-index 4fb535f..b1e0a1d 100644
---- a/mitogen/master.py
-+++ b/mitogen/master.py
-@@ -37,7 +37,6 @@ contexts.
-
- import dis
- import errno
--import imp
- import inspect
- import itertools
- import logging
-@@ -50,6 +49,16 @@ import threading
- import types
- import zlib
-
-+try:
-+ # Python >= 3.4, PEP 451 ModuleSpec API
-+ import importlib.machinery
-+ import importlib.util
-+ from _imp import is_builtin as _is_builtin
-+except ImportError:
-+ # Python < 3.4, PEP 302 Import Hooks
-+ import imp
-+ from imp import is_builtin as _is_builtin
-+
- try:
- import sysconfig
- except ImportError:
-@@ -122,14 +131,16 @@ def is_stdlib_name(modname):
- """
- Return :data:`True` if `modname` appears to come from the standard library.
- """
-- # `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
-+ # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
-+ # Returns 1 if modname names a module that is "builtin" to the the Python
-+ # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces').
- #
- # """
- # Main is a little special - imp.is_builtin("__main__") will return False,
- # but BuiltinImporter is still the most appropriate initial setting for
- # its __loader__ attribute.
- # """ -- comment in CPython pylifecycle.c:add_main_module()
-- if imp.is_builtin(modname) != 0:
-+ if _is_builtin(modname) != 0:
- return True
-
- module = sys.modules.get(modname)
-@@ -460,6 +471,9 @@ class FinderMethod(object):
- name according to the running Python interpreter. You'd think this was a
- simple task, right? Naive young fellow, welcome to the real world.
- """
-+ def __init__(self):
-+ self.log = LOG.getChild(self.__class__.__name__)
-+
- def __repr__(self):
- return '%s()' % (type(self).__name__,)
-
-@@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod):
- return path, source, is_pkg
-
-
--class ParentEnumerationMethod(FinderMethod):
-+class ParentImpEnumerationMethod(FinderMethod):
- """
- Attempt to fetch source code by examining the module's (hopefully less
- insane) parent package, and if no insane parents exist, simply use
-@@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod):
- def _find_one_component(self, modname, search_path):
- try:
- #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
-+ # FIXME The imp module was removed in Python 3.12.
- return imp.find_module(modname, search_path)
- except ImportError:
- e = sys.exc_info()[1]
-@@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod):
- """
- See implementation for a description of how this works.
- """
-+ if sys.version_info >= (3, 4):
-+ return None
-+
- #if fullname not in sys.modules:
- # Don't attempt this unless a module really exists in sys.modules,
- # else we could return junk.
-@@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod):
- return self._found_module(fullname, path, fp)
-
-
-+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
-+ def _find_parent_spec(self, fullname):
-+ #history = []
-+ debug = self.log.debug
-+ children = []
-+ for parent_name, child_name in self._iter_parents(fullname):
-+ children.insert(0, child_name)
-+ if not parent_name:
-+ debug('abandoning %r, reached top-level', fullname)
-+ return None, children
-+
-+ try:
-+ parent = sys.modules[parent_name]
-+ except KeyError:
-+ debug('skipping %r, not in sys.modules', parent_name)
-+ continue
-+
-+ try:
-+ spec = parent.__spec__
-+ except AttributeError:
-+ debug('skipping %r: %r.__spec__ is absent',
-+ parent_name, parent)
-+ continue
-+
-+ if not spec:
-+ debug('skipping %r: %r.__spec__=%r',
-+ parent_name, parent, spec)
-+ continue
-+
-+ if spec.name != parent_name:
-+ debug('skipping %r: %r.__spec__.name=%r does not match',
-+ parent_name, parent, spec.name)
-+ continue
-+
-+ if not spec.submodule_search_locations:
-+ debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
-+ parent_name, parent, spec.submodule_search_locations)
-+ continue
-+
-+ return spec, children
-+
-+ raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
-+ % (self.__class__.__name__, fullname))
-+
-+ def find(self, fullname):
-+ # Returns absolute path, ParentImpEnumerationMethod returns relative
-+ # >>> spec_pem.find('six_brokenpkg._six')[::2]
-+ # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
-+
-+ if sys.version_info < (3, 4):
-+ return None
-+
-+ fullname = to_text(fullname)
-+ spec, children = self._find_parent_spec(fullname)
-+ for child_name in children:
-+ if spec:
-+ name = '%s.%s' % (spec.name, child_name)
-+ submodule_search_locations = spec.submodule_search_locations
-+ else:
-+ name = child_name
-+ submodule_search_locations = None
-+ spec = importlib.util._find_spec(name, submodule_search_locations)
-+ if spec is None:
-+ self.log.debug('%r spec unavailable from %s', fullname, spec)
-+ return None
-+
-+ is_package = spec.submodule_search_locations is not None
-+ if name != fullname:
-+ if not is_package:
-+ self.log.debug('%r appears to be child of non-package %r',
-+ fullname, spec)
-+ return None
-+ continue
-+
-+ if not spec.has_location:
-+ self.log.debug('%r.origin cannot be read as a file', spec)
-+ return None
-+
-+ if os.path.splitext(spec.origin)[1] != '.py':
-+ self.log.debug('%r.origin does not contain Python source code',
-+ spec)
-+ return None
-+
-+ # FIXME This should use loader.get_source()
-+ with open(spec.origin, 'rb') as f:
-+ source = f.read()
-+
-+ return spec.origin, source, is_package
-+
-+ raise ValueError('%s.find(%r) unexpectedly reached bottom'
-+ % (self.__class__.__name__, fullname))
-+
-+
- class ModuleFinder(object):
- """
- Given the name of a loaded module, make a best-effort attempt at finding
-@@ -838,7 +949,8 @@ class ModuleFinder(object):
- DefectivePython3xMainMethod(),
- PkgutilMethod(),
- SysModulesMethod(),
-- ParentEnumerationMethod(),
-+ ParentSpecEnumerationMethod(),
-+ ParentImpEnumerationMethod(),
- ]
-
- def get_module_source(self, fullname):
-diff --git a/mitogen/parent.py b/mitogen/parent.py
-index 32aa3cb..fe3e8d1 100644
---- a/mitogen/parent.py
-+++ b/mitogen/parent.py
-@@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive
- connection.
- """
-
--import codecs
-+import binascii
- import errno
- import fcntl
- import getpass
-@@ -1405,10 +1405,14 @@ class Connection(object):
- # file descriptor 0 as 100, creates a pipe, then execs a new interpreter
- # with a custom argv.
- # * Optimized for minimum byte count after minification & compression.
-+ # The script preamble_size.py measures this.
- # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with
- # their respective values.
- # * CONTEXT_NAME must be prefixed with the name of the Python binary in
- # order to allow virtualenvs to detect their install prefix.
-+ #
-+ # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible
-+ # module test_echo_module, used by the integration tests.
- # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
- # switcher as /usr/bin/python, which introspects argv0. To workaround
- # it we redirect attempts to call /usr/bin/python with an explicit
-@@ -1417,7 +1421,8 @@ class Connection(object):
- # do something slightly different. The Python executable is patched to
- # perform an extra execvp(). I don't fully understand the details, but
- # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
-- # * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
-+ # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python.
-+ # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python
- #
- # Locals:
- # R: read side of interpreter stdin.
-@@ -1445,7 +1450,7 @@ class Connection(object):
- os.environ['ARGV0']=sys.executable
- os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
- os.write(1,'MITO000\n'.encode())
-- C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
-+ C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN))
- fp=os.fdopen(W,'wb',0)
- fp.write(C)
- fp.close()
-@@ -1477,16 +1482,16 @@ class Connection(object):
- source = source.replace('PREAMBLE_COMPRESSED_LEN',
- str(len(preamble_compressed)))
- compressed = zlib.compress(source.encode(), 9)
-- encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b(''))
-- # We can't use bytes.decode() in 3.x since it was restricted to always
-- # return unicode, so codecs.decode() is used instead. In 3.x
-- # codecs.decode() requires a bytes object. Since we must be compatible
-- # with 2.4 (no bytes literal), an extra .encode() either returns the
-- # same str (2.x) or an equivalent bytes (3.x).
-+ encoded = binascii.b2a_base64(compressed).replace(b('\n'), b(''))
-+
-+ # Just enough to decode, decompress, and exec the first stage.
-+ # Priorities: wider compatibility, faster startup, shorter length.
-+ # `import os` here, instead of stage 1, to save a few bytes.
-+ # `sys.path=...` for https://github.com/python/cpython/issues/115911.
- return self.get_python_argv() + [
- '-c',
-- 'import codecs,os,sys;_=codecs.decode;'
-- 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
-+ 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;'
-+ 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),),
- ]
-
- def get_econtext_config(self):
-diff --git a/setup.py b/setup.py
-index 4d7fadf..b17dab9 100644
---- a/setup.py
-+++ b/setup.py
-@@ -78,6 +78,7 @@ setup(
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
- 'Programming Language :: Python :: 3.11',
-+ 'Programming Language :: Python :: 3.12',
- 'Programming Language :: Python :: Implementation :: CPython',
- 'Topic :: System :: Distributed Computing',
- 'Topic :: System :: Systems Administration',
-diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show
-new file mode 100755
-index 0000000..42ef319
---- /dev/null
-+++ b/tests/ansible/files/cwd_show
-@@ -0,0 +1,22 @@
-+#!/usr/bin/env bash
-+
-+# Show permissions and identities that impact the current working directory.
-+# On macOS libc cwd() can return EACCES after su or sudo.
-+# See also
-+# - https://github.com/ansible/ansible/pull/7078
-+# - https://github.com/python/cpython/issues/115911
-+
-+set -o errexit
-+set -o nounset
-+set -o pipefail
-+
-+whoami
-+groups
-+pwd
-+
-+d=$(pwd)
-+while [[ "$d" != "/" && -n "$d" ]]; do
-+ ls -ld "$d"
-+ d=$(dirname "$d")
-+done
-+ls -ld /
-diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts
-index a18b90d..ac744ed 100644
---- a/tests/ansible/hosts/become_same_user.hosts
-+++ b/tests/ansible/hosts/become_same_user.hosts
-@@ -1,3 +1,5 @@
-+# code: language=ini
-+# vim: syntax=dosini
-
- # become_same_user.yml
- bsu-joe ansible_user=joe
-diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts
-index a22bd5d..4ae861b 100644
---- a/tests/ansible/hosts/connection_delegation.hosts
-+++ b/tests/ansible/hosts/connection_delegation.hosts
-@@ -1,3 +1,4 @@
-+# code: language=ini
- # vim: syntax=dosini
-
- # Connection delegation scenarios. It's impossible to connect to them, but their would-be
-diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts
-index 7ef231b..4f5ea4c 100644
---- a/tests/ansible/hosts/default.hosts
-+++ b/tests/ansible/hosts/default.hosts
-@@ -1,9 +1,12 @@
-+# code: language=ini
- # vim: syntax=dosini
-
- # When running the tests outside CI, make a single 'target' host which is the
- # local machine. The ansible_user override is necessary since some tests want a
- # fixed ansible.cfg remote_user setting to test against.
--target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}"
-+# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022
-+# and os.environ['USER'] is not populated on Azure macOS runners.
-+target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
-
- [test-targets]
- target
-diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts
-index 34e1ff9..b210164 100644
---- a/tests/ansible/hosts/k3.hosts
-+++ b/tests/ansible/hosts/k3.hosts
-@@ -1,3 +1,4 @@
-+# code: language=ini
- # vim: syntax=dosini
-
- # Used for manual testing.
-diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts
-index 41af412..e42221e 100644
---- a/tests/ansible/hosts/localhost.hosts
-+++ b/tests/ansible/hosts/localhost.hosts
-@@ -1,3 +1,4 @@
-+# code: language=ini
- # vim: syntax=dosini
-
- # issue #511, #536: we must not define an explicit localhost, as some
-diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts
-index d6a15c9..1c1c2e1 100644
---- a/tests/ansible/hosts/transport_config.hosts
-+++ b/tests/ansible/hosts/transport_config.hosts
-@@ -1,3 +1,6 @@
-+# code: language=ini
-+# vim: syntax=dosini
-+
- # integration/transport_config
- # Hosts with twiddled configs that need to be checked somehow.
-
-@@ -17,8 +20,9 @@ tc_remote_user
- tc_transport
-
- [transport_config_undiscover:vars]
--# If python interpreter path is unset, Ansible tries to connect & discover it.
--# That causes approx 10 seconds timeout per task - there's no host to connect to.
-+# If ansible_*_interpreter isn't set Ansible tries to connect & discover it.
-+# If that target doesn't exist we must wait $timeout seconds for each attempt.
-+# Setting a known (invalid) interpreter skips discovery & the many timeouts.
- # This optimisation should not be relied in any test.
- # Note: tc-python-path-* are intentionally not included.
- ansible_python_interpreter = python3000
-diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml
-index bd6a0ae..52d420d 100644
---- a/tests/ansible/integration/become/su_password.yml
-+++ b/tests/ansible/integration/become/su_password.yml
-@@ -1,5 +1,4 @@
- # Verify passwordful su behaviour
-- # Ansible can't handle this on OS X. I don't care why.
-
- - name: integration/become/su_password.yml
- hosts: test-targets
-@@ -44,20 +43,54 @@
- fail_msg: out={{out}}
- when: is_mitogen
-
-- - name: Ensure password su succeeds.
-+ - name: Ensure password su with chdir succeeds
- shell: whoami
-+ args:
-+ chdir: ~mitogen__user1
- become: true
- become_user: mitogen__user1
- register: out
- vars:
- ansible_become_pass: user1_password
-- when: is_mitogen
-+ when:
-+ # https://github.com/ansible/ansible/pull/70785
-+ - ansible_facts.distribution not in ["MacOSX"]
-+ or ansible_version.full is version("2.11", ">=", strict=True)
-+ or is_mitogen
-
- - assert:
- that:
- - out.stdout == 'mitogen__user1'
- fail_msg: out={{out}}
-- when: is_mitogen
-+ when:
-+ # https://github.com/ansible/ansible/pull/70785
-+ - ansible_facts.distribution not in ["MacOSX"]
-+ or ansible_version.full is version("2.11", ">=", strict=True)
-+ or is_mitogen
-+
-+ - name: Ensure password su without chdir succeeds
-+ shell: whoami
-+ become: true
-+ become_user: mitogen__user1
-+ register: out
-+ vars:
-+ ansible_become_pass: user1_password
-+ when:
-+ # https://github.com/ansible/ansible/pull/70785
-+ - ansible_facts.distribution not in ["MacOSX"]
-+ or ansible_version.full is version("2.11", ">=", strict=True)
-+ or is_mitogen
-+
-+ - assert:
-+ that:
-+ - out.stdout == 'mitogen__user1'
-+ fail_msg: out={{out}}
-+ when:
-+ # https://github.com/ansible/ansible/pull/70785
-+ - ansible_facts.distribution not in ["MacOSX"]
-+ or ansible_version.full is version("2.11", ">=", strict=True)
-+ or is_mitogen
-+
- tags:
- - su
- - su_password
-diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
-index 36de165..a2d5398 100644
---- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml
-+++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
-@@ -41,7 +41,7 @@
- 'keepalive_count': 10,
- 'password': null,
- 'port': null,
-- 'python_path': ["/usr/bin/python"],
-+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
- 'remote_name': null,
- 'ssh_args': [
- '-o',
-@@ -71,7 +71,7 @@
- 'keepalive_count': 10,
- 'password': null,
- 'port': null,
-- 'python_path': ["/usr/bin/python"],
-+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
- 'remote_name': null,
- 'ssh_args': [
- '-o',
-diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
-index ea828aa..6ac9bad 100644
---- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
-+++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
-@@ -37,6 +37,7 @@
- vars:
- ansible_python_interpreter: auto
- test_echo_module:
-+ facts_copy: "{{ ansible_facts }}"
- register: echoout
-
- # can't test this assertion:
-@@ -44,11 +45,24 @@
- # because Mitogen's ansible_python_interpreter is a connection-layer configurable that
- # "must be extracted during each task execution to form the complete connection-layer configuration".
- # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered
-- - assert:
-+ - name: assert discovered python matches invoked python
-+ assert:
- that:
- - auto_out.ansible_facts.discovered_interpreter_python is defined
-- - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python
-- fail_msg: auto_out={{auto_out}} echoout={{echoout}}
-+ - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
-+ - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved
-+ fail_msg:
-+ - "auto_out: {{ auto_out }}"
-+ - "echoout: {{ echoout }}"
-+ when:
-+ # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always
-+ # reports the same path. I can't reach via symlinks.
-+ # >>> sys.executable
-+ # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
-+ - is_mitogen
-+ or echoout.running_python.sys.version_info.major != 2
-+ or not (echoout.running_python.sys.platform == "darwin"
-+ and echoout.running_python.platform.release.major == 20)
-
-
- - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result
-@@ -128,7 +142,8 @@
- - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter
- block:
- - test_echo_module:
-- facts:
-+ facts_copy: "{{ ansible_facts }}"
-+ facts_to_override:
- ansible_discovered_interpreter_bogus: from module
- discovered_interpreter_bogus: from_module
- ansible_bogus_interpreter: from_module
-@@ -189,13 +204,6 @@
- - distro == 'ubuntu'
- - distro_version is version('16.04', '>=', strict=True)
-
-- - name: mac assertions
-- assert:
-- that:
-- - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
-- fail_msg: auto_out={{auto_out}}
-- when: os_family == 'Darwin'
--
- always:
- - meta: clear_facts
- when:
-diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml
-index 38d1012..6ffff5f 100644
---- a/tests/ansible/integration/interpreter_discovery/complex_args.yml
-+++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml
-@@ -4,6 +4,10 @@
- - name: integration/interpreter_discovery/complex_args.yml
- hosts: test-targets
- gather_facts: true
-+ environment:
-+ http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}"
-+ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}"
-+ no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}"
- tasks:
- - name: create temp file to source
- file:
-@@ -21,28 +25,24 @@
- # special_python: source /tmp/fake && python
- - name: set python using sourced file
- set_fact:
-- special_python: source /tmp/fake || true && python
-+ # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of).
-+ special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
-
- - name: run get_url with specially-sourced python
- get_url:
-- url: https://google.com
-+ # Plain http for wider Ansible & Python version compatibility
-+ url: http://httpbin.org/get
- dest: "/tmp/"
- mode: 0644
-- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
-- validate_certs: no
- vars:
- ansible_python_interpreter: "{{ special_python }}"
-- environment:
-- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
-- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
-
- - name: run get_url with specially-sourced python including jinja
- get_url:
-- url: https://google.com
-+ # Plain http for wider Ansible & Python version compatibility
-+ url: http://httpbin.org/get
- dest: "/tmp/"
- mode: 0644
-- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
-- validate_certs: no
- vars:
- ansible_python_interpreter: >
- {% if "1" == "1" %}
-@@ -50,8 +50,5 @@
- {% else %}
- python
- {% endif %}
-- environment:
-- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
-- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
- tags:
- - complex_args
-diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
-index 0c620da..0d7cf1b 100644
---- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
-+++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
-@@ -2,6 +2,11 @@
- - name: integration/runner/custom_python_new_style_module.yml
- hosts: test-targets
- tasks:
-+ # FIXME Without Mitogen Ansible often reads stdin before the module.
-+ # Either don't read directly from stdin, or figure out the cause.
-+ - meta: end_play
-+ when: not is_mitogen
-+
- - custom_python_new_style_missing_interpreter:
- foo: true
- with_sequence: start=0 end={{end|default(1)}}
-diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml
-index e2384f8..8435b15 100644
---- a/tests/ansible/integration/runner/custom_python_new_style_module.yml
-+++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml
-@@ -1,7 +1,8 @@
- - name: integration/runner/custom_python_new_style_module.yml
- hosts: test-targets
- tasks:
-- # without Mitogen Ansible 2.10 hangs on this play
-+ # FIXME Without Mitogen Ansible often reads stdin before the module.
-+ # Either don't read directly from stdin, or figure out the cause.
- - meta: end_play
- when: not is_mitogen
-
-diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
-index f2a3eef..ebe34cc 100644
---- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
-+++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
-@@ -1,3 +1,7 @@
-+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
-+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
-+# as it is sent to a target. There are probably very few modules in the wild
-+# that still do this, if any - reload() is a Python 2.x builtin function.
- # issue #555
-
- - name: integration/runner/custom_python_prehistoric_module.yml
-@@ -5,9 +9,11 @@
- tasks:
- - custom_python_prehistoric_module:
- register: out
-+ when: is_mitogen
-
- - assert:
- that: out.ok
- fail_msg: out={{out}}
-+ when: is_mitogen
- tags:
- - custom_python_prehistoric_module
-diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py
-index c7a222e..d2ceaf0 100644
---- a/tests/ansible/lib/modules/custom_python_detect_environment.py
-+++ b/tests/ansible/lib/modules/custom_python_detect_environment.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible new-style Python module. I return details about the Python
- # interpreter I run within.
-
-@@ -25,6 +25,11 @@ except NameError:
- def main():
- module = AnsibleModule(argument_spec={})
- module.exit_json(
-+ fs={
-+ '/tmp': {
-+ 'resolved': os.path.realpath('/tmp'),
-+ },
-+ },
- python={
- 'version': {
- 'full': '%i.%i.%i' % sys.version_info[:3],
-diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py
-index ae1b78c..507e53d 100644
---- a/tests/ansible/lib/modules/custom_python_external_module.py
-+++ b/tests/ansible/lib/modules/custom_python_external_module.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I expect the quote from modules2/module_utils/joker.py.
-
- from ansible.module_utils.basic import AnsibleModule
-diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py
-index be9acb2..95bd0c7 100644
---- a/tests/ansible/lib/modules/custom_python_external_pkg.py
-+++ b/tests/ansible/lib/modules/custom_python_external_pkg.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
-
- from ansible.module_utils.basic import AnsibleModule
- from ansible.module_utils.externalpkg import extmod
-diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py
-index 6164057..846037e 100755
---- a/tests/ansible/lib/modules/custom_python_json_args_module.py
-+++ b/tests/ansible/lib/modules/custom_python_json_args_module.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible Python JSONARGS module. I should receive an encoding string.
-
- json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""
-diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
-index 255e372..1d34232 100644
---- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
-+++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible new-style Python module. I leak state from each invocation
- # into a class variable and a global variable.
-
-diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py
-index 51b7452..9767f85 100644
---- a/tests/ansible/lib/modules/custom_python_modify_environ.py
-+++ b/tests/ansible/lib/modules/custom_python_modify_environ.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible new-style Python module. I modify the process environment and
- # don't clean up after myself.
-
-diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
-index 2e0ef0d..728685f 100644
---- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
-+++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
-@@ -1,6 +1,20 @@
- # I am an Ansible new-style Python module, but I lack an interpreter.
-+# See also custom_python_new_style_module, we should be updated in tandem.
-
-+import io
-+import json
-+import select
-+import signal
- import sys
-+import warnings
-+
-+# Ansible 2.7 changed how new style modules are invoked. It seems that module
-+# parameters are *sometimes* read before the module runs. Modules that try
-+# to read directly from stdin, such as this, are unable to. However it doesn't
-+# always fail, influences seem to include Ansible & Python version. As noted
-+# in ansible.module_utils.basic._load_params() we should probably use that.
-+# I think (medium confidence) I narrowed the inflection (with git bisect) to
-+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
-
- # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980
- # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10
-@@ -8,11 +22,46 @@ import sys
- # from ansible.module_utils.
- # import ansible.module_utils.
-
-+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
-+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
-+SELECT_TIMEOUT = 5.0 # seconds
-+SIGNAL_TIMEOUT = 10 # seconds
-+
-+
-+def fail_json(msg, **kwargs):
-+ kwargs.update(failed=True, msg=msg)
-+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
-+ sys.exit(1)
-+
-+
-+def sigalrm_handler(signum, frame):
-+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
-+
-
- def usage():
- sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
- sys.exit(1)
-
-+
-+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
-+signal.signal(signal.SIGALRM, sigalrm_handler)
-+signal.alarm(SIGNAL_TIMEOUT)
-+
-+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
-+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
-+# Consider buffering, line buffering, `f.read()` vs `f.read1()`.
-+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
-+try:
-+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
-+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
-+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
-+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
-+else:
-+ if not inputs_ready:
-+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
-+ % (SELECT_TIMEOUT,))
-+
-+# Read all data on stdin. May block forever, if EOF is not reached.
- input_json = sys.stdin.read()
-
- print("{")
-diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py
-index 1e7270c..c84d241 100755
---- a/tests/ansible/lib/modules/custom_python_new_style_module.py
-+++ b/tests/ansible/lib/modules/custom_python_new_style_module.py
-@@ -1,16 +1,65 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible new-style Python module. I should receive an encoding string.
-+# See also custom_python_new_style_module, we should be updated in tandem.
-
-+import io
-+import json
-+import select
-+import signal
- import sys
-+import warnings
-+
-+# Ansible 2.7 changed how new style modules are invoked. It seems that module
-+# parameters are *sometimes* read before the module runs. Modules that try
-+# to read directly from stdin, such as this, are unable to. However it doesn't
-+# always fail, influences seem to include Ansible & Python version. As noted
-+# in ansible.module_utils.basic._load_params() we should probably use that.
-+# I think (medium confidence) I narrowed the inflection (with git bisect) to
-+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
-
- # This is the magic marker Ansible looks for:
- # from ansible.module_utils.
-
-+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
-+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
-+SELECT_TIMEOUT = 5.0 # seconds
-+SIGNAL_TIMEOUT = 10 # seconds
-+
-+
-+def fail_json(msg, **kwargs):
-+ kwargs.update(failed=True, msg=msg)
-+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
-+ sys.exit(1)
-+
-+
-+def sigalrm_handler(signum, frame):
-+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
-+
-
- def usage():
- sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
- sys.exit(1)
-
-+
-+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
-+signal.signal(signal.SIGALRM, sigalrm_handler)
-+signal.alarm(SIGNAL_TIMEOUT)
-+
-+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
-+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
-+# Consider buffering, line buffering, `f.read()` vs `f.read1()`.
-+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
-+try:
-+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
-+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
-+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
-+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
-+else:
-+ if not inputs_ready:
-+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
-+ % (SELECT_TIMEOUT,))
-+
-+# Read all data on stdin. May block forever, if EOF is not reached.
- input_json = sys.stdin.read()
-
- print("{")
-diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py
-index c5e264a..d465ac9 100644
---- a/tests/ansible/lib/modules/custom_python_os_getcwd.py
-+++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
- # process environment.
-
-diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py
-index 0cf9774..6139748 100644
---- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py
-+++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py
-@@ -1,4 +1,9 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
-+
-+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
-+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
-+# as it is sent to a target. There are probably very few modules in the wild
-+# that still do this, reload() is a Python 2.x builtin function.
- # issue #555: I'm a module that cutpastes an old hack.
-
- from ansible.module_utils.basic import AnsibleModule
-diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py
-index d6a839a..4a6243d 100644
---- a/tests/ansible/lib/modules/custom_python_run_script.py
-+++ b/tests/ansible/lib/modules/custom_python_run_script.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible new-style Python module. I run the script provided in the
- # parameter.
-
-diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py
-index 03f3b6a..6b3a356 100644
---- a/tests/ansible/lib/modules/custom_python_uses_distro.py
-+++ b/tests/ansible/lib/modules/custom_python_uses_distro.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # issue #590: I am an Ansible new-style Python module that tries to use
- # ansible.module_utils.distro.
-
-diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py
-index f5e3386..23eeeb5 100755
---- a/tests/ansible/lib/modules/custom_python_want_json_module.py
-+++ b/tests/ansible/lib/modules/custom_python_want_json_module.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
-
- import json
-diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
-index 1b80a48..289e966 100644
---- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
-+++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
-
- # I am a module that indirectly depends on glibc cached /etc/resolv.conf state.
-
-diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py
-new file mode 100644
-index 0000000..41cf1c1
---- /dev/null
-+++ b/tests/ansible/lib/modules/module_finder_test.py
-@@ -0,0 +1,12 @@
-+from __future__ import absolute_import, division, print_function
-+__metaclass__ = type
-+
-+import os
-+import sys
-+
-+import ansible.module_utils.external1
-+
-+from ansible.module_utils.externalpkg.extmod import path as epem_path
-+
-+def main():
-+ pass
-diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py
-index 1f71e87..d6a5fb9 100644
---- a/tests/ansible/lib/modules/test_echo_module.py
-+++ b/tests/ansible/lib/modules/test_echo_module.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/python
- # -*- coding: utf-8 -*-
-
- # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-@@ -9,28 +9,61 @@
- from __future__ import absolute_import, division, print_function
- __metaclass__ = type
-
-+import os
- import platform
- import sys
- from ansible.module_utils.basic import AnsibleModule
-
-
- def main():
-- result = dict(changed=False)
--
- module = AnsibleModule(argument_spec=dict(
-- facts=dict(type=dict, default={})
-+ facts_copy=dict(type=dict, default={}),
-+ facts_to_override=dict(type=dict, default={})
- ))
-
-- result['ansible_facts'] = module.params['facts']
- # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info
-- if sys.platform == 'darwin' and sys.executable != '/usr/bin/python':
-- if int(platform.release()[:2]) < 19:
-- sys.executable = sys.executable[:-3]
-- else:
-+ # NB This must be synced with mitogen.parent.Connection.get_boot_command()
-+ platform_release_major = int(platform.release().partition('.')[0])
-+ if sys.modules.get('mitogen') and sys.platform == 'darwin':
-+ if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7':
-+ sys.executable = '/usr/bin/python'
-+ if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7):
- # only for tests to check version of running interpreter -- Mac 10.15+ changed python2
- # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
- sys.executable = "/usr/bin/python"
-- result['running_python_interpreter'] = sys.executable
-+
-+ facts_copy = module.params['facts_copy']
-+ discovered_interpreter_python = facts_copy['discovered_interpreter_python']
-+ result = {
-+ 'changed': False,
-+ 'ansible_facts': module.params['facts_to_override'],
-+ 'discovered_and_running_samefile': os.path.samefile(
-+ os.path.realpath(discovered_interpreter_python),
-+ os.path.realpath(sys.executable),
-+ ),
-+ 'discovered_python': {
-+ 'as_seen': discovered_interpreter_python,
-+ 'resolved': os.path.realpath(discovered_interpreter_python),
-+ },
-+ 'running_python': {
-+ 'platform': {
-+ 'release': {
-+ 'major': platform_release_major,
-+ },
-+ },
-+ 'sys': {
-+ 'executable': {
-+ 'as_seen': sys.executable,
-+ 'resolved': os.path.realpath(sys.executable),
-+ },
-+ 'platform': sys.platform,
-+ 'version_info': {
-+ 'major': sys.version_info[0],
-+ 'minor': sys.version_info[1],
-+ },
-+ },
-+ },
-+ }
-
- module.exit_json(**result)
-
-diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
-index f4c47ab..610eaf3 100644
---- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
-+++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
-@@ -23,9 +23,16 @@
- when:
- - lout.python.version.full is version('2.7', '>=', strict=True)
-
-- - assert:
-+ - name: Check virtualenv was used
-+ # On macOS runners a symlink /tmp -> /private/tmp has been seen
-+ vars:
-+ requested_executable: /tmp/issue_152_virtualenv/bin/python
-+ expected_executables:
-+ - "{{ requested_executable }}"
-+ - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}"
-+ assert:
- that:
-- - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python"
-+ - out.sys_executable in expected_executables
- fail_msg: out={{out}}
- when:
- - lout.python.version.full is version('2.7', '>=', strict=True)
-diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml
-index 2ca6b97..8903494 100644
---- a/tests/ansible/setup/all.yml
-+++ b/tests/ansible/setup/all.yml
-@@ -1 +1,2 @@
--- import_playbook: report.yml
-+- import_playbook: report_controller.yml
-+- import_playbook: report_targets.yml
-diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml
-deleted file mode 100644
-index 450e4fb..0000000
---- a/tests/ansible/setup/report.yml
-+++ /dev/null
-@@ -1,20 +0,0 @@
--- name: Report runtime settings
-- hosts: localhost:test-targets
-- gather_facts: true
-- tasks:
-- - debug: {var: ansible_facts.distribution}
-- - debug: {var: ansible_facts.distribution_major_version}
-- - debug: {var: ansible_facts.distribution_release}
-- - debug: {var: ansible_facts.distribution_version}
-- - debug: {var: ansible_facts.kernel}
-- - debug: {var: ansible_facts.kernel_version}
-- - debug: {var: ansible_facts.os_family}
-- - debug: {var: ansible_facts.osrevision}
-- - debug: {var: ansible_facts.osversion}
-- - debug: {var: ansible_facts.python}
-- - debug: {var: ansible_facts.system}
-- - debug: {var: ansible_forks}
-- - debug: {var: ansible_run_tags}
-- - debug: {var: ansible_skip_tags}
-- - debug: {var: ansible_version.full}
-- - debug: {var: is_mitogen}
-diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml
-new file mode 100644
-index 0000000..d0d5cc1
---- /dev/null
-+++ b/tests/ansible/setup/report_controller.yml
-@@ -0,0 +1,17 @@
-+- name: Report controller parameters
-+ hosts: localhost
-+ gather_facts: false
-+ tasks:
-+ - debug:
-+ msg:
-+ - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('<unset>') }}"
-+ - ${USER}: "{{ lookup('env', 'USER') | default('<unset>') }}"
-+ - $(groups): "{{ lookup('pipe', 'groups') }}"
-+ - $(pwd): "{{ lookup('pipe', 'pwd') }}"
-+ - $(whoami): "{{ lookup('pipe', 'whoami') }}"
-+ - ansible_run_tags: "{{ ansible_run_tags | default('<unset>') }}"
-+ - ansible_playbook_python: "{{ ansible_playbook_python | default('<unset>') }}"
-+ - ansible_skip_tags: "{{ ansible_skip_tags | default('<unset>') }}"
-+ - ansible_version.full: "{{ ansible_version.full | default('<unset>') }}"
-+ - is_mitogen: "{{ is_mitogen | default('<unset>') }}"
-+ - playbook_dir: "{{ playbook_dir | default('<unset>') }}"
-diff --git a/tests/ansible/setup/report_targets.yml b/tests/ansible/setup/report_targets.yml
-new file mode 100644
-index 0000000..5aa6712
---- /dev/null
-+++ b/tests/ansible/setup/report_targets.yml
-@@ -0,0 +1,15 @@
-+- name: Report target facts
-+ hosts: localhost:test-targets
-+ gather_facts: true
-+ tasks:
-+ - debug: {var: ansible_facts.distribution}
-+ - debug: {var: ansible_facts.distribution_major_version}
-+ - debug: {var: ansible_facts.distribution_release}
-+ - debug: {var: ansible_facts.distribution_version}
-+ - debug: {var: ansible_facts.kernel}
-+ - debug: {var: ansible_facts.kernel_version}
-+ - debug: {var: ansible_facts.os_family}
-+ - debug: {var: ansible_facts.osrevision}
-+ - debug: {var: ansible_facts.osversion}
-+ - debug: {var: ansible_facts.python}
-+ - debug: {var: ansible_facts.system}
-diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py
-new file mode 100644
-index 0000000..79e8fdb
---- /dev/null
-+++ b/tests/ansible/tests/module_finder_test.py
-@@ -0,0 +1,80 @@
-+import os.path
-+import sys
-+import textwrap
-+import unittest
-+
-+import ansible_mitogen.module_finder
-+
-+import testlib
-+
-+
-+class ScanFromListTest(testlib.TestCase):
-+ def test_absolute_imports(self):
-+ source = textwrap.dedent('''\
-+ from __future__ import absolute_import
-+ import a; import b.c; from d.e import f; from g import h, i
-+ ''')
-+ code = compile(source, '<str>', 'exec')
-+ self.assertEqual(
-+ list(ansible_mitogen.module_finder.scan_fromlist(code)),
-+ [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')],
-+ )
-+
-+
-+class WalkImportsTest(testlib.TestCase):
-+ def test_absolute_imports(self):
-+ source = textwrap.dedent('''\
-+ from __future__ import absolute_import
-+ import a; import b; import b.c; from b.d import e, f
-+ ''')
-+ code = compile(source, '<str>', 'exec')
-+
-+ self.assertEqual(
-+ list(ansible_mitogen.module_finder.walk_imports(code)),
-+ ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'],
-+ )
-+ self.assertEqual(
-+ list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')),
-+ ['b.c', 'b.d', 'b.d.e', 'b.d.f'],
-+ )
-+
-+
-+class ScanTest(testlib.TestCase):
-+ module_name = 'ansible_module_module_finder_test__this_should_not_matter'
-+ module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py')
-+ search_path = (
-+ 'does_not_exist/module_utils',
-+ testlib.ANSIBLE_MODULE_UTILS_DIR,
-+ )
-+
-+ @staticmethod
-+ def relpath(path):
-+ return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR)
-+
-+ @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable')
-+ def test_importlib_find_spec(self):
-+ scan = ansible_mitogen.module_finder._scan_importlib_find_spec
-+ actual = scan(self.module_name, self.module_path, self.search_path)
-+ self.assertEqual(
-+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
-+ [
-+ ('ansible.module_utils.external1', 'external1.py', False),
-+ ('ansible.module_utils.external2', 'external2.py', False),
-+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
-+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
-+ ],
-+ )
-+
-+ @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred')
-+ def test_imp_find_module(self):
-+ scan = ansible_mitogen.module_finder._scan_imp_find_module
-+ actual = scan(self.module_name, self.module_path, self.search_path)
-+ self.assertEqual(
-+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
-+ [
-+ ('ansible.module_utils.external1', 'external1.py', False),
-+ ('ansible.module_utils.external2', 'external2.py', False),
-+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
-+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
-+ ],
-+ )
-diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml
-index 6224b61..0b6d5e6 100644
---- a/tests/image_prep/_user_accounts.yml
-+++ b/tests/image_prep/_user_accounts.yml
-@@ -4,6 +4,8 @@
- # WARNING: this creates non-privilged accounts with pre-set passwords!
- #
-
-+- import_playbook: ../ansible/setup/report_controller.yml
-+
- - hosts: all
- gather_facts: true
- strategy: mitogen_free
-@@ -37,12 +39,12 @@
-
- normal_users: "{{
- lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
-- }}"
-+ }}"
-
- all_users: "{{
- special_users +
- normal_users
-- }}"
-+ }}"
- tasks:
- - name: Disable non-localhost SSH for Mitogen users
- when: false
-@@ -100,6 +102,7 @@
- with_items: "{{all_users}}"
- copy:
- dest: /var/lib/AccountsService/users/mitogen__{{item}}
-+ mode: u=rw,go=
- content: |
- [User]
- SystemAccount=true
-@@ -108,7 +111,7 @@
- when: ansible_system == 'Linux' and out.stat.exists
- service:
- name: accounts-daemon
-- restarted: true
-+ state: restarted
-
- - name: Readonly homedir for one account
- shell: "chown -R root: ~mitogen__readonly_homedir"
-@@ -117,6 +120,9 @@
- copy:
- dest: ~mitogen__slow_user/.{{item}}
- src: ../data/docker/mitogen__slow_user.profile
-+ owner: mitogen__slow_user
-+ group: mitogen__group
-+ mode: u=rw,go=r
- with_items:
- - bashrc
- - profile
-@@ -125,6 +131,9 @@
- copy:
- dest: ~mitogen__permdenied/.{{item}}
- src: ../data/docker/mitogen__permdenied.profile
-+ owner: mitogen__permdenied
-+ group: mitogen__group
-+ mode: u=rw,go=r
- with_items:
- - bashrc
- - profile
-@@ -136,20 +145,13 @@
- state: directory
- mode: go=
- owner: mitogen__has_sudo_pubkey
-+ group: mitogen__group
- - copy:
- dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
- src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
- mode: go=
- owner: mitogen__has_sudo_pubkey
--
-- - name: Install slow profile for one account
-- block:
-- - copy:
-- dest: ~mitogen__slow_user/.profile
-- src: ../data/docker/mitogen__slow_user.profile
-- - copy:
-- dest: ~mitogen__slow_user/.bashrc
-- src: ../data/docker/mitogen__slow_user.profile
-+ group: mitogen__group
-
- - name: Require a TTY for two accounts
- lineinfile:
-diff --git a/tests/importer_test.py b/tests/importer_test.py
-index e48c02a..e86af8a 100644
---- a/tests/importer_test.py
-+++ b/tests/importer_test.py
-@@ -2,6 +2,7 @@ import sys
- import threading
- import types
- import zlib
-+import unittest
-
- import mock
-
-@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin):
- super(ImporterMixin, self).tearDown()
-
-
-+class InvalidNameTest(ImporterMixin, testlib.TestCase):
-+ modname = 'trailingdot.'
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ response = (modname, None, None, None, None)
-+
-+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+ def test_find_spec_invalid(self):
-+ self.set_get_module_response(self.response)
-+ self.assertEqual(self.importer.find_spec(self.modname, path=None), None)
-+
-+
-+class MissingModuleTest(ImporterMixin, testlib.TestCase):
-+ modname = 'missing'
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ response = (modname, None, None, None, None)
-+
-+ @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
-+ def test_load_module_missing(self):
-+ self.set_get_module_response(self.response)
-+ self.assertRaises(ImportError, self.importer.load_module, self.modname)
-+
-+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+ def test_find_spec_missing(self):
-+ """
-+ Importer should optimistically offer itself as a module loader
-+ when there are no disqualifying criteria.
-+ """
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = self.importer.find_spec(self.modname, path=None)
-+ self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
-+ self.assertEqual(spec.name, self.modname)
-+ self.assertEqual(spec.loader, self.importer)
-+
-+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+ def test_create_module_missing(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ self.assertRaises(ImportError, self.importer.create_module, spec)
-+
-+
-+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
- class LoadModuleTest(ImporterMixin, testlib.TestCase):
- data = zlib.compress(b("data = 1\n\n"))
- path = 'fake_module.py'
-@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
- # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
- response = (modname, None, path, data, [])
-
-- def test_no_such_module(self):
-- self.set_get_module_response(
-- # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-- (self.modname, None, None, None, None)
-- )
-- self.assertRaises(ImportError,
-- lambda: self.importer.load_module(self.modname))
--
- def test_module_added_to_sys_modules(self):
- self.set_get_module_response(self.response)
- mod = self.importer.load_module(self.modname)
-@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
- self.assertIsNone(mod.__package__)
-
-
-+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+class ModuleSpecTest(ImporterMixin, testlib.TestCase):
-+ data = zlib.compress(b("data = 1\n\n"))
-+ path = 'fake_module.py'
-+ modname = 'fake_module'
-+
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ response = (modname, None, path, data, [])
-+
-+ def test_module_attributes(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ mod = self.importer.create_module(spec)
-+ self.assertIsInstance(mod, types.ModuleType)
-+ self.assertEqual(mod.__name__, 'fake_module')
-+ #self.assertFalse(hasattr(mod, '__file__'))
-+
-+
-+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
- class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
- data = zlib.compress(b("data = 1\n\n"))
- path = 'fake_module.py'
-@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
- self.assertEqual(mod.__package__, 'mypkg')
-
-
-+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase):
-+ data = zlib.compress(b("data = 1\n\n"))
-+ path = 'fake_module.py'
-+ modname = 'mypkg.fake_module'
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ response = (modname, None, path, data, [])
-+
-+ def test_module_attributes(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ mod = self.importer.create_module(spec)
-+ self.assertIsInstance(mod, types.ModuleType)
-+ self.assertEqual(mod.__name__, 'mypkg.fake_module')
-+ #self.assertFalse(hasattr(mod, '__file__'))
-+
-+
-+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
- class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
- data = zlib.compress(b("func = lambda: 1\n\n"))
- path = 'fake_pkg/__init__.py'
-@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
- self.assertEqual(mod.func.__module__, self.modname)
-
-
-+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+class PackageSpecTest(ImporterMixin, testlib.TestCase):
-+ data = zlib.compress(b("func = lambda: 1\n\n"))
-+ path = 'fake_pkg/__init__.py'
-+ modname = 'fake_pkg'
-+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-+ response = (modname, [], path, data, [])
-+
-+ def test_module_attributes(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ mod = self.importer.create_module(spec)
-+ self.assertIsInstance(mod, types.ModuleType)
-+ self.assertEqual(mod.__name__, 'fake_pkg')
-+ #self.assertFalse(hasattr(mod, '__file__'))
-+
-+ def test_get_filename(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ _ = self.importer.create_module(spec)
-+ filename = self.importer.get_filename(self.modname)
-+ self.assertEqual('master:fake_pkg/__init__.py', filename)
-+
-+ def test_get_source(self):
-+ import importlib.machinery
-+ self.set_get_module_response(self.response)
-+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
-+ _ = self.importer.create_module(spec)
-+ source = self.importer.get_source(self.modname)
-+ self.assertEqual(source,
-+ mitogen.core.to_text(zlib.decompress(self.data)))
-+
-+
- class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase):
- def initdir(self, caplog):
- self.caplog = caplog
-diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
-index 68ceeb4..072968f 100644
---- a/tests/module_finder_test.py
-+++ b/tests/module_finder_test.py
-@@ -139,9 +139,7 @@ class SysModulesMethodTest(testlib.TestCase):
- self.assertIsNone(tup)
-
-
--class GetModuleViaParentEnumerationTest(testlib.TestCase):
-- klass = mitogen.master.ParentEnumerationMethod
--
-+class ParentEnumerationMixin(object):
- def call(self, fullname):
- return self.klass().find(fullname)
-
-@@ -231,6 +229,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
- self.assertEqual(is_pkg, False)
-
-
-+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4')
-+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
-+ klass = mitogen.master.ParentImpEnumerationMethod
-+
-+
-+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
-+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
-+ klass = mitogen.master.ParentSpecEnumerationMethod
-+
-+
- class ResolveRelPathTest(testlib.TestCase):
- klass = mitogen.master.ModuleFinder
-
-diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt
-new file mode 100644
-index 0000000..bc7f7c2
---- /dev/null
-+++ b/tests/requirements-tox.txt
-@@ -0,0 +1,4 @@
-+tox==3.28; python_version == '2.7'
-+tox==3.28; python_version == '3.6'
-+tox==4.8.0; python_version == '3.7'
-+tox>=4.13.0,~=4.0; python_version >= '3.8'
-diff --git a/tests/requirements.txt b/tests/requirements.txt
-index 9bbfe46..12a1379 100644
---- a/tests/requirements.txt
-+++ b/tests/requirements.txt
-@@ -16,4 +16,7 @@ urllib3[secure]==1.23; python_version < '2.7'
- urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9'
- # Last idna compatible with Python 2.6 was idna 2.7.
- idna==2.7; python_version < '2.7'
--virtualenv==20.10.0
-+
-+virtualenv==20.15.1; python_version == '2.7'
-+virtualenv==20.17.1; python_version == '3.6'
-+virtualenv==20.25.1; python_version >= '3.7'
-diff --git a/tests/testlib.py b/tests/testlib.py
-index da4d46f..8f020b3 100644
---- a/tests/testlib.py
-+++ b/tests/testlib.py
-@@ -47,8 +47,13 @@ except NameError:
-
-
- LOG = logging.getLogger(__name__)
--DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
--MODS_DIR = os.path.join(DATA_DIR, 'importer')
-+
-+TESTS_DIR = os.path.join(os.path.dirname(__file__))
-+ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
-+ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
-+ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules')
-+DATA_DIR = os.path.join(TESTS_DIR, 'data')
-+MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer')
-
- sys.path.append(DATA_DIR)
- sys.path.append(MODS_DIR)
-diff --git a/tox.ini b/tox.ini
-index 3b3cb9f..b3e0eb1 100644
---- a/tox.ini
-+++ b/tox.ini
-@@ -27,6 +27,9 @@
- # ansible == 4.* ansible-core ~= 2.11.0
- # ansible == 5.* ansible-core ~= 2.12.0
- # ansible == 6.* ansible-core ~= 2.13.0
-+# ansible == 7.x ansible-core ~= 2.14.0
-+# ansible == 8.x ansible-core ~= 2.15.0
-+# ansible == 9.x ansible-core ~= 2.16.0
-
- # pip --no-python-version-warning
- # pip --disable-pip-version-check
-@@ -60,8 +63,8 @@ deps =
- ansible2.10: ansible==2.10.7
- ansible3: ansible==3.4.0
- ansible4: ansible==4.10.0
-- ansible5: ansible==5.8.0
-- ansible6: ansible==6.0.0
-+ ansible5: ansible~=5.0
-+ ansible6: ansible~=6.0
- install_command =
- python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
- commands_pre =
-@@ -82,7 +85,6 @@ passenv =
- HOME
- setenv =
- # See also azure-pipelines.yml
-- ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive
- ANSIBLE_STRATEGY = mitogen_linear
- NOCOVERAGE_ERASE = 1
- NOCOVERAGE_REPORT = 1
-@@ -114,10 +116,26 @@ setenv =
- distros_ubuntu1804: DISTROS=ubuntu1804
- distros_ubuntu2004: DISTROS=ubuntu2004
- mode_ansible: MODE=ansible
-+ mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
- mode_debops_common: MODE=debops_common
-+ mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
- mode_mitogen: MODE=mitogen
- strategy_linear: ANSIBLE_STRATEGY=linear
-+allowlist_externals =
-+ # Added: Tox 3.18: Tox 4.0+
-+ *_install.py
-+ *_tests.py
-+ aws
-+ docker
-+ docker-credential-secretservice
-+ echo
-+ gpg2
-+ pass
- whitelist_externals =
-+ # Deprecated: Tox 3.18+; Removed: Tox 4.0
-+ *_install.py
-+ *_tests.py
-+ aws
- docker
- docker-credential-secretservice
- echo
diff --git a/debian/patches/python3 b/debian/patches/python3
index 91a6c3c..3a04255 100644
--- a/debian/patches/python3
+++ b/debian/patches/python3
@@ -187,7 +187,7 @@ index 6573e1b..98e93ec 100644
argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV'))
diff --git a/tests/ssh_test.py b/tests/ssh_test.py
-index 429fdb5..34e3e03 100644
+index 3149fcb..92cc703 100644
--- a/tests/ssh_test.py
+++ b/tests/ssh_test.py
@@ -18,6 +18,7 @@ class StubSshMixin(testlib.RouterMixin):
diff --git a/debian/patches/series b/debian/patches/series
index 1c1ad92..9a29acd 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -4,10 +4,7 @@ skip-failing-tests
skip-django-tests
skip-python2.7-test
poll-poller
-no-subprocess32
ansible-2.14
ask-become-pass
interpreter_python
ansible-2.16
-python-3.11
-python-3.12
diff --git a/debian/patches/skip-django-tests b/debian/patches/skip-django-tests
index e024d01..a6054e6 100644
--- a/debian/patches/skip-django-tests
+++ b/debian/patches/skip-django-tests
@@ -4,19 +4,139 @@ Subject: Skip Django tests
They require Django, and tend to fail for us.
---
- tests/module_finder_test.py | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
+ tests/module_finder_test.py | 125 ++++++++++++++++++++++----------------------
+ 1 file changed, 63 insertions(+), 62 deletions(-)
diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
-index ff18bbc..68ceeb4 100644
+index 67e937e..f5012a8 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
-@@ -296,7 +296,7 @@ class FindRelatedTest(testlib.TestCase):
+@@ -305,65 +305,66 @@ class FindRelatedTest(testlib.TestCase):
self.assertEqual(set(related), self.SIMPLE_EXPECT)
--if sys.version_info > (2, 6):
+-class DjangoMixin(object):
+- WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject')
+-
+- @classmethod
+- def modules_expected_path(cls):
+- if sys.version_info[0:2] < (3, 0):
+- modules_expected_filename = 'modules_expected_py2x.json'
+- elif sys.version_info[0:2] <= (3, 6):
+- modules_expected_filename = 'modules_expected_py3x-legacy.json'
+- elif sys.version_info[0:2] >= (3, 10):
+- modules_expected_filename = 'modules_expected_py3x-new.json'
+- return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename)
+-
+- @classmethod
+- def setUpClass(cls):
+- super(DjangoMixin, cls).setUpClass()
+- sys.path.append(cls.WEBPROJECT_PATH)
+- os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings'
+- with open(cls.modules_expected_path(), 'rb') as f:
+- cls.MODULES_EXPECTED = json.load(f)
+-
+- @classmethod
+- def tearDownClass(cls):
+- sys.path.remove(cls.WEBPROJECT_PATH)
+- del os.environ['DJANGO_SETTINGS_MODULE']
+- super(DjangoMixin, cls).tearDownClass()
+-
+-
+-class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase):
+- maxDiff = None
+-
+- def test_django_db(self):
+- import django.db
+- module_finder = mitogen.master.ModuleFinder()
+- related = module_finder.find_related('django.db')
+- expected = self.MODULES_EXPECTED['find_related']['django.db']
+- self.assertEqual(related, expected)
+-
+- def test_django_db_models(self):
+- import django.db.models
+- module_finder = mitogen.master.ModuleFinder()
+- related = module_finder.find_related('django.db.models')
+- expected = self.MODULES_EXPECTED['find_related']['django.db.models']
+- self.assertEqual(related, expected)
+-
+-
+-class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase):
+- maxDiff = None
+-
+- def test_django_db(self):
+- import django.db
+- module_finder = mitogen.master.ModuleFinder()
+- related = module_finder.find_related_imports('django.db')
+- expected = self.MODULES_EXPECTED['find_related_imports']['django.db']
+- self.assertEqual(related, expected)
+-
+- def test_django_db_models(self):
+- import django.db.models
+- module_finder = mitogen.master.ModuleFinder()
+- related = module_finder.find_related_imports('django.db.models')
+- expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models']
+- self.assertEqual(related, expected)
+if False: # sys.version_info > (2, 6):
- class DjangoMixin(object):
- WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject')
-
++ class DjangoMixin(object):
++ WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject')
++
++ @classmethod
++ def modules_expected_path(cls):
++ if sys.version_info[0:2] < (3, 0):
++ modules_expected_filename = 'modules_expected_py2x.json'
++ elif sys.version_info[0:2] <= (3, 6):
++ modules_expected_filename = 'modules_expected_py3x-legacy.json'
++ elif sys.version_info[0:2] >= (3, 10):
++ modules_expected_filename = 'modules_expected_py3x-new.json'
++ return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename)
++
++ @classmethod
++ def setUpClass(cls):
++ super(DjangoMixin, cls).setUpClass()
++ sys.path.append(cls.WEBPROJECT_PATH)
++ os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings'
++ with open(cls.modules_expected_path(), 'rb') as f:
++ cls.MODULES_EXPECTED = json.load(f)
++
++ @classmethod
++ def tearDownClass(cls):
++ sys.path.remove(cls.WEBPROJECT_PATH)
++ del os.environ['DJANGO_SETTINGS_MODULE']
++ super(DjangoMixin, cls).tearDownClass()
++
++
++ class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase):
++ maxDiff = None
++
++ def test_django_db(self):
++ import django.db
++ module_finder = mitogen.master.ModuleFinder()
++ related = module_finder.find_related('django.db')
++ expected = self.MODULES_EXPECTED['find_related']['django.db']
++ self.assertEqual(related, expected)
++
++ def test_django_db_models(self):
++ import django.db.models
++ module_finder = mitogen.master.ModuleFinder()
++ related = module_finder.find_related('django.db.models')
++ expected = self.MODULES_EXPECTED['find_related']['django.db.models']
++ self.assertEqual(related, expected)
++
++
++ class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase):
++ maxDiff = None
++
++ def test_django_db(self):
++ import django.db
++ module_finder = mitogen.master.ModuleFinder()
++ related = module_finder.find_related_imports('django.db')
++ expected = self.MODULES_EXPECTED['find_related_imports']['django.db']
++ self.assertEqual(related, expected)
++
++ def test_django_db_models(self):
++ import django.db.models
++ module_finder = mitogen.master.ModuleFinder()
++ related = module_finder.find_related_imports('django.db.models')
++ expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models']
++ self.assertEqual(related, expected)
diff --git a/docs/_templates/github.html b/docs/_templates/github.html
index bb2b5ee..e6ed304 100644
--- a/docs/_templates/github.html
+++ b/docs/_templates/github.html
@@ -1,4 +1,4 @@
<p>
<br>
-<a class="github-button" href="https://github.com/dw/mitogen/" data-size="large" data-show-count="true" aria-label="Star dw/mitogen on GitHub">Star</a>
+<a class="github-button" href="https://github.com/mitogen-hq/mitogen/" data-size="large" data-show-count="true" aria-label="Star mitogen on GitHub">Star</a>
</p>
diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
index 449771b..cb83aa7 100644
--- a/docs/ansible_detailed.rst
+++ b/docs/ansible_detailed.rst
@@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged.
.. _Ansible: https://www.ansible.com/
-.. _Bug reports: https://goo.gl/yLKZiJ
+.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose
Overview
@@ -147,8 +147,11 @@ Noteworthy Differences
* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
Mitogen 0.3.1+ supports
- - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10
- - Ansible 5 and 6; with Python 3.8-3.10
+
+ - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11
+ - Ansible 5; with Python 3.8-3.11
+ - Ansible 6; with Python 3.8-3.12
+
Verify your installation is running one of these versions by checking
``ansible --version`` output.
@@ -1291,7 +1294,7 @@ Sample Profiles
---------------
The summaries below may be reproduced using data and scripts maintained in the
-`pcaps branch <https://github.com/dw/mitogen/tree/pcaps/>`_. Traces were
+`pcaps branch <https://github.com/mitogen-hq/mitogen/tree/pcaps/>`_. Traces were
recorded using Ansible 2.5.14.
@@ -1300,7 +1303,7 @@ Trivial Loop: Local Host
This demonstrates Mitogen vs. SSH pipelining to the local machine running
`bench/loop-100-items.yml
-<https://github.com/dw/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_,
+<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_,
executing a simple command 100 times. Most Ansible controller overhead is
isolated, characterizing just module executor and connection layer performance.
Mitogen requires **63x less bandwidth and 5.9x less time**.
@@ -1328,7 +1331,7 @@ File Transfer: UK to France
~~~~~~~~~~~~~~~~~~~~~~~~~~~
`This playbook
-<https://github.com/dw/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_
+<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_
was used to compare file transfer performance over a ~26 ms link. It uses the
``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the
target.
diff --git a/docs/changelog.rst b/docs/changelog.rst
index d9a54bc..1c7441c 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -15,7 +15,18 @@ Release Notes
</style>
To avail of fixes in an unreleased version, please download a ZIP file
-`directly from GitHub <https://github.com/dw/mitogen/>`_.
+`directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
+
+v0.3.5 (2024-03-17)
+-------------------
+
+* :gh:issue:`987` Support Python 3.11
+* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when
+ becoming an unprivileged user with Python 3.x
+* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>`_,
+ required by Python 3.12
+* :gh:issue:`1033` Support Python 3.12
+
v0.3.4 (2023-07-02)
-------------------
@@ -65,7 +76,7 @@ v0.3.0 (2021-10-28)
-------------------
This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+.
-`See here for details <https://github.com/dw/mitogen/pull/715#issuecomment-750697248>`_.
+`See here for details <https://github.com/mitogen-hq/mitogen/pull/715#issuecomment-750697248>`_.
* :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports
* :gh:issue:`770` better check for supported Ansible version
@@ -86,7 +97,7 @@ v0.2.10 (2021-10-28)
* :gh:issue:`756` ssh connections with `check_host_keys='accept'` would
timeout, when using recent OpenSSH client versions.
* :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in
- :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
+ :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
* :gh:issue:`775` Test with Python 3.9
* :gh:issue:`775` Add msvcrt to the default module deny list
* :gh:issue:`847` Removed historic Continuous Integration reverse shell
@@ -173,7 +184,7 @@ Mitogen for Ansible
:linux:man7:`unix` sockets across privilege domains.
* :gh:issue:`467`: an incompatibility running Mitogen under `Molecule
- <https://molecule.readthedocs.io/en/stable/>`_ was resolved.
+ <https://ansible.readthedocs.io/projects/molecule/>`_ was resolved.
* :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of
connections, ``async`` tasks, tasks using custom :mod:`module_utils`,
@@ -1225,9 +1236,8 @@ Core Library
parameter may specify an argument vector prefix rather than a string program
path.
-* :gh:issue:`300`: the broker could crash on
- OS X during shutdown due to scheduled `kqueue
- <https://www.freebsd.org/cgi/man.cgi?query=kqueue>`_ filter changes for
+* :gh:issue:`300`: the broker could crash on OS X during shutdown due to
+ scheduled :freebsd:man2:`kqueue` filter changes for
descriptors that were closed before the IO loop resumes. As a temporary
workaround, kqueue's bulk change feature is not used.
diff --git a/docs/conf.py b/docs/conf.py
index 54e3a5c..fb9974c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,10 +1,8 @@
-import os
import sys
-sys.path.append('..')
sys.path.append('.')
-import mitogen
-VERSION = '%s.%s.%s' % mitogen.__version__
+
+VERSION = '0.3.5'
author = u'Network Genomics'
copyright = u'2021, the Mitogen authors'
@@ -44,15 +42,15 @@ version = VERSION
domainrefs = {
'gh:commit': {
'text': '%s',
- 'url': 'https://github.com/dw/mitogen/commit/%s',
+ 'url': 'https://github.com/mitogen-hq/mitogen/commit/%s',
},
'gh:issue': {
'text': '#%s',
- 'url': 'https://github.com/dw/mitogen/issues/%s',
+ 'url': 'https://github.com/mitogen-hq/mitogen/issues/%s',
},
'gh:pull': {
'text': '#%s',
- 'url': 'https://github.com/dw/mitogen/pull/%s',
+ 'url': 'https://github.com/mitogen-hq/mitogen/pull/%s',
},
'ans:mod': {
'text': '%s module',
@@ -64,30 +62,36 @@ domainrefs = {
},
'freebsd:man2': {
'text': '%s(2)',
- 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s',
+ 'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s',
},
'linux:man1': {
'text': '%s(1)',
- 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html',
+ 'url': 'https://man7.org/linux/man-pages/man1/%s.1.html',
},
'linux:man2': {
'text': '%s(2)',
- 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html',
+ 'url': 'https://man7.org/linux/man-pages/man2/%s.2.html',
},
'linux:man3': {
'text': '%s(3)',
- 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html',
+ 'url': 'https://man7.org/linux/man-pages/man3/%s.3.html',
},
'linux:man7': {
'text': '%s(7)',
- 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html',
+ 'url': 'https://man7.org/linux/man-pages/man7/%s.7.html',
},
}
+# > ## Official guidance
+# > Query PyPI’s JSON API to determine where to download files from.
+# > ## Predictable URLs
+# > You can use our conveyor service to fetch this file, which exists for
+# > cases where using the API is impractical or impossible.
+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
rst_epilog = """
.. |mitogen_version| replace:: %(VERSION)s
-.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__
+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
""" % locals()
diff --git a/docs/contributors.rst b/docs/contributors.rst
index 584c4cd..61a9eb1 100644
--- a/docs/contributors.rst
+++ b/docs/contributors.rst
@@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
<li>randy &mdash; <em>desperate for automation</em></li>
<li>Michael & Vicky Twomey-Lee</li>
<li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+ <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
</ul>
diff --git a/docs/index.rst b/docs/index.rst
index d33cf29..32083db 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or
`Fabric`_ can be built, and while the API is quite friendly and comparable to
`Fabric`_, ultimately it is not intended for direct use by consumer software.
-.. _Salt: https://docs.saltstack.com/en/latest/
+.. _Salt: https://docs.saltproject.io/en/latest/
.. _Ansible: https://docs.ansible.com/
.. _Fabric: https://www.fabfile.org/
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 3c4674f..a2894c8 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,3 +1,6 @@
+docutils<0.18
+Jinja2<3
+MarkupSafe<2.1
Sphinx==2.1.2; python_version > '3.0'
sphinxcontrib-programoutput==0.14; python_version > '3.0'
alabaster==0.7.10; python_version > '3.0'
diff --git a/mitogen/__init__.py b/mitogen/__init__.py
index 8b32cbc..d030b96 100644
--- a/mitogen/__init__.py
+++ b/mitogen/__init__.py
@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
#: Library version as a tuple.
-__version__ = (0, 3, 4)
+__version__ = (0, 3, 5)
#: This is :data:`False` in slave contexts. Previously it was used to prevent
diff --git a/mitogen/core.py b/mitogen/core.py
index bee722e..cd02012 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
@@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the
bootstrap implementation sent to every new slave context.
"""
+import sys
+try:
+ import _frozen_importlib_external
+except ImportError:
+ pass
+else:
+ class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder):
+ """
+ Meta path finder for sys.path and package __path__ attributes.
+
+ Patched for https://github.com/python/cpython/issues/115911.
+ """
+ @classmethod
+ def _path_importer_cache(cls, path):
+ if path == '':
+ try:
+ path = _frozen_importlib_external._os.getcwd()
+ except (FileNotFoundError, PermissionError):
+ return None
+ return super()._path_importer_cache(path)
+
+ if sys.version_info[:2] <= (3, 12):
+ for i, mpf in enumerate(sys.meta_path):
+ if mpf is _frozen_importlib_external.PathFinder:
+ sys.meta_path[i] = MonkeyPatchedPathFinder
+ del i, mpf
+
+
import binascii
import collections
import encodings.latin_1
@@ -49,18 +77,22 @@ import pstats
import signal
import socket
import struct
-import sys
import syslog
import threading
import time
import traceback
+import types
import warnings
import weakref
import zlib
-# Python >3.7 deprecated the imp module.
-warnings.filterwarnings('ignore', message='the imp module is deprecated')
-import imp
+try:
+ # Python >= 3.4, PEP 451 ModuleSpec API
+ import importlib.machinery
+ import importlib.util
+except ImportError:
+ # Python < 3.4, PEP 302 Import Hooks
+ import imp
# Absolute imports for <2.5.
select = __import__('select')
@@ -1353,6 +1385,19 @@ class Importer(object):
def __repr__(self):
return 'Importer'
+ @staticmethod
+ def _loader_from_module(module, default=None):
+ """Return the loader for a module object."""
+ try:
+ return module.__spec__.loader
+ except AttributeError:
+ pass
+ try:
+ return module.__loader__
+ except AttributeError:
+ pass
+ return default
+
def builtin_find_module(self, fullname):
# imp.find_module() will always succeed for __main__, because it is a
# built-in module. That means it exists on a special linked list deep
@@ -1360,12 +1405,19 @@ class Importer(object):
if fullname == '__main__':
raise ModuleNotFoundError()
+ # For a module inside a package (e.g. pkg_a.mod_b) use the search path
+ # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']).
parent, _, modname = str_rpartition(fullname, '.')
if parent:
path = sys.modules[parent].__path__
else:
path = None
+ # For a top-level module search builtin modules, frozen modules,
+ # system specific locations (e.g. Windows registry, site-packages).
+ # Otherwise use search path of the parent package.
+ # Works for both stdlib modules & third-party modules.
+ # If the search is unsuccessful then raises ImportError.
fp, pathname, description = imp.find_module(modname, path)
if fp:
fp.close()
@@ -1377,8 +1429,9 @@ class Importer(object):
Implements importlib.abc.MetaPathFinder.find_module().
Deprecrated in Python 3.4+, replaced by find_spec().
Raises ImportWarning in Python 3.10+.
+ Removed in Python 3.12.
- fullname A (fully qualified?) module name, e.g. "os.path".
+ fullname Fully qualified module name, e.g. "os.path".
path __path__ of parent packge. None for a top level module.
"""
if hasattr(_tls, 'running'):
@@ -1388,14 +1441,13 @@ class Importer(object):
try:
#_v and self._log.debug('Python requested %r', fullname)
fullname = to_text(fullname)
- pkgname, dot, _ = str_rpartition(fullname, '.')
+ pkgname, _, suffix = str_rpartition(fullname, '.')
pkg = sys.modules.get(pkgname)
if pkgname and getattr(pkg, '__loader__', None) is not self:
self._log.debug('%s is submodule of a locally loaded package',
fullname)
return None
- suffix = fullname[len(pkgname+dot):]
if pkgname and suffix not in self._present.get(pkgname, ()):
self._log.debug('%s has no submodule %s', pkgname, suffix)
return None
@@ -1415,6 +1467,66 @@ class Importer(object):
finally:
del _tls.running
+ def find_spec(self, fullname, path, target=None):
+ """
+ Return a `ModuleSpec` for module with `fullname` if we will load it.
+ Otherwise return `None`, allowing other finders to try.
+
+ fullname Fully qualified name of the module (e.g. foo.bar.baz)
+ path Path entries to search. None for a top-level module.
+ target Existing module to be reloaded (if any).
+
+ Implements importlib.abc.MetaPathFinder.find_spec()
+ Python 3.4+.
+ """
+ # Presence of _tls.running indicates we've re-invoked importlib.
+ # Abort early to prevent infinite recursion. See below.
+ if hasattr(_tls, 'running'):
+ return None
+
+ log = self._log.getChild('find_spec')
+
+ if fullname.endswith('.'):
+ return None
+
+ pkgname, _, modname = fullname.rpartition('.')
+ if pkgname and modname not in self._present.get(pkgname, ()):
+ log.debug('Skipping %s. Parent %s has no submodule %s',
+ fullname, pkgname, modname)
+ return None
+
+ pkg = sys.modules.get(pkgname)
+ pkg_loader = self._loader_from_module(pkg)
+ if pkgname and pkg_loader is not self:
+ log.debug('Skipping %s. Parent %s was loaded by %r',
+ fullname, pkgname, pkg_loader)
+ return None
+
+ # #114: whitelisted prefixes override any system-installed package.
+ if self.whitelist != ['']:
+ if any(s and fullname.startswith(s) for s in self.whitelist):
+ log.debug('Handling %s. It is whitelisted', fullname)
+ return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+ if fullname == '__main__':
+ log.debug('Handling %s. A special case', fullname)
+ return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+ # Re-invoke the import machinery to allow other finders to try.
+ # Set a guard, so we don't infinitely recurse. See top of this method.
+ _tls.running = True
+ try:
+ spec = importlib.util._find_spec(fullname, path, target)
+ finally:
+ del _tls.running
+
+ if spec:
+ log.debug('Skipping %s. Available as %r', fullname, spec)
+ return spec
+
+ log.debug('Handling %s. Unavailable locally', fullname)
+ return importlib.machinery.ModuleSpec(fullname, loader=self)
+
blacklisted_msg = (
'%r is present in the Mitogen importer blacklist, therefore this '
'context will not attempt to request it from the master, as the '
@@ -1501,6 +1613,64 @@ class Importer(object):
if present:
callback()
+ def create_module(self, spec):
+ """
+ Return a module object for the given ModuleSpec.
+
+ Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+ Unlike Loader.load_module() this shouldn't populate sys.modules or
+ set module attributes. Both are done by Python.
+ """
+ self._log.debug('Creating module for %r', spec)
+
+ # FIXME Should this be done in find_spec()? Can it?
+ self._refuse_imports(spec.name)
+
+ # FIXME "create_module() should properly handle the case where it is
+ # called more than once for the same spec/module." -- PEP-451
+ event = threading.Event()
+ self._request_module(spec.name, callback=event.set)
+ event.wait()
+
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ _, pkg_present, path, _, _ = self._cache[spec.name]
+
+ if path is None:
+ raise ImportError(self.absent_msg % (spec.name))
+
+ spec.origin = self.get_filename(spec.name)
+ if pkg_present is not None:
+ # TODO Namespace packages
+ spec.submodule_search_locations = []
+ self._present[spec.name] = pkg_present
+
+ module = types.ModuleType(spec.name)
+ # FIXME create_module() shouldn't initialise module attributes
+ module.__file__ = spec.origin
+ return module
+
+ def exec_module(self, module):
+ """
+ Execute the module to initialise it. Don't return anything.
+
+ Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+ """
+ name = module.__spec__.name
+ origin = module.__spec__.origin
+ self._log.debug('Executing %s from %s', name, origin)
+ source = self.get_source(name)
+ try:
+ # Compile the source into a code object. Don't add any __future__
+ # flags and don't inherit any from this module.
+ # FIXME Should probably be exposed as get_code()
+ code = compile(source, origin, 'exec', flags=0, dont_inherit=True)
+ except SyntaxError:
+ # FIXME Why is this LOG, rather than self._log?
+ LOG.exception('while importing %r', name)
+ raise
+
+ exec(code, module.__dict__)
+
def load_module(self, fullname):
"""
Return the loaded module specified by fullname.
@@ -1516,11 +1686,11 @@ class Importer(object):
self._request_module(fullname, event.set)
event.wait()
- ret = self._cache[fullname]
- if ret[2] is None:
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ _, pkg_present, path, _, _ = self._cache[fullname]
+ if path is None:
raise ModuleNotFoundError(self.absent_msg % (fullname,))
- pkg_present = ret[1]
mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
mod.__file__ = self.get_filename(fullname)
mod.__loader__ = self
@@ -3921,7 +4091,7 @@ class ExternalContext(object):
def _setup_package(self):
global mitogen
- mitogen = imp.new_module('mitogen')
+ mitogen = types.ModuleType('mitogen')
mitogen.__package__ = 'mitogen'
mitogen.__path__ = []
mitogen.__loader__ = self.importer
diff --git a/mitogen/master.py b/mitogen/master.py
index 4fb535f..b1e0a1d 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -37,7 +37,6 @@ contexts.
import dis
import errno
-import imp
import inspect
import itertools
import logging
@@ -51,6 +50,16 @@ import types
import zlib
try:
+ # Python >= 3.4, PEP 451 ModuleSpec API
+ import importlib.machinery
+ import importlib.util
+ from _imp import is_builtin as _is_builtin
+except ImportError:
+ # Python < 3.4, PEP 302 Import Hooks
+ import imp
+ from imp import is_builtin as _is_builtin
+
+try:
import sysconfig
except ImportError:
sysconfig = None
@@ -122,14 +131,16 @@ def is_stdlib_name(modname):
"""
Return :data:`True` if `modname` appears to come from the standard library.
"""
- # `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
+ # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
+ # Returns 1 if modname names a module that is "builtin" to the the Python
+ # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces').
#
# """
# Main is a little special - imp.is_builtin("__main__") will return False,
# but BuiltinImporter is still the most appropriate initial setting for
# its __loader__ attribute.
# """ -- comment in CPython pylifecycle.c:add_main_module()
- if imp.is_builtin(modname) != 0:
+ if _is_builtin(modname) != 0:
return True
module = sys.modules.get(modname)
@@ -460,6 +471,9 @@ class FinderMethod(object):
name according to the running Python interpreter. You'd think this was a
simple task, right? Naive young fellow, welcome to the real world.
"""
+ def __init__(self):
+ self.log = LOG.getChild(self.__class__.__name__)
+
def __repr__(self):
return '%s()' % (type(self).__name__,)
@@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod):
return path, source, is_pkg
-class ParentEnumerationMethod(FinderMethod):
+class ParentImpEnumerationMethod(FinderMethod):
"""
Attempt to fetch source code by examining the module's (hopefully less
insane) parent package, and if no insane parents exist, simply use
@@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod):
def _find_one_component(self, modname, search_path):
try:
#fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
+ # FIXME The imp module was removed in Python 3.12.
return imp.find_module(modname, search_path)
except ImportError:
e = sys.exc_info()[1]
@@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod):
"""
See implementation for a description of how this works.
"""
+ if sys.version_info >= (3, 4):
+ return None
+
#if fullname not in sys.modules:
# Don't attempt this unless a module really exists in sys.modules,
# else we could return junk.
@@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod):
return self._found_module(fullname, path, fp)
+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
+ def _find_parent_spec(self, fullname):
+ #history = []
+ debug = self.log.debug
+ children = []
+ for parent_name, child_name in self._iter_parents(fullname):
+ children.insert(0, child_name)
+ if not parent_name:
+ debug('abandoning %r, reached top-level', fullname)
+ return None, children
+
+ try:
+ parent = sys.modules[parent_name]
+ except KeyError:
+ debug('skipping %r, not in sys.modules', parent_name)
+ continue
+
+ try:
+ spec = parent.__spec__
+ except AttributeError:
+ debug('skipping %r: %r.__spec__ is absent',
+ parent_name, parent)
+ continue
+
+ if not spec:
+ debug('skipping %r: %r.__spec__=%r',
+ parent_name, parent, spec)
+ continue
+
+ if spec.name != parent_name:
+ debug('skipping %r: %r.__spec__.name=%r does not match',
+ parent_name, parent, spec.name)
+ continue
+
+ if not spec.submodule_search_locations:
+ debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
+ parent_name, parent, spec.submodule_search_locations)
+ continue
+
+ return spec, children
+
+ raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
+ % (self.__class__.__name__, fullname))
+
+ def find(self, fullname):
+ # Returns absolute path, ParentImpEnumerationMethod returns relative
+ # >>> spec_pem.find('six_brokenpkg._six')[::2]
+ # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
+
+ if sys.version_info < (3, 4):
+ return None
+
+ fullname = to_text(fullname)
+ spec, children = self._find_parent_spec(fullname)
+ for child_name in children:
+ if spec:
+ name = '%s.%s' % (spec.name, child_name)
+ submodule_search_locations = spec.submodule_search_locations
+ else:
+ name = child_name
+ submodule_search_locations = None
+ spec = importlib.util._find_spec(name, submodule_search_locations)
+ if spec is None:
+ self.log.debug('%r spec unavailable from %s', fullname, spec)
+ return None
+
+ is_package = spec.submodule_search_locations is not None
+ if name != fullname:
+ if not is_package:
+ self.log.debug('%r appears to be child of non-package %r',
+ fullname, spec)
+ return None
+ continue
+
+ if not spec.has_location:
+ self.log.debug('%r.origin cannot be read as a file', spec)
+ return None
+
+ if os.path.splitext(spec.origin)[1] != '.py':
+ self.log.debug('%r.origin does not contain Python source code',
+ spec)
+ return None
+
+ # FIXME This should use loader.get_source()
+ with open(spec.origin, 'rb') as f:
+ source = f.read()
+
+ return spec.origin, source, is_package
+
+ raise ValueError('%s.find(%r) unexpectedly reached bottom'
+ % (self.__class__.__name__, fullname))
+
+
class ModuleFinder(object):
"""
Given the name of a loaded module, make a best-effort attempt at finding
@@ -838,7 +949,8 @@ class ModuleFinder(object):
DefectivePython3xMainMethod(),
PkgutilMethod(),
SysModulesMethod(),
- ParentEnumerationMethod(),
+ ParentSpecEnumerationMethod(),
+ ParentImpEnumerationMethod(),
]
def get_module_source(self, fullname):
diff --git a/mitogen/parent.py b/mitogen/parent.py
index 32aa3cb..29bcf66 100644
--- a/mitogen/parent.py
+++ b/mitogen/parent.py
@@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive
connection.
"""
-import codecs
+import binascii
import errno
import fcntl
import getpass
@@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller):
class EpollPoller(mitogen.core.Poller):
"""
- Poller based on the Linux :linux:man2:`epoll` interface.
+ Poller based on the Linux :linux:man7:`epoll` interface.
"""
SUPPORTED = hasattr(select, 'epoll')
_repr = 'EpollPoller()'
@@ -1405,10 +1405,14 @@ class Connection(object):
# file descriptor 0 as 100, creates a pipe, then execs a new interpreter
# with a custom argv.
# * Optimized for minimum byte count after minification & compression.
+ # The script preamble_size.py measures this.
# * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with
# their respective values.
# * CONTEXT_NAME must be prefixed with the name of the Python binary in
# order to allow virtualenvs to detect their install prefix.
+ #
+ # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible
+ # module test_echo_module, used by the integration tests.
# * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
# switcher as /usr/bin/python, which introspects argv0. To workaround
# it we redirect attempts to call /usr/bin/python with an explicit
@@ -1417,7 +1421,8 @@ class Connection(object):
# do something slightly different. The Python executable is patched to
# perform an extra execvp(). I don't fully understand the details, but
# setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
- # * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
+ # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python.
+ # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python
#
# Locals:
# R: read side of interpreter stdin.
@@ -1445,7 +1450,7 @@ class Connection(object):
os.environ['ARGV0']=sys.executable
os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
os.write(1,'MITO000\n'.encode())
- C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
+ C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN))
fp=os.fdopen(W,'wb',0)
fp.write(C)
fp.close()
@@ -1477,16 +1482,16 @@ class Connection(object):
source = source.replace('PREAMBLE_COMPRESSED_LEN',
str(len(preamble_compressed)))
compressed = zlib.compress(source.encode(), 9)
- encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b(''))
- # We can't use bytes.decode() in 3.x since it was restricted to always
- # return unicode, so codecs.decode() is used instead. In 3.x
- # codecs.decode() requires a bytes object. Since we must be compatible
- # with 2.4 (no bytes literal), an extra .encode() either returns the
- # same str (2.x) or an equivalent bytes (3.x).
+ encoded = binascii.b2a_base64(compressed).replace(b('\n'), b(''))
+
+ # Just enough to decode, decompress, and exec the first stage.
+ # Priorities: wider compatibility, faster startup, shorter length.
+ # `import os` here, instead of stage 1, to save a few bytes.
+ # `sys.path=...` for https://github.com/python/cpython/issues/115911.
return self.get_python_argv() + [
'-c',
- 'import codecs,os,sys;_=codecs.decode;'
- 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
+ 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;'
+ 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),),
]
def get_econtext_config(self):
diff --git a/mitogen/service.py b/mitogen/service.py
index 0e5f641..7fde901 100644
--- a/mitogen/service.py
+++ b/mitogen/service.py
@@ -752,10 +752,12 @@ class PushFileService(Service):
One size fits all method to ensure a target context has been preloaded
with a set of small files and Python modules.
- overridden_sources: optional dict containing source code to override path's source code
- extra_sys_paths: loads additional sys paths for use in finding modules; beneficial
- in situations like loading Ansible Collections because source code
- dependencies come from different file paths than where the source lives
+ :param dict overridden_sources:
+ Optional dict containing source code to override path's source code
+ :param extra_sys_paths:
+ Loads additional sys paths for use in finding modules; beneficial
+ in situations like loading Ansible Collections because source code
+ dependencies come from different file paths than where the source lives
"""
for path in paths:
overridden_source = None
diff --git a/setup.py b/setup.py
index 9d2ff36..b17dab9 100644
--- a/setup.py
+++ b/setup.py
@@ -77,6 +77,8 @@ setup(
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11',
+ 'Programming Language :: Python :: 3.12',
'Programming Language :: Python :: Implementation :: CPython',
'Topic :: System :: Distributed Computing',
'Topic :: System :: Systems Administration',
diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg
index ce4511f..c34dd21 100644
--- a/tests/ansible/ansible.cfg
+++ b/tests/ansible/ansible.cfg
@@ -48,5 +48,21 @@ host_pattern_mismatch = error
task_output_limit = 10
[ssh_connection]
-ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s
+# https://www.openssh.com/legacy.html
+# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd.
+# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04).
+# Duplicated cases in
+# - tests/ansible/ansible.cfg
+# - tests/ansible/integration/connection_delegation/delegate_to_template.yml
+# - tests/ansible/integration/connection_delegation/stack_construction.yml
+# - tests/ansible/integration/process/unix_socket_cleanup.yml
+# - tests/ansible/integration/ssh/variables.yml
+# - tests/testlib.py
+ssh_args =
+ -o ControlMaster=auto
+ -o ControlPersist=60s
+ -o ForwardAgent=yes
+ -o HostKeyAlgorithms=+ssh-rsa
+ -o PubkeyAcceptedKeyTypes=+ssh-rsa
+ -o UserKnownHostsFile=/dev/null
pipelining = True
diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show
new file mode 100755
index 0000000..42ef319
--- /dev/null
+++ b/tests/ansible/files/cwd_show
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Show permissions and identities that impact the current working directory.
+# On macOS libc cwd() can return EACCES after su or sudo.
+# See also
+# - https://github.com/ansible/ansible/pull/7078
+# - https://github.com/python/cpython/issues/115911
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+whoami
+groups
+pwd
+
+d=$(pwd)
+while [[ "$d" != "/" && -n "$d" ]]; do
+ ls -ld "$d"
+ d=$(dirname "$d")
+done
+ls -ld /
diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts
index a18b90d..ac744ed 100644
--- a/tests/ansible/hosts/become_same_user.hosts
+++ b/tests/ansible/hosts/become_same_user.hosts
@@ -1,3 +1,5 @@
+# code: language=ini
+# vim: syntax=dosini
# become_same_user.yml
bsu-joe ansible_user=joe
diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts
index a22bd5d..4ae861b 100644
--- a/tests/ansible/hosts/connection_delegation.hosts
+++ b/tests/ansible/hosts/connection_delegation.hosts
@@ -1,3 +1,4 @@
+# code: language=ini
# vim: syntax=dosini
# Connection delegation scenarios. It's impossible to connect to them, but their would-be
diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts
index d40c3dd..1bec001 100644
--- a/tests/ansible/hosts/default.hosts
+++ b/tests/ansible/hosts/default.hosts
@@ -1,9 +1,12 @@
+# code: language=ini
# vim: syntax=dosini
# When running the tests outside CI, make a single 'target' host which is the
# local machine. The ansible_user override is necessary since some tests want a
# fixed ansible.cfg remote_user setting to test against.
-target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}"
+# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022
+# and os.environ['USER'] is not populated on Azure macOS runners.
+target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
[test-targets]
target
diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts
index 34e1ff9..b210164 100644
--- a/tests/ansible/hosts/k3.hosts
+++ b/tests/ansible/hosts/k3.hosts
@@ -1,3 +1,4 @@
+# code: language=ini
# vim: syntax=dosini
# Used for manual testing.
diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts
index 41af412..e42221e 100644
--- a/tests/ansible/hosts/localhost.hosts
+++ b/tests/ansible/hosts/localhost.hosts
@@ -1,3 +1,4 @@
+# code: language=ini
# vim: syntax=dosini
# issue #511, #536: we must not define an explicit localhost, as some
diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts
index 7d7b526..1c1c2e1 100644
--- a/tests/ansible/hosts/transport_config.hosts
+++ b/tests/ansible/hosts/transport_config.hosts
@@ -1,3 +1,6 @@
+# code: language=ini
+# vim: syntax=dosini
+
# integration/transport_config
# Hosts with twiddled configs that need to be checked somehow.
@@ -17,11 +20,12 @@ tc_remote_user
tc_transport
[transport_config_undiscover:vars]
-# If python interpreter path is unset, Ansible tries to connect & discover it.
-# That causes approx 10 seconds timeout per task - there's no host to connect to.
+# If ansible_*_interpreter isn't set Ansible tries to connect & discover it.
+# If that target doesn't exist we must wait $timeout seconds for each attempt.
+# Setting a known (invalid) interpreter skips discovery & the many timeouts.
# This optimisation should not be relied in any test.
# Note: tc-python-path-* are intentionally not included.
-ansible_python_interpreter = python3000 # Not expected to exist
+ansible_python_interpreter = python3000
[tc_transport]
tc-transport-unset
diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml
index bd6a0ae..52d420d 100644
--- a/tests/ansible/integration/become/su_password.yml
+++ b/tests/ansible/integration/become/su_password.yml
@@ -1,5 +1,4 @@
# Verify passwordful su behaviour
- # Ansible can't handle this on OS X. I don't care why.
- name: integration/become/su_password.yml
hosts: test-targets
@@ -44,20 +43,54 @@
fail_msg: out={{out}}
when: is_mitogen
- - name: Ensure password su succeeds.
+ - name: Ensure password su with chdir succeeds
shell: whoami
+ args:
+ chdir: ~mitogen__user1
become: true
become_user: mitogen__user1
register: out
vars:
ansible_become_pass: user1_password
- when: is_mitogen
+ when:
+ # https://github.com/ansible/ansible/pull/70785
+ - ansible_facts.distribution not in ["MacOSX"]
+ or ansible_version.full is version("2.11", ">=", strict=True)
+ or is_mitogen
- assert:
that:
- out.stdout == 'mitogen__user1'
fail_msg: out={{out}}
- when: is_mitogen
+ when:
+ # https://github.com/ansible/ansible/pull/70785
+ - ansible_facts.distribution not in ["MacOSX"]
+ or ansible_version.full is version("2.11", ">=", strict=True)
+ or is_mitogen
+
+ - name: Ensure password su without chdir succeeds
+ shell: whoami
+ become: true
+ become_user: mitogen__user1
+ register: out
+ vars:
+ ansible_become_pass: user1_password
+ when:
+ # https://github.com/ansible/ansible/pull/70785
+ - ansible_facts.distribution not in ["MacOSX"]
+ or ansible_version.full is version("2.11", ">=", strict=True)
+ or is_mitogen
+
+ - assert:
+ that:
+ - out.stdout == 'mitogen__user1'
+ fail_msg: out={{out}}
+ when:
+ # https://github.com/ansible/ansible/pull/70785
+ - ansible_facts.distribution not in ["MacOSX"]
+ or ansible_version.full is version("2.11", ">=", strict=True)
+ or is_mitogen
+
tags:
- su
- su_password
diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
index 36de165..7d33a16 100644
--- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml
+++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml
@@ -41,17 +41,15 @@
'keepalive_count': 10,
'password': null,
'port': null,
- 'python_path': ["/usr/bin/python"],
+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -71,17 +69,15 @@
'keepalive_count': 10,
'password': null,
'port': null,
- 'python_path': ["/usr/bin/python"],
+ 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml
index 380b319..279a7b2 100644
--- a/tests/ansible/integration/connection_delegation/stack_construction.yml
+++ b/tests/ansible/integration/connection_delegation/stack_construction.yml
@@ -81,14 +81,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -126,14 +124,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -182,14 +178,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -227,14 +221,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -257,14 +249,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -313,14 +303,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
@@ -359,14 +347,12 @@
"python_path": ["/usr/bin/python"],
'remote_name': null,
'ssh_args': [
- '-o',
- 'UserKnownHostsFile=/dev/null',
- '-o',
- 'ForwardAgent=yes',
- '-o',
- 'ControlMaster=auto',
- '-o',
- 'ControlPersist=60s',
+ -o, ControlMaster=auto,
+ -o, ControlPersist=60s,
+ -o, ForwardAgent=yes,
+ -o, HostKeyAlgorithms=+ssh-rsa,
+ -o, PubkeyAcceptedKeyTypes=+ssh-rsa,
+ -o, UserKnownHostsFile=/dev/null,
],
'ssh_debug_level': null,
'ssh_path': 'ssh',
diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
index ea828aa..6ac9bad 100644
--- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
+++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml
@@ -37,6 +37,7 @@
vars:
ansible_python_interpreter: auto
test_echo_module:
+ facts_copy: "{{ ansible_facts }}"
register: echoout
# can't test this assertion:
@@ -44,11 +45,24 @@
# because Mitogen's ansible_python_interpreter is a connection-layer configurable that
# "must be extracted during each task execution to form the complete connection-layer configuration".
# Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered
- - assert:
+ - name: assert discovered python matches invoked python
+ assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python is defined
- - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python
- fail_msg: auto_out={{auto_out}} echoout={{echoout}}
+ - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
+ - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved
+ fail_msg:
+ - "auto_out: {{ auto_out }}"
+ - "echoout: {{ echoout }}"
+ when:
+ # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always
+ # reports the same path. I can't reach via symlinks.
+ # >>> sys.executable
+ # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
+ - is_mitogen
+ or echoout.running_python.sys.version_info.major != 2
+ or not (echoout.running_python.sys.platform == "darwin"
+ and echoout.running_python.platform.release.major == 20)
- name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result
@@ -128,7 +142,8 @@
- name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter
block:
- test_echo_module:
- facts:
+ facts_copy: "{{ ansible_facts }}"
+ facts_to_override:
ansible_discovered_interpreter_bogus: from module
discovered_interpreter_bogus: from_module
ansible_bogus_interpreter: from_module
@@ -189,13 +204,6 @@
- distro == 'ubuntu'
- distro_version is version('16.04', '>=', strict=True)
- - name: mac assertions
- assert:
- that:
- - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
- fail_msg: auto_out={{auto_out}}
- when: os_family == 'Darwin'
-
always:
- meta: clear_facts
when:
diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml
index 38d1012..6ffff5f 100644
--- a/tests/ansible/integration/interpreter_discovery/complex_args.yml
+++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml
@@ -4,6 +4,10 @@
- name: integration/interpreter_discovery/complex_args.yml
hosts: test-targets
gather_facts: true
+ environment:
+ http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}"
+ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}"
+ no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}"
tasks:
- name: create temp file to source
file:
@@ -21,28 +25,24 @@
# special_python: source /tmp/fake && python
- name: set python using sourced file
set_fact:
- special_python: source /tmp/fake || true && python
+ # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of).
+ special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
- name: run get_url with specially-sourced python
get_url:
- url: https://google.com
+ # Plain http for wider Ansible & Python version compatibility
+ url: http://httpbin.org/get
dest: "/tmp/"
mode: 0644
- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
- validate_certs: no
vars:
ansible_python_interpreter: "{{ special_python }}"
- environment:
- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
- name: run get_url with specially-sourced python including jinja
get_url:
- url: https://google.com
+ # Plain http for wider Ansible & Python version compatibility
+ url: http://httpbin.org/get
dest: "/tmp/"
mode: 0644
- # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
- validate_certs: no
vars:
ansible_python_interpreter: >
{% if "1" == "1" %}
@@ -50,8 +50,5 @@
{% else %}
python
{% endif %}
- environment:
- https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
- no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
tags:
- complex_args
diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml
index 625ad7b..b5d40b7 100644
--- a/tests/ansible/integration/process/unix_socket_cleanup.yml
+++ b/tests/ansible/integration/process/unix_socket_cleanup.yml
@@ -9,7 +9,7 @@
- shell: >
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -c local -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
index 0c620da..0d7cf1b 100644
--- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
+++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml
@@ -2,6 +2,11 @@
- name: integration/runner/custom_python_new_style_module.yml
hosts: test-targets
tasks:
+ # FIXME Without Mitogen Ansible often reads stdin before the module.
+ # Either don't read directly from stdin, or figure out the cause.
+ - meta: end_play
+ when: not is_mitogen
+
- custom_python_new_style_missing_interpreter:
foo: true
with_sequence: start=0 end={{end|default(1)}}
diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml
index e2384f8..8435b15 100644
--- a/tests/ansible/integration/runner/custom_python_new_style_module.yml
+++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml
@@ -1,7 +1,8 @@
- name: integration/runner/custom_python_new_style_module.yml
hosts: test-targets
tasks:
- # without Mitogen Ansible 2.10 hangs on this play
+ # FIXME Without Mitogen Ansible often reads stdin before the module.
+ # Either don't read directly from stdin, or figure out the cause.
- meta: end_play
when: not is_mitogen
diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
index f2a3eef..ebe34cc 100644
--- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
+++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml
@@ -1,3 +1,7 @@
+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
+# as it is sent to a target. There are probably very few modules in the wild
+# that still do this, if any - reload() is a Python 2.x builtin function.
# issue #555
- name: integration/runner/custom_python_prehistoric_module.yml
@@ -5,9 +9,11 @@
tasks:
- custom_python_prehistoric_module:
register: out
+ when: is_mitogen
- assert:
that: out.ok
fail_msg: out={{out}}
+ when: is_mitogen
tags:
- custom_python_prehistoric_module
diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml
index 8e912b2..6041d30 100644
--- a/tests/ansible/integration/ssh/variables.yml
+++ b/tests/ansible/integration/ssh/variables.yml
@@ -17,7 +17,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -34,7 +34,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -59,7 +59,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -76,7 +76,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -101,7 +101,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -118,7 +118,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -148,7 +148,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
@@ -165,7 +165,7 @@
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
- ANSIBLE_SSH_ARGS=""
+ ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py
index 5923f4b..72264cf 100644
--- a/tests/ansible/lib/action/assert_equal.py
+++ b/tests/ansible/lib/action/assert_equal.py
@@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase
TEMPLATE_KWARGS = {}
-_argspec = inspect.getargspec(ansible.template.Templar.template)
+try:
+ # inspect.getfullargspec() Added: 3.0
+ _argspec = inspect.getfullargspec(ansible.template.Templar.template)
+except AttributeError:
+ # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11
+ _argspec = inspect.getargspec(ansible.template.Templar.template)
if 'bare_deprecated' in _argspec.args:
TEMPLATE_KWARGS['bare_deprecated'] = False
diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py
index c7a222e..d2ceaf0 100644
--- a/tests/ansible/lib/modules/custom_python_detect_environment.py
+++ b/tests/ansible/lib/modules/custom_python_detect_environment.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible new-style Python module. I return details about the Python
# interpreter I run within.
@@ -25,6 +25,11 @@ except NameError:
def main():
module = AnsibleModule(argument_spec={})
module.exit_json(
+ fs={
+ '/tmp': {
+ 'resolved': os.path.realpath('/tmp'),
+ },
+ },
python={
'version': {
'full': '%i.%i.%i' % sys.version_info[:3],
diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py
index ae1b78c..507e53d 100644
--- a/tests/ansible/lib/modules/custom_python_external_module.py
+++ b/tests/ansible/lib/modules/custom_python_external_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I expect the quote from modules2/module_utils/joker.py.
from ansible.module_utils.basic import AnsibleModule
diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py
index be9acb2..95bd0c7 100644
--- a/tests/ansible/lib/modules/custom_python_external_pkg.py
+++ b/tests/ansible/lib/modules/custom_python_external_pkg.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.externalpkg import extmod
diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py
index 6164057..846037e 100755
--- a/tests/ansible/lib/modules/custom_python_json_args_module.py
+++ b/tests/ansible/lib/modules/custom_python_json_args_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible Python JSONARGS module. I should receive an encoding string.
json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""
diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
index 255e372..1d34232 100644
--- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
+++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible new-style Python module. I leak state from each invocation
# into a class variable and a global variable.
diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py
index 51b7452..9767f85 100644
--- a/tests/ansible/lib/modules/custom_python_modify_environ.py
+++ b/tests/ansible/lib/modules/custom_python_modify_environ.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible new-style Python module. I modify the process environment and
# don't clean up after myself.
diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
index 2e0ef0d..728685f 100644
--- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
+++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py
@@ -1,6 +1,20 @@
# I am an Ansible new-style Python module, but I lack an interpreter.
+# See also custom_python_new_style_module, we should be updated in tandem.
+import io
+import json
+import select
+import signal
import sys
+import warnings
+
+# Ansible 2.7 changed how new style modules are invoked. It seems that module
+# parameters are *sometimes* read before the module runs. Modules that try
+# to read directly from stdin, such as this, are unable to. However it doesn't
+# always fail, influences seem to include Ansible & Python version. As noted
+# in ansible.module_utils.basic._load_params() we should probably use that.
+# I think (medium confidence) I narrowed the inflection (with git bisect) to
+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
# As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980
# NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10
@@ -8,11 +22,46 @@ import sys
# from ansible.module_utils.
# import ansible.module_utils.
+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
+SELECT_TIMEOUT = 5.0 # seconds
+SIGNAL_TIMEOUT = 10 # seconds
+
+
+def fail_json(msg, **kwargs):
+ kwargs.update(failed=True, msg=msg)
+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
+ sys.exit(1)
+
+
+def sigalrm_handler(signum, frame):
+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
+
def usage():
sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
sys.exit(1)
+
+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
+signal.signal(signal.SIGALRM, sigalrm_handler)
+signal.alarm(SIGNAL_TIMEOUT)
+
+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
+# Consider buffering, line buffering, `f.read()` vs `f.read1()`.
+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
+try:
+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
+else:
+ if not inputs_ready:
+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
+ % (SELECT_TIMEOUT,))
+
+# Read all data on stdin. May block forever, if EOF is not reached.
input_json = sys.stdin.read()
print("{")
diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py
index 1e7270c..c84d241 100755
--- a/tests/ansible/lib/modules/custom_python_new_style_module.py
+++ b/tests/ansible/lib/modules/custom_python_new_style_module.py
@@ -1,16 +1,65 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible new-style Python module. I should receive an encoding string.
+# See also custom_python_new_style_module, we should be updated in tandem.
+import io
+import json
+import select
+import signal
import sys
+import warnings
+
+# Ansible 2.7 changed how new style modules are invoked. It seems that module
+# parameters are *sometimes* read before the module runs. Modules that try
+# to read directly from stdin, such as this, are unable to. However it doesn't
+# always fail, influences seem to include Ansible & Python version. As noted
+# in ansible.module_utils.basic._load_params() we should probably use that.
+# I think (medium confidence) I narrowed the inflection (with git bisect) to
+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
# This is the magic marker Ansible looks for:
# from ansible.module_utils.
+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
+SELECT_TIMEOUT = 5.0 # seconds
+SIGNAL_TIMEOUT = 10 # seconds
+
+
+def fail_json(msg, **kwargs):
+ kwargs.update(failed=True, msg=msg)
+ print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
+ sys.exit(1)
+
+
+def sigalrm_handler(signum, frame):
+ fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
+
def usage():
sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
sys.exit(1)
+
+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
+signal.signal(signal.SIGALRM, sigalrm_handler)
+signal.alarm(SIGNAL_TIMEOUT)
+
+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
+# Consider buffering, line buffering, `f.read()` vs `f.read1()`.
+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
+try:
+ inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
+ # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
+ warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
+else:
+ if not inputs_ready:
+ fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
+ % (SELECT_TIMEOUT,))
+
+# Read all data on stdin. May block forever, if EOF is not reached.
input_json = sys.stdin.read()
print("{")
diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py
index c5e264a..d465ac9 100644
--- a/tests/ansible/lib/modules/custom_python_os_getcwd.py
+++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
# process environment.
diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py
index 0cf9774..6139748 100644
--- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py
+++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/python
+
+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
+# as it is sent to a target. There are probably very few modules in the wild
+# that still do this, reload() is a Python 2.x builtin function.
# issue #555: I'm a module that cutpastes an old hack.
from ansible.module_utils.basic import AnsibleModule
diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py
index d6a839a..4a6243d 100644
--- a/tests/ansible/lib/modules/custom_python_run_script.py
+++ b/tests/ansible/lib/modules/custom_python_run_script.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible new-style Python module. I run the script provided in the
# parameter.
diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py
index 03f3b6a..6b3a356 100644
--- a/tests/ansible/lib/modules/custom_python_uses_distro.py
+++ b/tests/ansible/lib/modules/custom_python_uses_distro.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# issue #590: I am an Ansible new-style Python module that tries to use
# ansible.module_utils.distro.
diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py
index f5e3386..23eeeb5 100755
--- a/tests/ansible/lib/modules/custom_python_want_json_module.py
+++ b/tests/ansible/lib/modules/custom_python_want_json_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
import json
diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
index 1b80a48..289e966 100644
--- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
+++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# I am a module that indirectly depends on glibc cached /etc/resolv.conf state.
diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py
new file mode 100644
index 0000000..41cf1c1
--- /dev/null
+++ b/tests/ansible/lib/modules/module_finder_test.py
@@ -0,0 +1,12 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+import ansible.module_utils.external1
+
+from ansible.module_utils.externalpkg.extmod import path as epem_path
+
+def main():
+ pass
diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py
index 1f71e87..d6a5fb9 100644
--- a/tests/ansible/lib/modules/test_echo_module.py
+++ b/tests/ansible/lib/modules/test_echo_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
# -*- coding: utf-8 -*-
# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
@@ -9,28 +9,61 @@
from __future__ import absolute_import, division, print_function
__metaclass__ = type
+import os
import platform
import sys
from ansible.module_utils.basic import AnsibleModule
def main():
- result = dict(changed=False)
-
module = AnsibleModule(argument_spec=dict(
- facts=dict(type=dict, default={})
+ facts_copy=dict(type=dict, default={}),
+ facts_to_override=dict(type=dict, default={})
))
- result['ansible_facts'] = module.params['facts']
# revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info
- if sys.platform == 'darwin' and sys.executable != '/usr/bin/python':
- if int(platform.release()[:2]) < 19:
- sys.executable = sys.executable[:-3]
- else:
+ # NB This must be synced with mitogen.parent.Connection.get_boot_command()
+ platform_release_major = int(platform.release().partition('.')[0])
+ if sys.modules.get('mitogen') and sys.platform == 'darwin':
+ if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7':
+ sys.executable = '/usr/bin/python'
+ if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7):
# only for tests to check version of running interpreter -- Mac 10.15+ changed python2
# so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
sys.executable = "/usr/bin/python"
- result['running_python_interpreter'] = sys.executable
+
+ facts_copy = module.params['facts_copy']
+ discovered_interpreter_python = facts_copy['discovered_interpreter_python']
+ result = {
+ 'changed': False,
+ 'ansible_facts': module.params['facts_to_override'],
+ 'discovered_and_running_samefile': os.path.samefile(
+ os.path.realpath(discovered_interpreter_python),
+ os.path.realpath(sys.executable),
+ ),
+ 'discovered_python': {
+ 'as_seen': discovered_interpreter_python,
+ 'resolved': os.path.realpath(discovered_interpreter_python),
+ },
+ 'running_python': {
+ 'platform': {
+ 'release': {
+ 'major': platform_release_major,
+ },
+ },
+ 'sys': {
+ 'executable': {
+ 'as_seen': sys.executable,
+ 'resolved': os.path.realpath(sys.executable),
+ },
+ 'platform': sys.platform,
+ 'version_info': {
+ 'major': sys.version_info[0],
+ 'minor': sys.version_info[1],
+ },
+ },
+ },
+ }
module.exit_json(**result)
diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
index f4c47ab..610eaf3 100644
--- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
+++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml
@@ -23,9 +23,16 @@
when:
- lout.python.version.full is version('2.7', '>=', strict=True)
- - assert:
+ - name: Check virtualenv was used
+ # On macOS runners a symlink /tmp -> /private/tmp has been seen
+ vars:
+ requested_executable: /tmp/issue_152_virtualenv/bin/python
+ expected_executables:
+ - "{{ requested_executable }}"
+ - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}"
+ assert:
that:
- - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python"
+ - out.sys_executable in expected_executables
fail_msg: out={{out}}
when:
- lout.python.version.full is version('2.7', '>=', strict=True)
diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml
index 2ca6b97..8903494 100644
--- a/tests/ansible/setup/all.yml
+++ b/tests/ansible/setup/all.yml
@@ -1 +1,2 @@
-- import_playbook: report.yml
+- import_playbook: report_controller.yml
+- import_playbook: report_targets.yml
diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml
new file mode 100644
index 0000000..d0d5cc1
--- /dev/null
+++ b/tests/ansible/setup/report_controller.yml
@@ -0,0 +1,17 @@
+- name: Report controller parameters
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - debug:
+ msg:
+ - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('<unset>') }}"
+ - ${USER}: "{{ lookup('env', 'USER') | default('<unset>') }}"
+ - $(groups): "{{ lookup('pipe', 'groups') }}"
+ - $(pwd): "{{ lookup('pipe', 'pwd') }}"
+ - $(whoami): "{{ lookup('pipe', 'whoami') }}"
+ - ansible_run_tags: "{{ ansible_run_tags | default('<unset>') }}"
+ - ansible_playbook_python: "{{ ansible_playbook_python | default('<unset>') }}"
+ - ansible_skip_tags: "{{ ansible_skip_tags | default('<unset>') }}"
+ - ansible_version.full: "{{ ansible_version.full | default('<unset>') }}"
+ - is_mitogen: "{{ is_mitogen | default('<unset>') }}"
+ - playbook_dir: "{{ playbook_dir | default('<unset>') }}"
diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report_targets.yml
index 450e4fb..5aa6712 100644
--- a/tests/ansible/setup/report.yml
+++ b/tests/ansible/setup/report_targets.yml
@@ -1,4 +1,4 @@
-- name: Report runtime settings
+- name: Report target facts
hosts: localhost:test-targets
gather_facts: true
tasks:
@@ -13,8 +13,3 @@
- debug: {var: ansible_facts.osversion}
- debug: {var: ansible_facts.python}
- debug: {var: ansible_facts.system}
- - debug: {var: ansible_forks}
- - debug: {var: ansible_run_tags}
- - debug: {var: ansible_skip_tags}
- - debug: {var: ansible_version.full}
- - debug: {var: is_mitogen}
diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py
new file mode 100644
index 0000000..79e8fdb
--- /dev/null
+++ b/tests/ansible/tests/module_finder_test.py
@@ -0,0 +1,80 @@
+import os.path
+import sys
+import textwrap
+import unittest
+
+import ansible_mitogen.module_finder
+
+import testlib
+
+
+class ScanFromListTest(testlib.TestCase):
+ def test_absolute_imports(self):
+ source = textwrap.dedent('''\
+ from __future__ import absolute_import
+ import a; import b.c; from d.e import f; from g import h, i
+ ''')
+ code = compile(source, '<str>', 'exec')
+ self.assertEqual(
+ list(ansible_mitogen.module_finder.scan_fromlist(code)),
+ [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')],
+ )
+
+
+class WalkImportsTest(testlib.TestCase):
+ def test_absolute_imports(self):
+ source = textwrap.dedent('''\
+ from __future__ import absolute_import
+ import a; import b; import b.c; from b.d import e, f
+ ''')
+ code = compile(source, '<str>', 'exec')
+
+ self.assertEqual(
+ list(ansible_mitogen.module_finder.walk_imports(code)),
+ ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'],
+ )
+ self.assertEqual(
+ list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')),
+ ['b.c', 'b.d', 'b.d.e', 'b.d.f'],
+ )
+
+
+class ScanTest(testlib.TestCase):
+ module_name = 'ansible_module_module_finder_test__this_should_not_matter'
+ module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py')
+ search_path = (
+ 'does_not_exist/module_utils',
+ testlib.ANSIBLE_MODULE_UTILS_DIR,
+ )
+
+ @staticmethod
+ def relpath(path):
+ return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR)
+
+ @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable')
+ def test_importlib_find_spec(self):
+ scan = ansible_mitogen.module_finder._scan_importlib_find_spec
+ actual = scan(self.module_name, self.module_path, self.search_path)
+ self.assertEqual(
+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+ [
+ ('ansible.module_utils.external1', 'external1.py', False),
+ ('ansible.module_utils.external2', 'external2.py', False),
+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+ ],
+ )
+
+ @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred')
+ def test_imp_find_module(self):
+ scan = ansible_mitogen.module_finder._scan_imp_find_module
+ actual = scan(self.module_name, self.module_path, self.search_path)
+ self.assertEqual(
+ [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+ [
+ ('ansible.module_utils.external1', 'external1.py', False),
+ ('ansible.module_utils.external2', 'external2.py', False),
+ ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+ ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+ ],
+ )
diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json
new file mode 100644
index 0000000..4bf0bd3
--- /dev/null
+++ b/tests/data/importer/webproject/modules_expected_py2x.json
@@ -0,0 +1,186 @@
+{
+ "find_related_imports": {
+ "django.db": [
+ "django",
+ "django.core",
+ "django.core.signals",
+ "django.db.utils"
+ ],
+ "django.db.models": [
+ "django",
+ "django.core.exceptions",
+ "django.db",
+ "django.db.models",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.deletion",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.query",
+ "django.db.models.signals"
+ ]
+ },
+ "find_related": {
+ "django.db": [
+ "django",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.core",
+ "django.core.exceptions",
+ "django.core.signals",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.dispatch.weakref_backports",
+ "django.utils",
+ "django.utils._os",
+ "django.utils.deprecation",
+ "django.utils.encoding",
+ "django.utils.functional",
+ "django.utils.inspect",
+ "django.utils.lru_cache",
+ "django.utils.module_loading",
+ "django.utils.six",
+ "django.utils.version"
+ ],
+ "django.db.models": [
+ "django",
+ "django.apps",
+ "django.apps.config",
+ "django.apps.registry",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.core",
+ "django.core.cache",
+ "django.core.cache.backends",
+ "django.core.cache.backends.base",
+ "django.core.checks",
+ "django.core.checks.caches",
+ "django.core.checks.compatibility",
+ "django.core.checks.compatibility.django_1_10",
+ "django.core.checks.compatibility.django_1_8_0",
+ "django.core.checks.database",
+ "django.core.checks.messages",
+ "django.core.checks.model_checks",
+ "django.core.checks.registry",
+ "django.core.checks.security",
+ "django.core.checks.security.base",
+ "django.core.checks.security.csrf",
+ "django.core.checks.security.sessions",
+ "django.core.checks.templates",
+ "django.core.checks.urls",
+ "django.core.checks.utils",
+ "django.core.exceptions",
+ "django.core.files",
+ "django.core.files.base",
+ "django.core.files.images",
+ "django.core.files.locks",
+ "django.core.files.move",
+ "django.core.files.storage",
+ "django.core.files.utils",
+ "django.core.signals",
+ "django.core.validators",
+ "django.db",
+ "django.db.backends",
+ "django.db.backends.utils",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.constants",
+ "django.db.models.deletion",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.fields.related_descriptors",
+ "django.db.models.fields.related_lookups",
+ "django.db.models.fields.reverse_related",
+ "django.db.models.functions",
+ "django.db.models.functions.base",
+ "django.db.models.functions.datetime",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.options",
+ "django.db.models.query",
+ "django.db.models.query_utils",
+ "django.db.models.signals",
+ "django.db.models.sql",
+ "django.db.models.sql.constants",
+ "django.db.models.sql.datastructures",
+ "django.db.models.sql.query",
+ "django.db.models.sql.subqueries",
+ "django.db.models.sql.where",
+ "django.db.models.utils",
+ "django.db.transaction",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.dispatch.weakref_backports",
+ "django.forms",
+ "django.forms.boundfield",
+ "django.forms.fields",
+ "django.forms.forms",
+ "django.forms.formsets",
+ "django.forms.models",
+ "django.forms.renderers",
+ "django.forms.utils",
+ "django.forms.widgets",
+ "django.template",
+ "django.template.backends",
+ "django.template.backends.base",
+ "django.template.backends.django",
+ "django.template.base",
+ "django.template.context",
+ "django.template.engine",
+ "django.template.exceptions",
+ "django.template.library",
+ "django.template.loader",
+ "django.template.utils",
+ "django.templatetags",
+ "django.templatetags.static",
+ "django.utils",
+ "django.utils._os",
+ "django.utils.crypto",
+ "django.utils.datastructures",
+ "django.utils.dateformat",
+ "django.utils.dateparse",
+ "django.utils.dates",
+ "django.utils.datetime_safe",
+ "django.utils.deconstruct",
+ "django.utils.decorators",
+ "django.utils.deprecation",
+ "django.utils.duration",
+ "django.utils.encoding",
+ "django.utils.formats",
+ "django.utils.functional",
+ "django.utils.html",
+ "django.utils.html_parser",
+ "django.utils.http",
+ "django.utils.inspect",
+ "django.utils.ipv6",
+ "django.utils.itercompat",
+ "django.utils.lru_cache",
+ "django.utils.module_loading",
+ "django.utils.numberformat",
+ "django.utils.safestring",
+ "django.utils.six",
+ "django.utils.text",
+ "django.utils.timezone",
+ "django.utils.translation",
+ "django.utils.tree",
+ "django.utils.version",
+ "pytz",
+ "pytz.exceptions",
+ "pytz.lazy",
+ "pytz.tzfile",
+ "pytz.tzinfo"
+ ]
+ }
+}
diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json
new file mode 100644
index 0000000..06e94e6
--- /dev/null
+++ b/tests/data/importer/webproject/modules_expected_py3x-legacy.json
@@ -0,0 +1,209 @@
+{
+ "find_related_imports": {
+ "django.db": [
+ "django",
+ "django.core",
+ "django.core.signals",
+ "django.db.utils",
+ "django.utils.connection"
+ ],
+ "django.db.models": [
+ "django",
+ "django.core.exceptions",
+ "django.db",
+ "django.db.models",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.constraints",
+ "django.db.models.deletion",
+ "django.db.models.enums",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.json",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.query",
+ "django.db.models.query_utils",
+ "django.db.models.signals"
+ ]
+ },
+ "find_related": {
+ "django.db": [
+ "asgiref",
+ "asgiref.compatibility",
+ "asgiref.current_thread_executor",
+ "asgiref.local",
+ "asgiref.sync",
+ "django",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.core",
+ "django.core.exceptions",
+ "django.core.signals",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.utils",
+ "django.utils.connection",
+ "django.utils.deprecation",
+ "django.utils.functional",
+ "django.utils.hashable",
+ "django.utils.inspect",
+ "django.utils.itercompat",
+ "django.utils.module_loading",
+ "django.utils.regex_helper",
+ "django.utils.version"
+ ],
+ "django.db.models": [
+ "asgiref",
+ "asgiref.compatibility",
+ "asgiref.current_thread_executor",
+ "asgiref.local",
+ "asgiref.sync",
+ "django",
+ "django.apps",
+ "django.apps.config",
+ "django.apps.registry",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.conf.locale",
+ "django.core",
+ "django.core.cache",
+ "django.core.cache.backends",
+ "django.core.cache.backends.base",
+ "django.core.cache.backends.filebased",
+ "django.core.checks",
+ "django.core.checks.async_checks",
+ "django.core.checks.caches",
+ "django.core.checks.database",
+ "django.core.checks.messages",
+ "django.core.checks.model_checks",
+ "django.core.checks.registry",
+ "django.core.checks.security",
+ "django.core.checks.security.base",
+ "django.core.checks.security.csrf",
+ "django.core.checks.security.sessions",
+ "django.core.checks.templates",
+ "django.core.checks.translation",
+ "django.core.checks.urls",
+ "django.core.exceptions",
+ "django.core.files",
+ "django.core.files.base",
+ "django.core.files.images",
+ "django.core.files.locks",
+ "django.core.files.move",
+ "django.core.files.storage",
+ "django.core.files.utils",
+ "django.core.signals",
+ "django.core.validators",
+ "django.db",
+ "django.db.backends",
+ "django.db.backends.utils",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.constants",
+ "django.db.models.constraints",
+ "django.db.models.deletion",
+ "django.db.models.enums",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.json",
+ "django.db.models.fields.mixins",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.fields.related_descriptors",
+ "django.db.models.fields.related_lookups",
+ "django.db.models.fields.reverse_related",
+ "django.db.models.functions",
+ "django.db.models.functions.comparison",
+ "django.db.models.functions.datetime",
+ "django.db.models.functions.math",
+ "django.db.models.functions.mixins",
+ "django.db.models.functions.text",
+ "django.db.models.functions.window",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.options",
+ "django.db.models.query",
+ "django.db.models.query_utils",
+ "django.db.models.signals",
+ "django.db.models.sql",
+ "django.db.models.sql.constants",
+ "django.db.models.sql.datastructures",
+ "django.db.models.sql.query",
+ "django.db.models.sql.subqueries",
+ "django.db.models.sql.where",
+ "django.db.models.utils",
+ "django.db.transaction",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.forms",
+ "django.forms.boundfield",
+ "django.forms.fields",
+ "django.forms.forms",
+ "django.forms.formsets",
+ "django.forms.models",
+ "django.forms.renderers",
+ "django.forms.utils",
+ "django.forms.widgets",
+ "django.template",
+ "django.template.backends",
+ "django.template.backends.base",
+ "django.template.backends.django",
+ "django.template.base",
+ "django.template.context",
+ "django.template.engine",
+ "django.template.exceptions",
+ "django.template.library",
+ "django.template.loader",
+ "django.template.utils",
+ "django.templatetags",
+ "django.templatetags.static",
+ "django.utils",
+ "django.utils._os",
+ "django.utils.autoreload",
+ "django.utils.connection",
+ "django.utils.crypto",
+ "django.utils.datastructures",
+ "django.utils.dateformat",
+ "django.utils.dateparse",
+ "django.utils.dates",
+ "django.utils.datetime_safe",
+ "django.utils.deconstruct",
+ "django.utils.deprecation",
+ "django.utils.duration",
+ "django.utils.encoding",
+ "django.utils.formats",
+ "django.utils.functional",
+ "django.utils.hashable",
+ "django.utils.html",
+ "django.utils.http",
+ "django.utils.inspect",
+ "django.utils.ipv6",
+ "django.utils.itercompat",
+ "django.utils.module_loading",
+ "django.utils.numberformat",
+ "django.utils.regex_helper",
+ "django.utils.safestring",
+ "django.utils.text",
+ "django.utils.timezone",
+ "django.utils.topological_sort",
+ "django.utils.translation",
+ "django.utils.translation.trans_real",
+ "django.utils.tree",
+ "django.utils.version",
+ "pytz",
+ "pytz.exceptions",
+ "pytz.lazy",
+ "pytz.tzfile",
+ "pytz.tzinfo"
+ ]
+ }
+}
diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json
new file mode 100644
index 0000000..dcbcc78
--- /dev/null
+++ b/tests/data/importer/webproject/modules_expected_py3x-new.json
@@ -0,0 +1,208 @@
+{
+ "find_related_imports": {
+ "django.db": [
+ "django",
+ "django.core",
+ "django.core.signals",
+ "django.db.utils",
+ "django.utils.connection"
+ ],
+ "django.db.models": [
+ "django",
+ "django.core.exceptions",
+ "django.db",
+ "django.db.models",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.constraints",
+ "django.db.models.deletion",
+ "django.db.models.enums",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.json",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.query",
+ "django.db.models.query_utils",
+ "django.db.models.signals"
+]
+ },
+ "find_related": {
+ "django.db": [
+ "asgiref",
+ "asgiref.current_thread_executor",
+ "asgiref.local",
+ "asgiref.sync",
+ "django",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.core",
+ "django.core.exceptions",
+ "django.core.signals",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.utils",
+ "django.utils.connection",
+ "django.utils.deprecation",
+ "django.utils.functional",
+ "django.utils.hashable",
+ "django.utils.inspect",
+ "django.utils.itercompat",
+ "django.utils.module_loading",
+ "django.utils.regex_helper",
+ "django.utils.version"
+ ],
+ "django.db.models": [
+ "asgiref",
+ "asgiref.current_thread_executor",
+ "asgiref.local",
+ "asgiref.sync",
+ "django",
+ "django.apps",
+ "django.apps.config",
+ "django.apps.registry",
+ "django.conf",
+ "django.conf.global_settings",
+ "django.conf.locale",
+ "django.core",
+ "django.core.cache",
+ "django.core.cache.backends",
+ "django.core.cache.backends.base",
+ "django.core.cache.backends.filebased",
+ "django.core.checks",
+ "django.core.checks.async_checks",
+ "django.core.checks.caches",
+ "django.core.checks.database",
+ "django.core.checks.messages",
+ "django.core.checks.model_checks",
+ "django.core.checks.registry",
+ "django.core.checks.security",
+ "django.core.checks.security.base",
+ "django.core.checks.security.csrf",
+ "django.core.checks.security.sessions",
+ "django.core.checks.templates",
+ "django.core.checks.translation",
+ "django.core.checks.urls",
+ "django.core.exceptions",
+ "django.core.files",
+ "django.core.files.base",
+ "django.core.files.images",
+ "django.core.files.locks",
+ "django.core.files.move",
+ "django.core.files.storage",
+ "django.core.files.utils",
+ "django.core.signals",
+ "django.core.validators",
+ "django.db",
+ "django.db.backends",
+ "django.db.backends.utils",
+ "django.db.models.aggregates",
+ "django.db.models.base",
+ "django.db.models.constants",
+ "django.db.models.constraints",
+ "django.db.models.deletion",
+ "django.db.models.enums",
+ "django.db.models.expressions",
+ "django.db.models.fields",
+ "django.db.models.fields.files",
+ "django.db.models.fields.json",
+ "django.db.models.fields.mixins",
+ "django.db.models.fields.proxy",
+ "django.db.models.fields.related",
+ "django.db.models.fields.related_descriptors",
+ "django.db.models.fields.related_lookups",
+ "django.db.models.fields.reverse_related",
+ "django.db.models.functions",
+ "django.db.models.functions.comparison",
+ "django.db.models.functions.datetime",
+ "django.db.models.functions.math",
+ "django.db.models.functions.mixins",
+ "django.db.models.functions.text",
+ "django.db.models.functions.window",
+ "django.db.models.indexes",
+ "django.db.models.lookups",
+ "django.db.models.manager",
+ "django.db.models.options",
+ "django.db.models.query",
+ "django.db.models.query_utils",
+ "django.db.models.signals",
+ "django.db.models.sql",
+ "django.db.models.sql.constants",
+ "django.db.models.sql.datastructures",
+ "django.db.models.sql.query",
+ "django.db.models.sql.subqueries",
+ "django.db.models.sql.where",
+ "django.db.models.utils",
+ "django.db.transaction",
+ "django.db.utils",
+ "django.dispatch",
+ "django.dispatch.dispatcher",
+ "django.forms",
+ "django.forms.boundfield",
+ "django.forms.fields",
+ "django.forms.forms",
+ "django.forms.formsets",
+ "django.forms.models",
+ "django.forms.renderers",
+ "django.forms.utils",
+ "django.forms.widgets",
+ "django.template",
+ "django.template.backends",
+ "django.template.backends.base",
+ "django.template.backends.django",
+ "django.template.base",
+ "django.template.context",
+ "django.template.engine",
+ "django.template.exceptions",
+ "django.template.library",
+ "django.template.loader",
+ "django.template.utils",
+ "django.templatetags",
+ "django.templatetags.static",
+ "django.utils",
+ "django.utils._os",
+ "django.utils.autoreload",
+ "django.utils.connection",
+ "django.utils.crypto",
+ "django.utils.datastructures",
+ "django.utils.dateformat",
+ "django.utils.dateparse",
+ "django.utils.dates",
+ "django.utils.datetime_safe",
+ "django.utils.deconstruct",
+ "django.utils.deprecation",
+ "django.utils.duration",
+ "django.utils.encoding",
+ "django.utils.formats",
+ "django.utils.functional",
+ "django.utils.hashable",
+ "django.utils.html",
+ "django.utils.http",
+ "django.utils.inspect",
+ "django.utils.ipv6",
+ "django.utils.itercompat",
+ "django.utils.module_loading",
+ "django.utils.numberformat",
+ "django.utils.regex_helper",
+ "django.utils.safestring",
+ "django.utils.text",
+ "django.utils.timezone",
+ "django.utils.topological_sort",
+ "django.utils.translation",
+ "django.utils.translation.trans_real",
+ "django.utils.tree",
+ "django.utils.version",
+ "pytz",
+ "pytz.exceptions",
+ "pytz.lazy",
+ "pytz.tzfile",
+ "pytz.tzinfo"
+ ]
+ }
+}
+ \ No newline at end of file
diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml
index 6224b61..0b6d5e6 100644
--- a/tests/image_prep/_user_accounts.yml
+++ b/tests/image_prep/_user_accounts.yml
@@ -4,6 +4,8 @@
# WARNING: this creates non-privilged accounts with pre-set passwords!
#
+- import_playbook: ../ansible/setup/report_controller.yml
+
- hosts: all
gather_facts: true
strategy: mitogen_free
@@ -37,12 +39,12 @@
normal_users: "{{
lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
- }}"
+ }}"
all_users: "{{
special_users +
normal_users
- }}"
+ }}"
tasks:
- name: Disable non-localhost SSH for Mitogen users
when: false
@@ -100,6 +102,7 @@
with_items: "{{all_users}}"
copy:
dest: /var/lib/AccountsService/users/mitogen__{{item}}
+ mode: u=rw,go=
content: |
[User]
SystemAccount=true
@@ -108,7 +111,7 @@
when: ansible_system == 'Linux' and out.stat.exists
service:
name: accounts-daemon
- restarted: true
+ state: restarted
- name: Readonly homedir for one account
shell: "chown -R root: ~mitogen__readonly_homedir"
@@ -117,6 +120,9 @@
copy:
dest: ~mitogen__slow_user/.{{item}}
src: ../data/docker/mitogen__slow_user.profile
+ owner: mitogen__slow_user
+ group: mitogen__group
+ mode: u=rw,go=r
with_items:
- bashrc
- profile
@@ -125,6 +131,9 @@
copy:
dest: ~mitogen__permdenied/.{{item}}
src: ../data/docker/mitogen__permdenied.profile
+ owner: mitogen__permdenied
+ group: mitogen__group
+ mode: u=rw,go=r
with_items:
- bashrc
- profile
@@ -136,20 +145,13 @@
state: directory
mode: go=
owner: mitogen__has_sudo_pubkey
+ group: mitogen__group
- copy:
dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
mode: go=
owner: mitogen__has_sudo_pubkey
-
- - name: Install slow profile for one account
- block:
- - copy:
- dest: ~mitogen__slow_user/.profile
- src: ../data/docker/mitogen__slow_user.profile
- - copy:
- dest: ~mitogen__slow_user/.bashrc
- src: ../data/docker/mitogen__slow_user.profile
+ group: mitogen__group
- name: Require a TTY for two accounts
lineinfile:
diff --git a/tests/importer_test.py b/tests/importer_test.py
index e48c02a..e86af8a 100644
--- a/tests/importer_test.py
+++ b/tests/importer_test.py
@@ -2,6 +2,7 @@ import sys
import threading
import types
import zlib
+import unittest
import mock
@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin):
super(ImporterMixin, self).tearDown()
+class InvalidNameTest(ImporterMixin, testlib.TestCase):
+ modname = 'trailingdot.'
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ response = (modname, None, None, None, None)
+
+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+ def test_find_spec_invalid(self):
+ self.set_get_module_response(self.response)
+ self.assertEqual(self.importer.find_spec(self.modname, path=None), None)
+
+
+class MissingModuleTest(ImporterMixin, testlib.TestCase):
+ modname = 'missing'
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ response = (modname, None, None, None, None)
+
+ @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
+ def test_load_module_missing(self):
+ self.set_get_module_response(self.response)
+ self.assertRaises(ImportError, self.importer.load_module, self.modname)
+
+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+ def test_find_spec_missing(self):
+ """
+ Importer should optimistically offer itself as a module loader
+ when there are no disqualifying criteria.
+ """
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = self.importer.find_spec(self.modname, path=None)
+ self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
+ self.assertEqual(spec.name, self.modname)
+ self.assertEqual(spec.loader, self.importer)
+
+ @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+ def test_create_module_missing(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ self.assertRaises(ImportError, self.importer.create_module, spec)
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
class LoadModuleTest(ImporterMixin, testlib.TestCase):
data = zlib.compress(b("data = 1\n\n"))
path = 'fake_module.py'
@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
# 0:fullname 1:pkg_present 2:path 3:compressed 4:related
response = (modname, None, path, data, [])
- def test_no_such_module(self):
- self.set_get_module_response(
- # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
- (self.modname, None, None, None, None)
- )
- self.assertRaises(ImportError,
- lambda: self.importer.load_module(self.modname))
-
def test_module_added_to_sys_modules(self):
self.set_get_module_response(self.response)
mod = self.importer.load_module(self.modname)
@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
self.assertIsNone(mod.__package__)
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ModuleSpecTest(ImporterMixin, testlib.TestCase):
+ data = zlib.compress(b("data = 1\n\n"))
+ path = 'fake_module.py'
+ modname = 'fake_module'
+
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ response = (modname, None, path, data, [])
+
+ def test_module_attributes(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ mod = self.importer.create_module(spec)
+ self.assertIsInstance(mod, types.ModuleType)
+ self.assertEqual(mod.__name__, 'fake_module')
+ #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
data = zlib.compress(b("data = 1\n\n"))
path = 'fake_module.py'
@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
self.assertEqual(mod.__package__, 'mypkg')
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase):
+ data = zlib.compress(b("data = 1\n\n"))
+ path = 'fake_module.py'
+ modname = 'mypkg.fake_module'
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ response = (modname, None, path, data, [])
+
+ def test_module_attributes(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ mod = self.importer.create_module(spec)
+ self.assertIsInstance(mod, types.ModuleType)
+ self.assertEqual(mod.__name__, 'mypkg.fake_module')
+ #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
data = zlib.compress(b("func = lambda: 1\n\n"))
path = 'fake_pkg/__init__.py'
@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
self.assertEqual(mod.func.__module__, self.modname)
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class PackageSpecTest(ImporterMixin, testlib.TestCase):
+ data = zlib.compress(b("func = lambda: 1\n\n"))
+ path = 'fake_pkg/__init__.py'
+ modname = 'fake_pkg'
+ # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+ response = (modname, [], path, data, [])
+
+ def test_module_attributes(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ mod = self.importer.create_module(spec)
+ self.assertIsInstance(mod, types.ModuleType)
+ self.assertEqual(mod.__name__, 'fake_pkg')
+ #self.assertFalse(hasattr(mod, '__file__'))
+
+ def test_get_filename(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ _ = self.importer.create_module(spec)
+ filename = self.importer.get_filename(self.modname)
+ self.assertEqual('master:fake_pkg/__init__.py', filename)
+
+ def test_get_source(self):
+ import importlib.machinery
+ self.set_get_module_response(self.response)
+ spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+ _ = self.importer.create_module(spec)
+ source = self.importer.get_source(self.modname)
+ self.assertEqual(source,
+ mitogen.core.to_text(zlib.decompress(self.data)))
+
+
class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase):
def initdir(self, caplog):
self.caplog = caplog
diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index ff18bbc..67e937e 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -1,4 +1,5 @@
import inspect
+import json
import os
import sys
import unittest
@@ -139,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase):
self.assertIsNone(tup)
-class GetModuleViaParentEnumerationTest(testlib.TestCase):
- klass = mitogen.master.ParentEnumerationMethod
-
+class ParentEnumerationMixin(object):
def call(self, fullname):
return self.klass().find(fullname)
@@ -231,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
self.assertEqual(is_pkg, False)
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4')
+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+ klass = mitogen.master.ParentImpEnumerationMethod
+
+
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+ klass = mitogen.master.ParentSpecEnumerationMethod
+
+
class ResolveRelPathTest(testlib.TestCase):
klass = mitogen.master.ModuleFinder
@@ -296,199 +305,65 @@ class FindRelatedTest(testlib.TestCase):
self.assertEqual(set(related), self.SIMPLE_EXPECT)
-if sys.version_info > (2, 6):
- class DjangoMixin(object):
- WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject')
-
- # TODO: rip out Django and replace with a static tree of weird imports
- # that don't depend on .. Django! The hack below is because the version
- # of Django we need to test against 2.6 doesn't actually run on 3.6.
- # But we don't care, we just need to be able to import it.
- #
- # File "django/utils/html_parser.py", line 12, in <module>
- # AttributeError: module 'html.parser' has no attribute
- # 'HTMLParseError'
- #
- from django.utils.six.moves import html_parser as _html_parser
- _html_parser.HTMLParseError = Exception
-
- @classmethod
- def setUpClass(cls):
- super(DjangoMixin, cls).setUpClass()
- sys.path.append(cls.WEBPROJECT_PATH)
- os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings'
-
- @classmethod
- def tearDownClass(cls):
- sys.path.remove(cls.WEBPROJECT_PATH)
- del os.environ['DJANGO_SETTINGS_MODULE']
- super(DjangoMixin, cls).tearDownClass()
-
-
- class FindRelatedImportsTest(DjangoMixin, testlib.TestCase):
- klass = mitogen.master.ModuleFinder
-
- def call(self, fullname):
- return self.klass().find_related_imports(fullname)
-
- def test_django_db(self):
- import django.db
- related = self.call('django.db')
- self.assertEqual(related, [
- 'django',
- 'django.core',
- 'django.core.signals',
- 'django.db.utils',
- 'django.utils.functional',
- ])
-
- def test_django_db_models(self):
- import django.db.models
- related = self.call('django.db.models')
- self.maxDiff=None
- self.assertEqual(related, [
- u'django',
- u'django.core.exceptions',
- u'django.db',
- u'django.db.models',
- u'django.db.models.aggregates',
- u'django.db.models.base',
- u'django.db.models.deletion',
- u'django.db.models.expressions',
- u'django.db.models.fields',
- u'django.db.models.fields.files',
- u'django.db.models.fields.related',
- u'django.db.models.fields.subclassing',
- u'django.db.models.loading',
- u'django.db.models.manager',
- u'django.db.models.query',
- u'django.db.models.signals',
- ])
-
-
- class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase):
- klass = mitogen.master.ModuleFinder
- maxDiff = None
-
- def call(self, fullname):
- return self.klass().find_related(fullname)
-
- def test_django_db(self):
- import django.db
- related = self.call('django.db')
- self.assertEqual(related, [
- u'django',
- u'django.conf',
- u'django.conf.global_settings',
- u'django.core',
- u'django.core.exceptions',
- u'django.core.signals',
- u'django.db.utils',
- u'django.dispatch',
- u'django.dispatch.dispatcher',
- u'django.dispatch.saferef',
- u'django.utils',
- u'django.utils._os',
- u'django.utils.encoding',
- u'django.utils.functional',
- u'django.utils.importlib',
- u'django.utils.module_loading',
- u'django.utils.six',
- ])
-
- @unittest.skipIf(
- condition=(sys.version_info >= (3, 0)),
- reason='broken due to ancient vendored six.py'
- )
- def test_django_db_models(self):
- import django.db.models
- related = self.call('django.db.models')
- self.assertEqual(related, [
- u'django',
- u'django.conf',
- u'django.conf.global_settings',
- u'django.core',
- u'django.core.exceptions',
- u'django.core.files',
- u'django.core.files.base',
- u'django.core.files.images',
- u'django.core.files.locks',
- u'django.core.files.move',
- u'django.core.files.storage',
- u'django.core.files.utils',
- u'django.core.signals',
- u'django.core.validators',
- u'django.db',
- u'django.db.backends',
- u'django.db.backends.signals',
- u'django.db.backends.util',
- u'django.db.models.aggregates',
- u'django.db.models.base',
- u'django.db.models.constants',
- u'django.db.models.deletion',
- u'django.db.models.expressions',
- u'django.db.models.fields',
- u'django.db.models.fields.files',
- u'django.db.models.fields.proxy',
- u'django.db.models.fields.related',
- u'django.db.models.fields.subclassing',
- u'django.db.models.loading',
- u'django.db.models.manager',
- u'django.db.models.options',
- u'django.db.models.query',
- u'django.db.models.query_utils',
- u'django.db.models.related',
- u'django.db.models.signals',
- u'django.db.models.sql',
- u'django.db.models.sql.aggregates',
- u'django.db.models.sql.constants',
- u'django.db.models.sql.datastructures',
- u'django.db.models.sql.expressions',
- u'django.db.models.sql.query',
- u'django.db.models.sql.subqueries',
- u'django.db.models.sql.where',
- u'django.db.transaction',
- u'django.db.utils',
- u'django.dispatch',
- u'django.dispatch.dispatcher',
- u'django.dispatch.saferef',
- u'django.forms',
- u'django.forms.fields',
- u'django.forms.forms',
- u'django.forms.formsets',
- u'django.forms.models',
- u'django.forms.util',
- u'django.forms.widgets',
- u'django.utils',
- u'django.utils._os',
- u'django.utils.crypto',
- u'django.utils.datastructures',
- u'django.utils.dateformat',
- u'django.utils.dateparse',
- u'django.utils.dates',
- u'django.utils.datetime_safe',
- u'django.utils.decorators',
- u'django.utils.deprecation',
- u'django.utils.encoding',
- u'django.utils.formats',
- u'django.utils.functional',
- u'django.utils.html',
- u'django.utils.html_parser',
- u'django.utils.importlib',
- u'django.utils.ipv6',
- u'django.utils.itercompat',
- u'django.utils.module_loading',
- u'django.utils.numberformat',
- u'django.utils.safestring',
- u'django.utils.six',
- u'django.utils.text',
- u'django.utils.timezone',
- u'django.utils.translation',
- u'django.utils.tree',
- u'django.utils.tzinfo',
- u'pytz',
- u'pytz.exceptions',
- u'pytz.lazy',
- u'pytz.tzfile',
- u'pytz.tzinfo',
- ])
+class DjangoMixin(object):
+ WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject')
+
+ @classmethod
+ def modules_expected_path(cls):
+ if sys.version_info[0:2] < (3, 0):
+ modules_expected_filename = 'modules_expected_py2x.json'
+ elif sys.version_info[0:2] <= (3, 6):
+ modules_expected_filename = 'modules_expected_py3x-legacy.json'
+ elif sys.version_info[0:2] >= (3, 10):
+ modules_expected_filename = 'modules_expected_py3x-new.json'
+ return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename)
+
+ @classmethod
+ def setUpClass(cls):
+ super(DjangoMixin, cls).setUpClass()
+ sys.path.append(cls.WEBPROJECT_PATH)
+ os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings'
+ with open(cls.modules_expected_path(), 'rb') as f:
+ cls.MODULES_EXPECTED = json.load(f)
+
+ @classmethod
+ def tearDownClass(cls):
+ sys.path.remove(cls.WEBPROJECT_PATH)
+ del os.environ['DJANGO_SETTINGS_MODULE']
+ super(DjangoMixin, cls).tearDownClass()
+
+
+class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase):
+ maxDiff = None
+
+ def test_django_db(self):
+ import django.db
+ module_finder = mitogen.master.ModuleFinder()
+ related = module_finder.find_related('django.db')
+ expected = self.MODULES_EXPECTED['find_related']['django.db']
+ self.assertEqual(related, expected)
+
+ def test_django_db_models(self):
+ import django.db.models
+ module_finder = mitogen.master.ModuleFinder()
+ related = module_finder.find_related('django.db.models')
+ expected = self.MODULES_EXPECTED['find_related']['django.db.models']
+ self.assertEqual(related, expected)
+
+
+class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase):
+ maxDiff = None
+
+ def test_django_db(self):
+ import django.db
+ module_finder = mitogen.master.ModuleFinder()
+ related = module_finder.find_related_imports('django.db')
+ expected = self.MODULES_EXPECTED['find_related_imports']['django.db']
+ self.assertEqual(related, expected)
+
+ def test_django_db_models(self):
+ import django.db.models
+ module_finder = mitogen.master.ModuleFinder()
+ related = module_finder.find_related_imports('django.db.models')
+ expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models']
+ self.assertEqual(related, expected)
diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt
new file mode 100644
index 0000000..bc7f7c2
--- /dev/null
+++ b/tests/requirements-tox.txt
@@ -0,0 +1,4 @@
+tox==3.28; python_version == '2.7'
+tox==3.28; python_version == '3.6'
+tox==4.8.0; python_version == '3.7'
+tox>=4.13.0,~=4.0; python_version >= '3.8'
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 7301bee..6d87d17 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,14 +1,25 @@
-psutil==5.4.8
-coverage==5.5; python_version < '3.7'
-coverage==6.4.4; python_version >= '3.7'
-Django==1.6.11 # Last version supporting 2.6.
-mock==2.0.0
-pytz==2018.5
-cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect
-pycparser==2.18 # Last version supporting 2.6.
-pytest-catchlog==1.2.2
-pytest==3.1.2
-subprocess32==3.5.4
+cffi==1.15.1; python_version < '3.8'
+cffi==1.16; python_version >= '3.8'
+
+coverage==5.5; python_version == '2.7'
+coverage==6.2; python_version == '3.6'
+coverage==7.2.7; python_version == '3.7'
+coverage==7.4.3; python_version >= '3.8'
+
+Django==1.11.29; python_version < '3.0'
+Django==3.2.20; python_version >= '3.6'
+
+mock==3.0.5; python_version == '2.7'
+mock==5.1.0; python_version >= '3.6'
+
+psutil==5.9.8
+
+pytest==4.6.11; python_version == '2.7'
+pytest==7.0.1; python_version == '3.6'
+pytest==7.4.4; python_version == '3.7'
+pytest==8.0.2; python_version >= '3.8'
+
+subprocess32==3.5.4; python_version < '3.0'
timeoutcontext==1.2.0
# Fix InsecurePlatformWarning while creating py26 tox environment
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
@@ -16,4 +27,7 @@ urllib3[secure]==1.23; python_version < '2.7'
urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9'
# Last idna compatible with Python 2.6 was idna 2.7.
idna==2.7; python_version < '2.7'
-virtualenv==20.10.0
+
+virtualenv==20.15.1; python_version == '2.7'
+virtualenv==20.17.1; python_version == '3.6'
+virtualenv==20.25.1; python_version >= '3.7'
diff --git a/tests/ssh_test.py b/tests/ssh_test.py
index 429fdb5..3149fcb 100644
--- a/tests/ssh_test.py
+++ b/tests/ssh_test.py
@@ -134,12 +134,13 @@ class SshTest(testlib.DockerMixin, testlib.TestCase):
def test_enforce_unknown_host_key(self):
fp = tempfile.NamedTemporaryFile()
+ ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', [])
try:
e = self.assertRaises(mitogen.ssh.HostKeyError,
lambda: self.docker_ssh(
username='mitogen__has_sudo_pubkey',
password='has_sudo_password',
- ssh_args=['-o', 'UserKnownHostsFile ' + fp.name],
+ ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name],
check_host_keys='enforce',
)
)
@@ -149,11 +150,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase):
def test_accept_enforce_host_keys(self):
fp = tempfile.NamedTemporaryFile()
+ ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', [])
try:
context = self.docker_ssh(
username='mitogen__has_sudo',
password='has_sudo_password',
- ssh_args=['-o', 'UserKnownHostsFile ' + fp.name],
+ ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name],
check_host_keys='accept',
)
context.shutdown(wait=True)
@@ -166,7 +168,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase):
context = self.docker_ssh(
username='mitogen__has_sudo',
password='has_sudo_password',
- ssh_args=['-o', 'UserKnownHostsFile ' + fp.name],
+ ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name],
check_host_keys='enforce',
)
context.shutdown(wait=True)
diff --git a/tests/testlib.py b/tests/testlib.py
index 83629c3..8c40e7f 100644
--- a/tests/testlib.py
+++ b/tests/testlib.py
@@ -18,7 +18,10 @@ except ImportError:
import ConfigParser as configparser
import psutil
-import subprocess32 as subprocess
+if sys.version_info < (3, 0):
+ import subprocess32 as subprocess
+else:
+ import subprocess
import mitogen.core
import mitogen.fork
@@ -47,8 +50,13 @@ except NameError:
LOG = logging.getLogger(__name__)
-DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
-MODS_DIR = os.path.join(DATA_DIR, 'importer')
+
+TESTS_DIR = os.path.join(os.path.dirname(__file__))
+ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
+ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
+ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules')
+DATA_DIR = os.path.join(TESTS_DIR, 'data')
+MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer')
sys.path.append(DATA_DIR)
sys.path.append(MODS_DIR)
@@ -87,7 +95,7 @@ def base_executable(executable=None):
base_executable = None
if base_executable and base_executable != executable:
- return 'be', base_executable
+ return base_executable
# Python 2.x only has sys.base_prefix if running outside a virtualenv.
try:
@@ -499,19 +507,18 @@ def get_docker_host():
class DockerizedSshDaemon(object):
- def _get_container_port(self):
- s = subprocess.check_output(['docker', 'port', self.container_name])
- for line in s.decode().splitlines():
- m = self.PORT_RE.match(line)
- if not m:
- continue
- dport, proto, _, bport = m.groups()
- if dport == '22' and proto == 'tcp':
- self.port = int(bport)
+ PORT_RE = re.compile(
+ # e.g. 0.0.0.0:32771, :::32771, [::]:32771'
+ r'(?P<addr>[0-9.]+|::|\[[a-f0-9:.]+\]):(?P<port>[0-9]+)',
+ )
- self.host = self.get_host()
- if self.port is None:
+ @classmethod
+ def get_port(cls, container):
+ s = subprocess.check_output(['docker', 'port', container, '22/tcp'])
+ m = cls.PORT_RE.search(s.decode())
+ if not m:
raise ValueError('could not find SSH port in: %r' % (s,))
+ return int(m.group('port'))
def start_container(self):
try:
@@ -530,7 +537,6 @@ class DockerizedSshDaemon(object):
self.image,
]
subprocess.check_output(args)
- self._get_container_port()
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
@@ -545,12 +551,9 @@ class DockerizedSshDaemon(object):
self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
-
- # 22/tcp -> 0.0.0.0:32771
- self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')
- self.port = None
-
self.start_container()
+ self.host = self.get_host()
+ self.port = self.get_port(self.container_name)
def get_host(self):
return get_docker_host()
@@ -631,12 +634,33 @@ class DockerMixin(RouterMixin):
cls.dockerized_ssh.close()
super(DockerMixin, cls).tearDownClass()
+ @property
+ def docker_ssh_default_kwargs(self):
+ return {
+ 'hostname': self.dockerized_ssh.host,
+ 'port': self.dockerized_ssh.port,
+ 'check_host_keys': 'ignore',
+ 'ssh_debug_level': 3,
+ # https://www.openssh.com/legacy.html
+ # ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd.
+ # Rejected by default in newer ssh clients (e.g. Ubuntu 22.04).
+ # Duplicated cases in
+ # - tests/ansible/ansible.cfg
+ # - tests/ansible/integration/connection_delegation/delegate_to_template.yml
+ # - tests/ansible/integration/connection_delegation/stack_construction.yml
+ # - tests/ansible/integration/process/unix_socket_cleanup.yml
+ # - tests/ansible/integration/ssh/variables.yml
+ # - tests/testlib.py
+ 'ssh_args': [
+ '-o', 'HostKeyAlgorithms +ssh-rsa',
+ '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa',
+ ],
+ 'python_path': self.dockerized_ssh.python_path,
+ }
+
def docker_ssh(self, **kwargs):
- kwargs.setdefault('hostname', self.dockerized_ssh.host)
- kwargs.setdefault('port', self.dockerized_ssh.port)
- kwargs.setdefault('check_host_keys', 'ignore')
- kwargs.setdefault('ssh_debug_level', 3)
- kwargs.setdefault('python_path', self.dockerized_ssh.python_path)
+ for k, v in self.docker_ssh_default_kwargs.items():
+ kwargs.setdefault(k, v)
return self.router.ssh(**kwargs)
def docker_ssh_any(self, **kwargs):
diff --git a/tox.ini b/tox.ini
index da71093..4a1772a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,23 +1,39 @@
-# This file is a local convenience. It is not a substitute for the full CI
-# suite, and does not cover the full range of Python versions for Mitogen.
-
-# I use this on Ubuntu 20.04, with the following additions
+# This configuration drives both CI and local development.
+# I use this locally on Ubuntu 22.04, with the following additions
#
# sudo add-apt-repository ppa:deadsnakes/ppa
# sudo apt update
-# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass
+# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox
-# Last version to support each python version
+# Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv
+# ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ==========
+# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8
+# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1
+# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0
+# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15²
+# 3.5 <= 2.11 <= 2.13 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15²
+# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17²
+# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0
+# 3.8 <= 2.12
+# 3.9 <= 2.15
+# 3.10
+# 3.11
+# 3.12 >= 2.13¹
+#
+# Notes
+# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13).
+# Python 3.12 removed support for find_module(), replaced by find_spec().
+# In Ansible <= 4.x ansible.module_utils.six lacks find_spec().
+# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3
+# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder
+# lacks find_spec(). https://github.com/ansible/ansible/pull/76225
+#
+# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15).
+# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments.
+# https://github.com/ansible/ansible/pull/80751
#
-# Python tox virt'env pip A cntllr A target coverage
-# ========== ======== ======== ======== ======== ======== ========
-# python2.4 1.4 1.8 1.1 2.3?
-# python2.5 1.6.1 1.9.1 1.3.1 ???
-# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 2.13 4.5.4
-# python2.7 20.3 2.11
-# python3.5 2.11
-# python3.6 2.11
-# python3.7 2.11
+# 2. Higher virtualenv versions cannot run under this Python version. They can
+# still generate virtual environments for it.
# Ansible Dependency
# ================== ======================
@@ -27,6 +43,9 @@
# ansible == 4.* ansible-core ~= 2.11.0
# ansible == 5.* ansible-core ~= 2.12.0
# ansible == 6.* ansible-core ~= 2.13.0
+# ansible == 7.x ansible-core ~= 2.14.0
+# ansible == 8.x ansible-core ~= 2.15.0
+# ansible == 9.x ansible-core ~= 2.16.0
# pip --no-python-version-warning
# pip --disable-pip-version-check
@@ -37,10 +56,11 @@
envlist =
init,
py{27,36}-mode_ansible-ansible{2.10,3,4},
- py{310}-mode_ansible-ansible{2.10,3,4,5,6},
- py{27,36,310}-mode_mitogen-distro_centos{6,7,8},
- py{27,36,310}-mode_mitogen-distro_debian{9,10,11},
- py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004},
+ py{311}-mode_ansible-ansible{2.10,3,4,5},
+ py{312}-mode_ansible-ansible{6},
+ py{27,36,312}-mode_mitogen-distro_centos{6,7,8},
+ py{27,36,312}-mode_mitogen-distro_debian{9,10,11},
+ py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004},
report,
[testenv]
@@ -53,14 +73,16 @@ basepython =
py38: python3.8
py39: python3.9
py310: python3.10
+ py311: python3.11
+ py312: python3.12
deps =
-r{toxinidir}/tests/requirements.txt
mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
ansible2.10: ansible==2.10.7
ansible3: ansible==3.4.0
ansible4: ansible==4.10.0
- ansible5: ansible==5.8.0
- ansible6: ansible==6.0.0
+ ansible5: ansible~=5.0
+ ansible6: ansible~=6.0
install_command =
python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
commands_pre =
@@ -81,7 +103,6 @@ passenv =
HOME
setenv =
# See also azure-pipelines.yml
- ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive
ANSIBLE_STRATEGY = mitogen_linear
NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1
@@ -113,10 +134,26 @@ setenv =
distros_ubuntu1804: DISTROS=ubuntu1804
distros_ubuntu2004: DISTROS=ubuntu2004
mode_ansible: MODE=ansible
+ mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
mode_debops_common: MODE=debops_common
+ mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
mode_mitogen: MODE=mitogen
strategy_linear: ANSIBLE_STRATEGY=linear
+allowlist_externals =
+ # Added: Tox 3.18: Tox 4.0+
+ *_install.py
+ *_tests.py
+ aws
+ docker
+ docker-credential-secretservice
+ echo
+ gpg2
+ pass
whitelist_externals =
+ # Deprecated: Tox 3.18+; Removed: Tox 4.0
+ *_install.py
+ *_tests.py
+ aws
docker
docker-credential-secretservice
echo
@@ -141,7 +178,9 @@ whitelist_externals =
echo
[testenv:docs]
-basepython = python3
+basepython = python3.8
changedir = docs
commands =
sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html
+deps =
+ -r docs/requirements.txt