diff options
| -rw-r--r-- | ansible_mitogen/planner.py | 74 | ||||
| -rw-r--r-- | ansible_mitogen/runner.py | 28 | ||||
| -rw-r--r-- | ansible_mitogen/target.py | 16 | ||||
| -rw-r--r-- | debian/changelog | 6 | ||||
| -rw-r--r-- | docs/changelog.rst | 14 | ||||
| -rw-r--r-- | docs/getting_started.rst | 8 | ||||
| -rw-r--r-- | mitogen/__init__.py | 2 | ||||
| -rw-r--r-- | tests/ansible/hosts/group_vars/alma9.yml | 2 | ||||
| -rw-r--r-- | tests/ansible/hosts/group_vars/debian10.yml | 6 | ||||
| -rw-r--r-- | tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml | 9 | ||||
| -rwxr-xr-x | tests/ansible/regression/issue_776__load_plugins_called_twice.yml | 37 | ||||
| -rw-r--r-- | tests/image_prep/host_vars/debian10.yml | 4 | ||||
| -rw-r--r-- | tests/image_prep/roles/package_manager_module/defaults/main.yml | 1 | ||||
| -rw-r--r-- | tests/image_prep/roles/package_manager_module/tasks/main.yml | 5 | ||||
| -rw-r--r-- | tests/image_prep/roles/package_manager_module/templates/install_deps.sh.j2 | 15 | ||||
| -rw-r--r-- | tox.ini | 5 |
16 files changed, 187 insertions, 45 deletions
diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 5d11de3..e93401e 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -38,6 +38,7 @@ from __future__ import absolute_import, division, print_function from __future__ import unicode_literals __metaclass__ = type +import inspect import json import logging import os @@ -55,6 +56,7 @@ import mitogen.service import ansible_mitogen.loaders import ansible_mitogen.parsing import ansible_mitogen.target +import ansible_mitogen.utils import ansible_mitogen.utils.unsafe @@ -507,6 +509,39 @@ _planners = [ ] +class _DummyDnfModule: + """ + Placeholder containing methods to be into Ansible's DNF module. + """ + def _execute_dnf_script(self, command, config, params=None): + """ + Run ansible.module_utils._embed.dnf.* using a Mitogen context, + instead of ansible.module_utils.embed.EmbedManager. + """ + params = params or {} + python_executable = self._interpreter or sys.executable + + router = sys.modules['__main__'].ansible_mitogen_injected_router + context = router.local(python_path=python_executable) + + if command == 'list': + list_command = params.get('list_command') + if not list_command: + return { + 'failed': True, + 'msg': 'No list_command specified for list operation', + } + return context.call(_embed_dnf.list_items, list_command) + + if command == 'ensure': + return context.call(_embed_dnf.ensure, config, params) + + if command == 'update-cache': + return context.call(_embed_dnf.update_cache_only, config) + + return {'failed': True, 'msg': 'Unknown command: %s' % (command,)} + + def py_modname_from_path(name, path): """ Fetch the logical name of a new-style module as it might appear in @@ -636,7 +671,7 @@ def _fix_py35(invocation, module_source): invocation._overridden_sources[invocation.module_path] = module_source -def _fix_dnf(invocation, module_source): +def _fix_dnf_import(invocation, module_source): """ Handles edge case where dnf ansible module showed failure due to a missing import in the dnf module. Specifically addresses errors like "Failed loading plugin 'debuginfo-install': module 'dnf' has no attribute 'cli'". @@ -653,6 +688,40 @@ def _fix_dnf(invocation, module_source): invocation._overridden_sources[invocation.module_path] = module_source +def _fix_dnf_ansible_core_2_21_embed(invocation): + """ + Ansible 14 (ansible-core 2.21) adds ansible.module_utils.embed.EmbedManager, + as atech preview to embed files in ansiballz. The DNF module uses it to + embed ansible.module_utils._embed.dnf and runs `{python} -m ..._embed.dnf` + on the target. See https://github.com/ansible/ansible/pull/86432. + + Mitogen has no ansiballz, so we replace Ansible's implementation of + ansible.modules.dnf.DnfModule._execute_dnf_script() with one that calls + ansible.module_utils._embed.dnf.ensure() et al through a Mitogen context. + """ + if ( + ansible_mitogen.utils.ansible_version[:2] !=(2, 21) + or invocation.module_name not in {'ansible.builtin.dnf', 'ansible.legacy.dnf', 'dnf'} + ): + return + + module_source = invocation._overridden_sources[invocation.module_path] + module_source = module_source.replace( + b'from ansible.module_utils.embed import EmbedManager\n', + b'', + 1, + ).replace( + b"dnfscript = EmbedManager.embed('..module_utils._embed', 'dnf.py')\n", + b'from ansible.module_utils._embed import dnf as _embed_dnf\n', + 1, + ).replace( + b' def _execute_dnf_script(self, command, config, params=None):\n', + inspect.getsource(_DummyDnfModule._execute_dnf_script).encode('ascii'), + 1, + ) + invocation._overridden_sources[invocation.module_path] = module_source + + def _load_collections(invocation): """ Special loader that ensures that `ansible_collections` exist as a module path for import @@ -690,7 +759,8 @@ def invoke(invocation): module_source = invocation.get_module_source() _fix_py35(invocation, module_source) - _fix_dnf(invocation, module_source) + _fix_dnf_import(invocation, module_source) + _fix_dnf_ansible_core_2_21_embed(invocation) _planner_by_path[invocation.module_path] = _get_planner( invocation, module_source diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 0de32ad..a66d14b 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -60,6 +60,7 @@ else: import ansible.module_utils.common.warnings import mitogen.core +import mitogen.parent import ansible_mitogen.target # TODO: circular import from mitogen.core import to_text @@ -71,6 +72,17 @@ try: except ImportError: from io import StringIO +# Ansible >= 2.7 ansible.module_utils.basic uses `import __main__` (PR #41749). +# Under vanilla Ansible it is benign: +# - controller: __main__ is a CLI module (e.g. ansible.cli.playbook) +# - target: __main__ is the executing Ansible module (e.g. command, dnf) +# Mitogen + Ansible refuses `import __main__` by mitogen.core.Importer policy. +# So inject a (temporary) placeholder. +if '__main__' not in sys.modules: + sys.modules['__main__'] = types.ModuleType( + 'ansible_mitogen.injected.i_cant_believe_its_not__main__', + ) + # Prevent accidental import of an Ansible module from hanging on stdin read. # FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None. import ansible.module_utils.basic @@ -993,11 +1005,6 @@ class NewStyleRunner(ScriptRunner): True, # dont_inherit )) - if sys.version_info >= (3, 0): - main_module_name = '__main__' - else: - main_module_name = b'__main__' - def _handle_magic_exception(self, mod, exc): """ Beginning with Ansible >2.6, some modules (file.py) install a @@ -1036,7 +1043,7 @@ class NewStyleRunner(ScriptRunner): return pkg.encode() def _run(self): - mod = types.ModuleType(self.main_module_name) + mod = types.ModuleType('__main__') mod.__package__ = self._get_module_package() # Some Ansible modules use __file__ to find the Ansiballz temporary # directory. We must provide some temporary path in __file__, but we @@ -1047,6 +1054,15 @@ class NewStyleRunner(ScriptRunner): 'ansible_module_' + os.path.basename(self.path), ) + if sys.version_info >= (3, 4): + mod.__spec__ = importlib.machinery.ModuleSpec( + self.py_module_name, + self, # FIXME Not an importlib Finder/Loader + ) + + mitogen.parent.upgrade_router(self.econtext) + mod.ansible_mitogen_injected_router = self.econtext.router + sys.modules['__main__'] = mod code = self._get_code() rc = 2 try: diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index e4e38d2..fba17c6 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -50,19 +50,11 @@ import subprocess import sys import tempfile import traceback -import types import mitogen.core import mitogen.parent import mitogen.service -# Ansible since PR #41749 inserts "import __main__" into -# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so -# we must setup a fake "__main__" before that module is ever imported. The -# str() is to cast Unicode to bytes on Python 2.6. -if not sys.modules.get(str('__main__')): - sys.modules[str('__main__')] = types.ModuleType(str('__main__')) - import ansible.module_utils.json_utils import ansible_mitogen.runner @@ -376,7 +368,8 @@ def spawn_isolated_child(econtext): return context -def run_module(kwargs): +@mitogen.core.takes_econtext +def run_module(kwargs, econtext): """ Set up the process environment in preparation for running an Ansible module. This monkey-patches the Ansible libraries in various places to @@ -385,7 +378,7 @@ def run_module(kwargs): """ runner_name = kwargs.pop('runner_name') klass = getattr(ansible_mitogen.runner, runner_name) - impl = klass(**mitogen.core.Kwargs(kwargs)) + impl = klass(econtext=econtext, **mitogen.core.Kwargs(kwargs)) return impl.run() @@ -449,10 +442,9 @@ class AsyncRunner(object): def _run_module(self): kwargs = dict(self.kwargs, **{ 'detach': True, - 'econtext': self.econtext, 'emulate_tty': False, }) - return run_module(kwargs) + return run_module(kwargs, self.econtext) def _parse_result(self, dct): filtered, warnings = ( diff --git a/debian/changelog b/debian/changelog index 3ecd919..9bf135b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-mitogen (0.3.50-1) unstable; urgency=medium + + * New upstream release. + + -- Stefano Rivera <stefanor@debian.org> Wed, 24 Jun 2026 08:51:28 -0400 + python-mitogen (0.3.49-1) unstable; urgency=medium * New upstream release. diff --git a/docs/changelog.rst b/docs/changelog.rst index 5baa72f..6921c41 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,20 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_. +v0.3.50 (2026-06-19) +-------------------- + +* :gh:issue:`1529` :mod:`ansible_mitogen`: Use Mitogen context for embedded DNF + script with Ansible 14 (ansible-core 2.21) +* :gh:issue:`1529` :mod:`ansible_mitogen`: Make injected ``__main__`` on target + more identifiable/greppable +* :gh:issue:`1529` :mod:`ansible_mitogen`: Remove + :attr:`ansible_mitogen.runner.NewStyleRunner.main_module_name` +* :gh:issue:`1523` tests: Bump Ansible 14 tests to 14.x +* :gh:issue:`1529` tests: Cleanup after issue 109 regression test +* :gh:issue:`1529` tests: Enable issue 776 regression test for recent Ansibles + + v0.3.49 (2026-06-03) -------------------- diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 12056c5..21f6f7f 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -341,13 +341,13 @@ The following built-in types may be used as parameters or return values in remote procedure calls: * :class:`bool` -* :func:`bytes` (:class:`str` on Python 2.x) +* :class:`bytes` (:class:`str` on Python 2.x) * :class:`dict` * :class:`int` -* :func:`list` +* :class:`list` * :class:`long` -* :func:`tuple` -* :func:`unicode` (:class:`str` on Python 3.x) +* :class:`str` (:class:`unicode` on Python 2.x) +* :class:`tuple` User-defined types may not be used, except for: diff --git a/mitogen/__init__.py b/mitogen/__init__.py index f9c2832..3438671 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 49) +__version__ = (0, 3, 50) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/tests/ansible/hosts/group_vars/alma9.yml b/tests/ansible/hosts/group_vars/alma9.yml index 25fd10b..ee6da0d 100644 --- a/tests/ansible/hosts/group_vars/alma9.yml +++ b/tests/ansible/hosts/group_vars/alma9.yml @@ -1,4 +1,4 @@ -pkg_mgr_python_interpreter: python3 +pkg_mgr_python_interpreter: /usr/libexec/platform-python # Alma Linux 9, RHEL 9, etc. lack a virtualenv package virtualenv_create_argv: diff --git a/tests/ansible/hosts/group_vars/debian10.yml b/tests/ansible/hosts/group_vars/debian10.yml index c6c0a26..dd85f44 100644 --- a/tests/ansible/hosts/group_vars/debian10.yml +++ b/tests/ansible/hosts/group_vars/debian10.yml @@ -1,4 +1,10 @@ +package_manager_module_packages: + - python-apt + - python3-apt + package_manager_repos: - dest: /etc/apt/sources.list content: | deb http://archive.debian.org/debian buster main contrib non-free + +pkg_mgr_python_interpreter: python3 diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml index 550a7f4..c547c96 100644 --- a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml +++ b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml @@ -4,6 +4,8 @@ - name: regression/issue_109__target_has_old_ansible_installed.yml hosts: test-targets gather_facts: true + vars: + naughty_ansible_dest: "{{ ansible_user_dir }}/ansible.py" tasks: - meta: end_play when: @@ -11,8 +13,8 @@ - name: Copy the naughty ansible into place copy: - dest: "{{ansible_user_dir}}/ansible.py" src: ansible.py + dest: "{{ naughty_ansible_dest }}" # Restart the connection. - meta: reset_connection @@ -33,5 +35,10 @@ - name: Run some new-style from ansible.module_utils... modules stat: path: / + + - name: Clean up the naughty ansible + file: + path: "{{ naughty_ansible_dest }}" + state: absent tags: - issue_109 diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index 4a4cf7b..b772975 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -1,31 +1,36 @@ # https://github.com/mitogen-hq/mitogen/issues/776 --- -- name: regression/issue_776__load_plugins_called_twice.yml +- name: regression/issue_776__load_plugins_called_twice.yml, config hosts: test-targets become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" # Delayed until after the end_play, due to Python version requirements gather_facts: false + roles: + - role: package_manager tags: - issue_776 - vars: - ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" - package: rsync # Chosen to exist in all tested distros/package managers - pre_tasks: - # The package management modules require using the same Python version - # as the target's package manager libraries. This is sometimes in conflict - # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not - # support Python 2.x on targets. - - meta: end_play - when: - - ansible_version_major_minor is version('2.17', '>=', strict=True) +- name: regression/issue_776__load_plugins_called_twice.yml, setup + hosts: test-targets + become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" + strategy: linear # Required for running raw module + # Delayed until after the end_play, due to Python version requirements + gather_facts: false roles: - - role: package_manager + - package_manager_module + tags: + - issue_776 +- name: regression/issue_776__load_plugins_called_twice.yml, tests + hosts: test-targets + become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" + gather_facts: true + tags: + - issue_776 + vars: + ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" + package: rsync # Chosen to exist in all tested distros/package managers tasks: - - name: Gather facts manually - setup: - - name: Update package index apt: update_cache: true diff --git a/tests/image_prep/host_vars/debian10.yml b/tests/image_prep/host_vars/debian10.yml index 1000043..ad3b80a 100644 --- a/tests/image_prep/host_vars/debian10.yml +++ b/tests/image_prep/host_vars/debian10.yml @@ -16,6 +16,10 @@ packages: - sudo - virtualenv +package_manager_module_packages: + - python-apt + - python3-apt + package_manager_repos: - dest: /etc/apt/sources.list content: | diff --git a/tests/image_prep/roles/package_manager_module/defaults/main.yml b/tests/image_prep/roles/package_manager_module/defaults/main.yml new file mode 100644 index 0000000..47144d7 --- /dev/null +++ b/tests/image_prep/roles/package_manager_module/defaults/main.yml @@ -0,0 +1 @@ +package_manager_module_packages: [] diff --git a/tests/image_prep/roles/package_manager_module/tasks/main.yml b/tests/image_prep/roles/package_manager_module/tasks/main.yml new file mode 100644 index 0000000..7c9d1eb --- /dev/null +++ b/tests/image_prep/roles/package_manager_module/tasks/main.yml @@ -0,0 +1,5 @@ +- name: Install package manager module dependencies + raw: "{{ lookup('template', 'install_deps.sh.j2') }}" + when: + - (package_manager_module_packages | length) > 0 + changed_when: true diff --git a/tests/image_prep/roles/package_manager_module/templates/install_deps.sh.j2 b/tests/image_prep/roles/package_manager_module/templates/install_deps.sh.j2 new file mode 100644 index 0000000..833e756 --- /dev/null +++ b/tests/image_prep/roles/package_manager_module/templates/install_deps.sh.j2 @@ -0,0 +1,15 @@ +set -o errexit +set -o nounset + +{% if (package_manager_module_packages | length) > 0 %} +if command -v apt-get; then + apt-get -y update + apt-get -y --no-install-recommends install {{ package_manager_module_packages | join(' ') }} +elif command -v dnf; then + dnf -y install {{ package_manager_module_packages | join(' ') }} +elif command -v yum; then + yum -y install {{ package_manager_module_packages | join(' ') }} +else + exit 43 +fi +{% endif %} @@ -78,7 +78,7 @@ deps = ans11: ansible~=11.0 ans12: ansible~=12.0 ans13: ansible~=13.0 - ans14: ansible>=14.0.0rc1 + ans14: ansible~=14.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -120,7 +120,8 @@ setenv = ans12: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2604-py3 # Ansible >= 13 (ansible-core >= 2.20) requires Python >= 3.9 on targets ans13: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2604-py3 - # Ansible 14.0rc1 (ansible-core 2.21.0) does not support sudo-rs + # Ansible 14.0 (ansible-core 2.21.0) does not support sudo-rs + # https://github.com/mitogen-hq/mitogen/issues/1530 ans14: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2404-py3 m_ans: ANSIBLE_SKIP_TAGS=resource_intensive m_lcl: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive |
