From 91bf9c36ecce8bcfaff5ed5cdbb82a29103a0e33 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 15 Feb 2026 20:14:21 +0000 Subject: [PATCH 1/4] mitogen: Refactor Importer.ALWAYS_BLACKLIST -> ImportPolicy.unsuitables --- docs/changelog.rst | 2 ++ mitogen/core.py | 37 +++++++++++-------------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 760583121..444fccf5c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ In progress (unreleased) * :gh:issue:`1451` :mod:`mitogen`: Refactor module whitelist & blacklist with module overrides and blocks. Improve error messages for denied modules. +* :gh:issue:`1451` :mod:`mitogen`: Move ``Importer.ALWAYS_BLACKLIST`` + to :attr:`mitogen.core.ImportPolicy.unsuitables` v0.3.41 (2026-02-10) diff --git a/mitogen/core.py b/mitogen/core.py index cfda0a979..f06715e2d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1320,17 +1320,25 @@ class ImportPolicy(object): Prefixes always denied by the responder, only local versions can be used. """ + unsuitables = set([ + '__builtin__', # Python 2.x built-in Imported as __builtins__. + 'builtins', # Python 3.x built-in. Imported as __builtins__. + 'msvcrt', # Windows only. Imported by subprocess in some versions. + 'org', # Jython only. Imported by copy, pickle, & xml.sax. + 'thread', # Python 2.x built-in. Renamed to _thread in 3.x + ]) + if sys.version_info >= (3, 0): unsuitables.add('cStringIO') + def __init__(self, overrides=(), blocks=()): self.overrides = set(overrides) self.blocks = set(blocks) - self._always = set(Importer.ALWAYS_BLACKLIST) def denied(self, fullname): fullnames = frozenset(module_lineage(fullname)) if self.overrides and not self.overrides.intersection(fullnames): return ModuleDeniedByOverridesError - if self.blocks.intersection(fullnames): return ModuleDeniedByBlocksError - if self._always.intersection(fullnames): return ModuleUnsuitableError + if self.blocks & fullnames: return ModuleDeniedByBlocksError + if self.unsuitables & fullnames: return ModuleUnsuitableError return False def denied_raise(self, fullname): @@ -1381,29 +1389,6 @@ class Importer(object): 'utils', ] - ALWAYS_BLACKLIST = [ - # 2.x generates needless imports for 'builtins', while 3.x does the - # same for '__builtin__'. The correct one is built-in, the other always - # a negative round-trip. - 'builtins', - '__builtin__', - - # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries - # to import of this Windows-only builtin module. - 'msvcrt', - - # Python 2.x module that was renamed to _thread in 3.x. - # This entry avoids a roundtrip on 2.x -> 3.x. - 'thread', - - # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but - # very unlikely to trigger a bug report. - 'org', - ] - - if sys.version_info >= (3, 0): - ALWAYS_BLACKLIST += ['cStringIO'] - def __init__(self, router, context, core_src, policy): self._log = logging.getLogger('mitogen.importer') self._context = context From 8c8aada628207e30c9a48b5570134870c5be2657 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 15 Feb 2026 22:57:25 +0000 Subject: [PATCH 2/4] mitogen: Delegate modules in ModulePolicy.unsuitables to Python's importers These are not modules that Mitogen should block import of., the previous name Importer.ALWAYS_BLACKLIST was a misnomer. It conflated their unsuitablility for serving by Mitogen with the concept of modules that should be blocked from import. Mitogen should have no opinion about whether they're imported from a local source. It is attempting to handle their import with Mitogen that we wish to avoid. With this cStringIO (& probably other platform/version specific modules) can be included regardless. --- docs/changelog.rst | 2 ++ mitogen/core.py | 12 +++++++++++- tests/importer_test.py | 12 +++++++++++- tests/responder_test.py | 7 ++++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 444fccf5c..c8e89ab62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ In progress (unreleased) module overrides and blocks. Improve error messages for denied modules. * :gh:issue:`1451` :mod:`mitogen`: Move ``Importer.ALWAYS_BLACKLIST`` to :attr:`mitogen.core.ImportPolicy.unsuitables` +* :gh:issue:`1451` :mod:`mitogen`: Always delegate modules in + :attr:`mitogen.core.ImportPolicy.unsuitables` to Python's own importers v0.3.41 (2026-02-10) diff --git a/mitogen/core.py b/mitogen/core.py index f06715e2d..58878a266 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1323,11 +1323,11 @@ class ImportPolicy(object): unsuitables = set([ '__builtin__', # Python 2.x built-in Imported as __builtins__. 'builtins', # Python 3.x built-in. Imported as __builtins__. + 'cStringIO', # Python 2.x extension 'msvcrt', # Windows only. Imported by subprocess in some versions. 'org', # Jython only. Imported by copy, pickle, & xml.sax. 'thread', # Python 2.x built-in. Renamed to _thread in 3.x ]) - if sys.version_info >= (3, 0): unsuitables.add('cStringIO') def __init__(self, overrides=(), blocks=()): self.overrides = set(overrides) @@ -1348,6 +1348,9 @@ def denied_raise(self, fullname): def overriden(self, fullname): return bool(self.overrides.intersection(module_lineage(fullname))) + def unsuited(self, fullname): + return bool(self.unsuitables.intersection(module_lineage(fullname))) + def __repr__(self): args = (type(self).__name__, self.overrides, self.blocks) return '%s(overrides=%r, blocks=%r)' % args @@ -1472,6 +1475,9 @@ def find_module(self, fullname, path=None): if hasattr(_tls, 'running'): return None + if self.policy.unsuited(fullname): + return None + _tls.running = True try: #_v and self._log.debug('Python requested %r', fullname) @@ -1521,6 +1527,10 @@ def find_spec(self, fullname, path, target=None): if fullname.endswith('.'): return None + if self.policy.unsuited(fullname): + log.debug('Skipping %s. It is unsuited.') + return None + pkgname, _, modname = fullname.rpartition('.') if pkgname and modname not in self._present.get(pkgname, ()): log.debug('Skipping %s. Parent %s has no submodule %s', diff --git a/tests/importer_test.py b/tests/importer_test.py index 7e564779e..935ecdd0a 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -23,7 +23,7 @@ class ImporterMixin(testlib.RouterMixin): def setUp(self): super(ImporterMixin, self).setUp() self.context = mock.Mock() - self.policy = mock.Mock() + self.policy = mitogen.core.ImportPolicy() self.importer = mitogen.core.Importer(self.router, self.context, '', self.policy) # TODO: this is a horrendous hack. Without it, we can't deliver a @@ -309,6 +309,16 @@ def test_overrides_and_blocks(self): self.assertTrue(policy.denied('__builtin__')) self.assertTrue(policy.denied('builtins')) + def test_unsuited(self): + policy = mitogen.core.ImportPolicy( + overrides=['pkg'], + blocks=['pkg'], + ) + self.assertTrue(policy.unsuited('__builtin__')) + self.assertTrue(policy.unsuited('builtins')) + self.assertFalse(policy.unsuited('pkg')) + self.assertFalse(policy.unsuited('otherpkg')) + class Python24LineCacheTest(testlib.TestCase): # TODO: mitogen.core.Importer._update_linecache() diff --git a/tests/responder_test.py b/tests/responder_test.py index da40c43b2..8c27fefa6 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -8,6 +8,7 @@ except ImportError: import mock +import mitogen.core import mitogen.master import testlib @@ -20,7 +21,7 @@ class NeutralizeMainTest(testlib.RouterMixin, testlib.TestCase): def call(self, *args, **kwargs): router = mock.Mock() - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() return self.klass(router, policy).neutralize_main(*args, **kwargs) def test_missing_exec_guard(self): @@ -120,7 +121,7 @@ def test_obviously_missing(self): ) msg.router = router - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() responder = mitogen.master.ModuleResponder(router, policy) responder._on_get_module(msg) self.assertEqual(1, len(router._async_route.mock_calls)) @@ -159,7 +160,7 @@ def test_ansible_six_messed_up_path(self): ) msg.router = router - policy = mock.Mock() + policy = mitogen.core.ImportPolicy() responder = mitogen.master.ModuleResponder(router, policy) responder._on_get_module(msg) self.assertEqual(1, len(router._async_route.mock_calls)) From 51d69425e8fef7e19b1ed1bc00b971717c72a1f0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 16 Feb 2026 01:59:17 +0000 Subject: [PATCH 3/4] mitogen: Add discovered stdlib modules to ImportPolicy.unsuitables --- docs/changelog.rst | 2 ++ mitogen/core.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c8e89ab62..a812abcf8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ In progress (unreleased) to :attr:`mitogen.core.ImportPolicy.unsuitables` * :gh:issue:`1451` :mod:`mitogen`: Always delegate modules in :attr:`mitogen.core.ImportPolicy.unsuitables` to Python's own importers +* :gh:issue:`1451` :mod:`mitogen`: Add discovered stdlib module names to + :attr:`mitogen.core.ImportPolicy.unsuitables` v0.3.41 (2026-02-10) diff --git a/mitogen/core.py b/mitogen/core.py index 58878a266..f675f1789 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -113,6 +113,7 @@ def set_blocking(fd, blocking): now = time.time if sys.version_info >= (3, 0): + from _compat_pickle import IMPORT_MAPPING from pickle import PicklingError, Unpickler as _Unpickler, UnpicklingError def find_deny(module, name): raise UnpicklingError('Denied: %s.%s' % (module, name)) @@ -122,6 +123,7 @@ def __init__(self, file, find_class=find_deny): super().__init__(file, encoding='bytes') else: from cPickle import PicklingError, Unpickler as _Unpickler, UnpicklingError + IMPORT_MAPPING = {} def find_deny(module, name): raise UnpicklingError('Denied: %s.%s' % (module, name)) def Unpickler(file, find_class=find_deny): @@ -1320,7 +1322,7 @@ class ImportPolicy(object): Prefixes always denied by the responder, only local versions can be used. """ - unsuitables = set([ + UNSUITABLES = set([ '__builtin__', # Python 2.x built-in Imported as __builtins__. 'builtins', # Python 3.x built-in. Imported as __builtins__. 'cStringIO', # Python 2.x extension @@ -1332,6 +1334,7 @@ class ImportPolicy(object): def __init__(self, overrides=(), blocks=()): self.overrides = set(overrides) self.blocks = set(blocks) + self.unsuitables = self.UNSUITABLES | self.unsuitables_discovered() def denied(self, fullname): fullnames = frozenset(module_lineage(fullname)) @@ -1348,6 +1351,13 @@ def denied_raise(self, fullname): def overriden(self, fullname): return bool(self.overrides.intersection(module_lineage(fullname))) + @classmethod + def unsuitables_discovered(cls): + names = set(sys.builtin_module_names) | set(IMPORT_MAPPING) + if sys.version_info >= (3, 10): names |= sys.stdlib_module_names + names -= set(['__main__']) + return names + def unsuited(self, fullname): return bool(self.unsuitables.intersection(module_lineage(fullname))) From 3780ddf693d500dfc289f06b7d70af67a0f64e14 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 16 Feb 2026 08:47:01 +0000 Subject: [PATCH 4/4] mitogen: Add controller-discovered modules to ImportPolicy.unsuitable --- docs/changelog.rst | 2 ++ mitogen/core.py | 18 ++++++++++-------- mitogen/parent.py | 11 +++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a812abcf8..ffb8b4863 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,8 @@ In progress (unreleased) :attr:`mitogen.core.ImportPolicy.unsuitables` to Python's own importers * :gh:issue:`1451` :mod:`mitogen`: Add discovered stdlib module names to :attr:`mitogen.core.ImportPolicy.unsuitables` +* :gh:issue:`1451` :mod:`mitogen`: Add modules discovered on the controller + :attr:`mitogen.core.ImportPolicy.unsuitables` v0.3.41 (2026-02-10) diff --git a/mitogen/core.py b/mitogen/core.py index f675f1789..6e73bb10d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1321,6 +1321,10 @@ class ImportPolicy(object): :param blocks: Prefixes always denied by the responder, only local versions can be used. + + :param unsuitables: + Prefixes unsuitable to be served, e.g. because they're Python stdlib, + platform specific. An optimisation to reduce futile round trips. """ UNSUITABLES = set([ '__builtin__', # Python 2.x built-in Imported as __builtins__. @@ -1331,10 +1335,11 @@ class ImportPolicy(object): 'thread', # Python 2.x built-in. Renamed to _thread in 3.x ]) - def __init__(self, overrides=(), blocks=()): + def __init__(self, overrides=(), blocks=(), unsuitables=()): self.overrides = set(overrides) self.blocks = set(blocks) self.unsuitables = self.UNSUITABLES | self.unsuitables_discovered() + self.unsuitables |= set(unsuitables) def denied(self, fullname): fullnames = frozenset(module_lineage(fullname)) @@ -1362,8 +1367,9 @@ def unsuited(self, fullname): return bool(self.unsuitables.intersection(module_lineage(fullname))) def __repr__(self): - args = (type(self).__name__, self.overrides, self.blocks) - return '%s(overrides=%r, blocks=%r)' % args + name = type(self).__name__ + args = (name, self.overrides, self.blocks, self.unsuitables) + return '%s(overrides=%r, blocks=%r, unsuitables=%r)' % (args) class Importer(object): @@ -4207,15 +4213,11 @@ def _setup_importer(self): else: core_src = None - policy = ImportPolicy( - self.config['import_overrides'], - self.config['import_blocks'], - ) importer = Importer( self.router, self.parent, core_src, - policy, + ImportPolicy(*self.config['policy']), ) self.importer = importer diff --git a/mitogen/parent.py b/mitogen/parent.py index 370b1f0aa..715377d7a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1498,14 +1498,17 @@ def get_econtext_config(self): assert self.options.max_message_size is not None parent_ids = mitogen.parent_ids[:] parent_ids.insert(0, mitogen.context_id) - if mitogen.is_master: import_policy = self._router.responder.policy - else: import_policy = self._router.importer.policy + if mitogen.is_master: policy = self._router.responder.policy + else: policy = self._router.importer.policy return { 'parent_ids': parent_ids, 'context_id': self.context.context_id, 'debug': self.options.debug, - 'import_blocks': list(import_policy.blocks), - 'import_overrides': list(import_policy.overrides), + 'policy': ( + list(policy.overrides), + list(policy.blocks), + list(policy.unsuitables), + ), 'profiling': self.options.profiling, 'unidirectional': self.options.unidirectional, 'log_level': get_log_level(),