mitogen: Refactor Importer.ALWAYS_BLACKLIST -> ImportPolicy.unsuitables

Populating ImportPolicy.unsuitables via ModuleReponder on the controller will
allow the controller to dynamically add unsuitable prefixes it is aware of.

Author: moreati

mitogen/core.py

@@ -1319,18 +1319,22 @@ class ImportPolicy(object):
     :param blocks:
         Prefixes always denied by the responder, only local versions can be
         used.
+
+    :param unsuitables:
+        Prefixes unsuitable to be served, e.g. because they're Python stdlib,
+        platform specific. An optimisation to reduce futile round trips.
     """
-    def __init__(self, overrides=(), blocks=()):
+    def __init__(self, overrides=(), blocks=(), unsuitables=()):
         self.overrides = set(overrides)
         self.blocks = set(blocks)
-        self._always = set(Importer.ALWAYS_BLACKLIST)
+        self.unsuitables = set(unsuitables)
 
     def denied(self, fullname):
         fullnames = frozenset(module_lineage(fullname))
         if self.overrides and not self.overrides.intersection(fullnames):
             return ModuleDeniedByOverridesError
-        if self.blocks.intersection(fullnames): return ModuleDeniedByBlocksError
-        if self._always.intersection(fullnames): return ModuleUnsuitableError
+        if self.blocks & fullnames: return ModuleDeniedByBlocksError
+        if self.unsuitables & fullnames: return ModuleUnsuitableError
         return False
 
     def denied_raise(self, fullname):
@@ -1381,29 +1385,6 @@ class Importer(object):
         'utils',
     ]
 
-    ALWAYS_BLACKLIST = [
-        # 2.x generates needless imports for 'builtins', while 3.x does the
-        # same for '__builtin__'. The correct one is built-in, the other always
-        # a negative round-trip.
-        'builtins',
-        '__builtin__',
-
-        # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries
-        # to import of this Windows-only builtin module.
-        'msvcrt',
-
-        # Python 2.x module that was renamed to _thread in 3.x.
-        # This entry avoids a roundtrip on 2.x -> 3.x.
-        'thread',
-
-        # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
-        # very unlikely to trigger a bug report.
-        'org',
-    ]
-
-    if sys.version_info >= (3, 0):
-        ALWAYS_BLACKLIST += ['cStringIO']
-
     def __init__(self, router, context, core_src, policy):
         self._log = logging.getLogger('mitogen.importer')
         self._context = context
@@ -4202,15 +4183,11 @@ def _setup_importer(self):
             else:
                 core_src = None
 
-            policy = ImportPolicy(
-                self.config['import_overrides'],
-                self.config['import_blocks'],
-            )
             importer = Importer(
                 self.router,
                 self.parent,
                 core_src,
-                policy,
+                ImportPolicy(*self.config['policy']),
             )
 
         self.importer = importer

mitogen/imports/__init__.py

@@ -36,3 +36,20 @@ def codeobj_imports(co):
         * `names`: tuple of names in `from mod import ..`.
     """
     return _code_imports(co.co_code, co.co_consts, co.co_names)
+
+
+def unsuitables():
+    """
+    Return module prefixes that are known to be unsuitable for ModuleResponder
+    to serve, due to e.g. legacy, platform specific, stdlib.
+    """
+    prefixes = {
+        '__builtin__',  # Python 2.x built-in Imported as __builtins__.
+        'builtins',     # Python 3.x built-in. Imported as __builtins__.
+        'cPickle',      # Python 2.x extension
+        'cStringIO',    # Python 2.x extension
+        'msvcrt',       # Windows only. Imported by subprocess in some versions.
+        'org',          # Jython only. Imported by copy, pickle, & xml.sax.
+        'thread',       # Python 2.x built-in. Renamed to _thread in 3.x
+    }
+    return prefixes

mitogen/master.py

@@ -1402,7 +1402,10 @@ def __init__(self, broker=None, max_message_size=None):
 
     def upgrade(self):
         self.id_allocator = IdAllocator(self)
-        self.responder = ModuleResponder(self, mitogen.core.ImportPolicy())
+        policy = mitogen.core.ImportPolicy(
+            unsuitables=mitogen.imports.unsuitables(),
+        )
+        self.responder = ModuleResponder(self, policy)
         self.resource_responder = ResourceResponder(self)
         self.log_forwarder = LogForwarder(self)
         self.route_monitor = mitogen.parent.RouteMonitor(router=self)

mitogen/parent.py

@@ -1498,14 +1498,13 @@ def get_econtext_config(self):
         assert self.options.max_message_size is not None
         parent_ids = mitogen.parent_ids[:]
         parent_ids.insert(0, mitogen.context_id)
-        if mitogen.is_master: import_policy = self._router.responder.policy
-        else: import_policy = self._router.importer.policy
+        if mitogen.is_master: policy = self._router.responder.policy
+        else: policy = self._router.importer.policy
         return {
             'parent_ids': parent_ids,
             'context_id': self.context.context_id,
             'debug': self.options.debug,
-            'import_blocks': list(import_policy.blocks),
-            'import_overrides': list(import_policy.overrides),
+            'policy': (policy.overrides, policy.blocks, policy.unsuitables),
             'profiling': self.options.profiling,
             'unidirectional': self.options.unidirectional,
             'log_level': get_log_level(),

tests/importer_test.py

@@ -11,6 +11,7 @@
 
 import mitogen.core
 import mitogen.utils
+import mitogen.imports
 from mitogen.core import b
 
 import testlib
@@ -274,7 +275,9 @@ def test_sys_module_not_fetched(self):
 
 class ImportPolicyTest(testlib.TestCase):
     def test_default(self):
-        policy = mitogen.core.ImportPolicy()
+        policy = mitogen.core.ImportPolicy(
+            unsuitables=mitogen.imports.unsuitables(),
+        )
         self.assertFalse(policy.denied('pkg'))
         self.assertFalse(policy.denied('pkg.mod'))
         self.assertFalse(policy.denied('otherpkg'))
@@ -283,7 +286,10 @@ def test_default(self):
         self.assertTrue(policy.denied('builtins'))
 
     def test_overrides(self):
-        policy = mitogen.core.ImportPolicy(overrides=['pkg'])
+        policy = mitogen.core.ImportPolicy(
+            overrides=['pkg'],
+            unsuitables=mitogen.imports.unsuitables(),
+        )
         self.assertFalse(policy.denied('pkg'))
         self.assertFalse(policy.denied('pkg.mod'))
         self.assertTrue(policy.denied('otherpkg'))
@@ -292,7 +298,10 @@ def test_overrides(self):
         self.assertTrue(policy.denied('builtins'))
 
     def test_blocks(self):
-        policy = mitogen.core.ImportPolicy(blocks=['pkg'])
+        policy = mitogen.core.ImportPolicy(
+            blocks=['pkg'],
+            unsuitables=mitogen.imports.unsuitables(),
+        )
         self.assertTrue(policy.denied('pkg'))
         self.assertTrue(policy.denied('pkg.mod'))
         self.assertFalse(policy.denied('otherpkg'))
@@ -301,7 +310,11 @@ def test_blocks(self):
         self.assertTrue(policy.denied('builtins'))
 
     def test_overrides_and_blocks(self):
-        policy = mitogen.core.ImportPolicy(overrides=['pkg'], blocks=['pkg'])
+        policy = mitogen.core.ImportPolicy(
+            overrides=['pkg'],
+            blocks=['pkg'],
+            unsuitables=mitogen.imports.unsuitables(),
+        )
         self.assertTrue(policy.denied('pkg'))
         self.assertTrue(policy.denied('pkg.mod'))
         self.assertTrue(policy.denied('otherpkg'))

tests/imports_test.py

@@ -128,3 +128,9 @@ def test_scoped_try_except(self):
             (level, 'in_except_exception', ('x', 'z')),
         ]
         self.assertEqual(list(self.func(co)), expected)
+
+
+class UnsuitablesTest(testlib.TestCase):
+    def test_unsuitables(self):
+        unsuitables = mitogen.imports.unsuitables()
+        self.assertTrue({'__builtin__', 'builtins'} & unsuitables)