fix(extract): skip the process pool up front when spawn can't re-import __main__ (#3669)

graphify/extract.py

@@ -6602,6 +6602,34 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]:
     return idx, result
 
 
+def _spawn_cannot_reimport_main() -> bool:
+    """True when a spawn-based process pool cannot bootstrap because the caller's
+    ``__main__`` has no importable file — stdin (``… | python -``), ``python -c``,
+    or a REPL.
+
+    Under the spawn start method (the Windows default, and macOS since 3.8) each
+    worker re-imports the parent's ``__main__`` by its ``__file__`` path. For a
+    stdin/-c/REPL caller that path is missing or bogus (``<stdin>``), so every
+    worker dies during bootstrap and the pool raises ``BrokenProcessPool`` before
+    any work is done. The shipped SKILL.md pipes a heredoc into the interpreter,
+    so on Windows this is the common path, not an edge case — detecting it up
+    front lets the caller run sequentially without a wall of worker tracebacks
+    (#3669). A script WITH a real ``__main__`` file but no ``if __name__ ==
+    "__main__"`` guard is a different failure this does not (and cannot) catch
+    here; that one still surfaces via the ``BrokenProcessPool`` fallback."""
+    import multiprocessing
+
+    if (
+        multiprocessing.get_start_method(allow_none=True) != "spawn"
+        and sys.platform != "win32"
+    ):
+        return False
+    import __main__
+
+    main_file = getattr(__main__, "__file__", None)
+    return main_file is None or not os.path.isfile(main_file)
+
+
 def _extract_parallel(
     uncached_work: list[tuple[int, Path]],
     per_file: list[dict | None],
@@ -6911,6 +6939,26 @@ def extract(
 
     # Phase 2: extract uncached files (parallel or sequential)
     if uncached_work:
+        # Skip the pool up front when spawn workers could not re-import our
+        # __main__ (stdin/-c/REPL) — otherwise every worker dies on bootstrap and
+        # the run is a wall of BrokenProcessPool tracebacks before falling back to
+        # the same sequential path anyway. This is exactly what SKILL.md's stdin
+        # invocation triggers on Windows (#3669).
+        if (
+            parallel
+            and len(uncached_work) >= _PARALLEL_THRESHOLD
+            and _spawn_cannot_reimport_main()
+        ):
+            print(
+                "  note: running AST extraction sequentially — a parallel pool "
+                "needs an importable __main__ to relaunch workers, which a stdin "
+                "(`… | python -`), `python -c`, or REPL invocation does not have. "
+                "Write the step to a .py file (or pass parallel=False) to silence "
+                "this.",
+                file=sys.stderr,
+                flush=True,
+            )
+            parallel = False
         ran_parallel = False
         if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD:
             ran_parallel = _extract_parallel(

tests/test_extract.py

@@ -2250,6 +2250,83 @@ def no_semaphores(*a, **kw):
     assert "No space left on device" in capsys.readouterr().out, "warning must name the OS error"
 
 
+def test_spawn_cannot_reimport_main_true_for_stdin_caller(monkeypatch):
+    """stdin (`… | python -`) leaves __main__.__file__ as a non-file (`<stdin>`),
+    which spawn workers cannot re-import — the pool is unusable up front (#3669)."""
+    import multiprocessing
+    import __main__
+    from graphify import extract as extract_mod
+
+    monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn")
+    monkeypatch.setattr(__main__, "__file__", "<stdin>", raising=False)
+    assert extract_mod._spawn_cannot_reimport_main() is True
+
+
+def test_spawn_cannot_reimport_main_true_for_repl_without_file(monkeypatch):
+    """A REPL / `python -c` __main__ has no __file__ attribute at all."""
+    import multiprocessing
+    import __main__
+    from graphify import extract as extract_mod
+
+    monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn")
+    monkeypatch.delattr(__main__, "__file__", raising=False)
+    assert extract_mod._spawn_cannot_reimport_main() is True
+
+
+def test_spawn_cannot_reimport_main_false_for_real_script(tmp_path, monkeypatch):
+    """A normal script whose __main__ is a real file CAN be re-imported, so the
+    pool is usable and must not be skipped (that case is the common happy path)."""
+    import multiprocessing
+    import __main__
+    from graphify import extract as extract_mod
+
+    script = tmp_path / "runner.py"
+    script.write_text("x = 1\n", encoding="utf-8")
+    monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn")
+    monkeypatch.setattr(__main__, "__file__", str(script), raising=False)
+    assert extract_mod._spawn_cannot_reimport_main() is False
+
+
+def test_spawn_cannot_reimport_main_false_under_fork(monkeypatch):
+    """The fork start method (Linux default) does not re-import __main__, so a
+    stdin caller is fine and the pool must not be pre-emptively skipped."""
+    import multiprocessing
+    import __main__
+    from graphify import extract as extract_mod
+
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "fork")
+    monkeypatch.setattr(__main__, "__file__", "<stdin>", raising=False)
+    assert extract_mod._spawn_cannot_reimport_main() is False
+
+
+def test_extract_skips_pool_up_front_on_unusable_main(tmp_path, monkeypatch, capsys):
+    """With >= _PARALLEL_THRESHOLD uncached files but an unusable __main__, the
+    pool is not attempted at all — extract() runs sequentially, producing correct
+    output with an explanatory note instead of a wall of BrokenProcessPool
+    tracebacks (#3669)."""
+    from graphify import extract as extract_mod
+
+    files = [FIXTURES / "sample.py"] * 25  # >= _PARALLEL_THRESHOLD
+    cache_root = tmp_path / "cache"
+    cache_root.mkdir()
+
+    calls = {"parallel": 0}
+
+    def fake_parallel(*a, **kw):
+        calls["parallel"] += 1
+        return True
+
+    monkeypatch.setattr(extract_mod, "_extract_parallel", fake_parallel)
+    monkeypatch.setattr(extract_mod, "_spawn_cannot_reimport_main", lambda: True)
+
+    result = extract_mod.extract(files, cache_root=cache_root)
+
+    assert calls["parallel"] == 0, "the pool must not be attempted when __main__ is unusable"
+    assert result["nodes"], "sequential extraction must still produce nodes"
+    assert "sequentially" in capsys.readouterr().err, "must explain the sequential fallback"
+
+
 def test_extract_parallel_skips_pool_when_max_workers_is_one(tmp_path, monkeypatch):
     """#2173: a resolved worker count of 1 must not spawn a ProcessPoolExecutor.