From b475196ed1e9d78fc3049ed7538884c3bfe7e34d Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Tue, 22 Sep 2026 23:17:29 +0530 Subject: [PATCH] fix(extract): skip the process pool up front when spawn can't re-import __main__ (#3669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under the spawn start method (the Windows default) each worker re-imports the parent's __main__ by its __file__ path. A stdin (`… | python -`), `python -c`, or REPL caller has no such file — the path is missing or `` — so every worker dies during bootstrap and the pool raises BrokenProcessPool before any work is done. The shipped SKILL.md pipes a heredoc into the interpreter, so on Windows this is the path most users take, not an edge case: every skill-driven run prints one full worker traceback per core, then a fallback warning that blames a missing `if __name__ == "__main__":` guard — a fix the caller cannot apply, because there is no file to put a guard in. Detect the unusable-__main__ case before creating the pool: when the start method is spawn (or the platform is Windows) and __main__ has no importable file, run sequentially and print one note that names the real cause (stdin/-c/ REPL) and the remedy (write the step to a .py file, or pass parallel=False). A script that DOES have a __main__ file but omits the guard is a different failure this cannot catch here; it still surfaces via the existing BrokenProcessPool fallback, whose __main__-guard message is now accurate for that remaining case. Tests cover the detector (stdin, REPL-without-__file__, a real script, and the fork start method) and that extract() with >= _PARALLEL_THRESHOLD uncached files and an unusable __main__ never attempts the pool yet still produces correct output. All five fail without the guard. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JJbLfztSxm2tH5cJwBbe9q --- graphify/extract.py | 48 +++++++++++++++++++++++++++ tests/test_extract.py | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index da66f41317..0e572e32a0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6602,6 +6602,34 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: return idx, result +def _spawn_cannot_reimport_main() -> bool: + """True when a spawn-based process pool cannot bootstrap because the caller's + ``__main__`` has no importable file — stdin (``… | python -``), ``python -c``, + or a REPL. + + Under the spawn start method (the Windows default, and macOS since 3.8) each + worker re-imports the parent's ``__main__`` by its ``__file__`` path. For a + stdin/-c/REPL caller that path is missing or bogus (````), so every + worker dies during bootstrap and the pool raises ``BrokenProcessPool`` before + any work is done. The shipped SKILL.md pipes a heredoc into the interpreter, + so on Windows this is the common path, not an edge case — detecting it up + front lets the caller run sequentially without a wall of worker tracebacks + (#3669). A script WITH a real ``__main__`` file but no ``if __name__ == + "__main__"`` guard is a different failure this does not (and cannot) catch + here; that one still surfaces via the ``BrokenProcessPool`` fallback.""" + import multiprocessing + + if ( + multiprocessing.get_start_method(allow_none=True) != "spawn" + and sys.platform != "win32" + ): + return False + import __main__ + + main_file = getattr(__main__, "__file__", None) + return main_file is None or not os.path.isfile(main_file) + + def _extract_parallel( uncached_work: list[tuple[int, Path]], per_file: list[dict | None], @@ -6911,6 +6939,26 @@ def extract( # Phase 2: extract uncached files (parallel or sequential) if uncached_work: + # Skip the pool up front when spawn workers could not re-import our + # __main__ (stdin/-c/REPL) — otherwise every worker dies on bootstrap and + # the run is a wall of BrokenProcessPool tracebacks before falling back to + # the same sequential path anyway. This is exactly what SKILL.md's stdin + # invocation triggers on Windows (#3669). + if ( + parallel + and len(uncached_work) >= _PARALLEL_THRESHOLD + and _spawn_cannot_reimport_main() + ): + print( + " note: running AST extraction sequentially — a parallel pool " + "needs an importable __main__ to relaunch workers, which a stdin " + "(`… | python -`), `python -c`, or REPL invocation does not have. " + "Write the step to a .py file (or pass parallel=False) to silence " + "this.", + file=sys.stderr, + flush=True, + ) + parallel = False ran_parallel = False if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD: ran_parallel = _extract_parallel( diff --git a/tests/test_extract.py b/tests/test_extract.py index 13a64dc457..4af3f694ca 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2250,6 +2250,83 @@ def no_semaphores(*a, **kw): assert "No space left on device" in capsys.readouterr().out, "warning must name the OS error" +def test_spawn_cannot_reimport_main_true_for_stdin_caller(monkeypatch): + """stdin (`… | python -`) leaves __main__.__file__ as a non-file (``), + which spawn workers cannot re-import — the pool is unusable up front (#3669).""" + import multiprocessing + import __main__ + from graphify import extract as extract_mod + + monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn") + monkeypatch.setattr(__main__, "__file__", "", raising=False) + assert extract_mod._spawn_cannot_reimport_main() is True + + +def test_spawn_cannot_reimport_main_true_for_repl_without_file(monkeypatch): + """A REPL / `python -c` __main__ has no __file__ attribute at all.""" + import multiprocessing + import __main__ + from graphify import extract as extract_mod + + monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn") + monkeypatch.delattr(__main__, "__file__", raising=False) + assert extract_mod._spawn_cannot_reimport_main() is True + + +def test_spawn_cannot_reimport_main_false_for_real_script(tmp_path, monkeypatch): + """A normal script whose __main__ is a real file CAN be re-imported, so the + pool is usable and must not be skipped (that case is the common happy path).""" + import multiprocessing + import __main__ + from graphify import extract as extract_mod + + script = tmp_path / "runner.py" + script.write_text("x = 1\n", encoding="utf-8") + monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "spawn") + monkeypatch.setattr(__main__, "__file__", str(script), raising=False) + assert extract_mod._spawn_cannot_reimport_main() is False + + +def test_spawn_cannot_reimport_main_false_under_fork(monkeypatch): + """The fork start method (Linux default) does not re-import __main__, so a + stdin caller is fine and the pool must not be pre-emptively skipped.""" + import multiprocessing + import __main__ + from graphify import extract as extract_mod + + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(multiprocessing, "get_start_method", lambda allow_none=True: "fork") + monkeypatch.setattr(__main__, "__file__", "", raising=False) + assert extract_mod._spawn_cannot_reimport_main() is False + + +def test_extract_skips_pool_up_front_on_unusable_main(tmp_path, monkeypatch, capsys): + """With >= _PARALLEL_THRESHOLD uncached files but an unusable __main__, the + pool is not attempted at all — extract() runs sequentially, producing correct + output with an explanatory note instead of a wall of BrokenProcessPool + tracebacks (#3669).""" + from graphify import extract as extract_mod + + files = [FIXTURES / "sample.py"] * 25 # >= _PARALLEL_THRESHOLD + cache_root = tmp_path / "cache" + cache_root.mkdir() + + calls = {"parallel": 0} + + def fake_parallel(*a, **kw): + calls["parallel"] += 1 + return True + + monkeypatch.setattr(extract_mod, "_extract_parallel", fake_parallel) + monkeypatch.setattr(extract_mod, "_spawn_cannot_reimport_main", lambda: True) + + result = extract_mod.extract(files, cache_root=cache_root) + + assert calls["parallel"] == 0, "the pool must not be attempted when __main__ is unusable" + assert result["nodes"], "sequential extraction must still produce nodes" + assert "sequentially" in capsys.readouterr().err, "must explain the sequential fallback" + + def test_extract_parallel_skips_pool_when_max_workers_is_one(tmp_path, monkeypatch): """#2173: a resolved worker count of 1 must not spawn a ProcessPoolExecutor.