diff --git a/graphify/build.py b/graphify/build.py index e5ca2cdef6..08df617b4a 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -893,6 +893,10 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ): extraction = dict(extraction, hyperedges=extraction["graph"]["hyperedges"]) + degraded = extraction.get("degraded_passes") + if degraded is None and isinstance(extraction.get("graph"), dict): + degraded = extraction["graph"].get("degraded_passes") + # Numeric ids from a loose backend become str before anything keys on them # (#2326) — after the links remap so aliased edges are covered too. _coerce_non_string_ids(extraction) @@ -1476,6 +1480,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat f"will be emptied on the next export.", file=sys.stderr, ) + if degraded: + G.graph["degraded_passes"] = list(degraded) # Runs LAST, after the alias-competition above (which relies on file-node # labels still being bare basenames): give colliding-basename file nodes a # directory-qualified display label so lookup/discovery can disambiguate @@ -1512,12 +1518,23 @@ def build( """ from graphify.dedup import deduplicate_entities combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + degraded_passes: list[dict] = [] for ext in extractions: combined["nodes"].extend(ext.get("nodes", [])) combined["edges"].extend(ext.get("edges", [])) combined["hyperedges"].extend(ext.get("hyperedges", [])) combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) + if ext.get("degraded_passes"): + degraded_passes.extend(ext["degraded_passes"]) + if degraded_passes: + seen_passes: set[str] = set() + deduped: list[dict] = [] + for dp in degraded_passes: + if isinstance(dp, dict) and dp.get("pass") and dp["pass"] not in seen_passes: + seen_passes.add(dp["pass"]) + deduped.append(dp) + combined["degraded_passes"] = deduped _root = str(Path(root).resolve()) if root else None if dedup and combined["nodes"]: # Numeric ids must be str before dedup, which keys on them and would @@ -1613,8 +1630,8 @@ def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dic return deduped_nodes, deduped_edges -def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | None": - """Load (nodes, edges, hyperedges, directed) from an existing graph.json for +def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool, list] | None": + """Load (nodes, edges, hyperedges, directed, degraded_passes) from an existing graph.json for an incremental merge, accepting both the ``links`` and ``edges`` spellings. Reads the JSON directly instead of going through node_link_graph(). @@ -1655,11 +1672,16 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N for item in edges: if isinstance(item, dict): item.setdefault("_origin", "ast" if _is_ast_tier(item) else "semantic") + degraded = data.get("degraded_passes") + if degraded is None and isinstance(data.get("graph"), dict): + degraded = data["graph"].get("degraded_passes") + degraded_passes = list(degraded) if isinstance(degraded, list) else [] return ( nodes, edges, list(data.get("hyperedges", [])), bool(data.get("directed", False)), + degraded_passes, ) @@ -1770,7 +1792,7 @@ def merge_raw_extraction( loaded = _load_existing_graph(graph_path) if loaded is None: return new - existing_nodes, existing_edges, existing_hyperedges, _ = loaded + existing_nodes, existing_edges, existing_hyperedges, _, existing_degraded = loaded _eff_root = ( str(Path(root).resolve()) if root is not None @@ -1870,6 +1892,33 @@ def _dropped(item: dict) -> bool: new["hyperedges"] = carried_hyper + list(new.get("hyperedges", [])) if unverified_semantic_shrink: new["_unverified_semantic_shrink"] = unverified_semantic_shrink + + surviving_ast_suffixes: set[str] = set() + for n in existing_nodes: + if isinstance(n, dict) and _is_ast_tier(n) and not _dropped(n): + sf = n.get("source_file") + if sf: + sfx = Path(sf).suffix.lower() + if sfx: + surviving_ast_suffixes.add(sfx) + + fresh_degraded = list(new.get("degraded_passes", [])) + fresh_pass_names = {d["pass"] for d in fresh_degraded if isinstance(d, dict) and "pass" in d} + reconciled_degraded = [] + for dp in existing_degraded: + if not isinstance(dp, dict) or "pass" not in dp: + continue + if dp["pass"] in fresh_pass_names: + continue + dp_suffixes = {s.lower() for s in dp.get("suffixes", [])} + if dp_suffixes & surviving_ast_suffixes: + reconciled_degraded.append(dp) + reconciled_degraded.extend(fresh_degraded) + reconciled_degraded.sort(key=lambda d: d.get("pass", "")) + if reconciled_degraded: + new["degraded_passes"] = reconciled_degraded + else: + new.pop("degraded_passes", None) return new @@ -1913,13 +1962,14 @@ def build_merge( graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) _loaded = _load_existing_graph(graph_path) if _loaded is not None: - existing_nodes, existing_edges, existing_hyperedges, existing_directed = _loaded + existing_nodes, existing_edges, existing_hyperedges, existing_directed, existing_degraded = _loaded had_graph = True else: existing_nodes = [] existing_edges = [] existing_hyperedges = [] existing_directed = False + existing_degraded = [] had_graph = False if directed is None: directed = existing_directed if had_graph else False @@ -2307,6 +2357,41 @@ def _explained(n: dict) -> bool: f"Pass prune_sources explicitly if you intend to remove them. (#479)" ) + surviving_ast_suffixes: set[str] = set() + for n in existing_nodes: + if isinstance(n, dict) and _is_ast_tier(n): + sf = n.get("source_file") + if sf and not _prune_match(sf): + sfx = Path(sf).suffix.lower() + if sfx: + surviving_ast_suffixes.add(sfx) + + fresh_degraded: list[dict] = [] + for ch in new_chunks: + if isinstance(ch, dict) and "degraded_passes" in ch: + fresh_degraded.extend(ch.get("degraded_passes") or []) + + if had_graph: + fresh_pass_names = {d["pass"] for d in fresh_degraded if isinstance(d, dict) and "pass" in d} + reconciled_degraded = [] + for dp in existing_degraded: + if not isinstance(dp, dict) or "pass" not in dp: + continue + if dp["pass"] in fresh_pass_names: + continue + dp_suffixes = {s.lower() for s in dp.get("suffixes", [])} + if dp_suffixes & surviving_ast_suffixes: + reconciled_degraded.append(dp) + reconciled_degraded.extend(fresh_degraded) + else: + reconciled_degraded = list(fresh_degraded) + reconciled_degraded.sort(key=lambda d: d.get("pass", "")) + + if reconciled_degraded: + G.graph["degraded_passes"] = reconciled_degraded + elif hasattr(G, "graph") and "degraded_passes" in G.graph: + del G.graph["degraded_passes"] + if unverified_semantic_shrink: G.graph["_unverified_semantic_shrink"] = unverified_semantic_shrink diff --git a/graphify/cli.py b/graphify/cli.py index 969d7d3c87..c844c22ce2 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -4200,6 +4200,8 @@ def _progress(idx: int, total: int, _result: dict) -> None: "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), "extracted_sources": list(ast_result.get("extracted_sources", [])), } + if ast_result.get("degraded_passes"): + merged["degraded_passes"] = list(ast_result["degraded_passes"]) graph_json_path = graphify_out / "graph.json" analysis_path = graphify_out / ".graphify_analysis.json" diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index 7e3baea6cd..ca8a2070e0 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -259,6 +259,12 @@ def diagnose_extraction( Path(extract_path) if extract_path else Path(__file__).with_name("extract.py") ) + degraded_passes = extraction.get("degraded_passes") + if degraded_passes is None and isinstance(extraction.get("graph"), dict): + degraded_passes = extraction["graph"].get("degraded_passes") + if not isinstance(degraded_passes, list): + degraded_passes = [] + return { "node_count": len(node_ids), "unverified_node_count": unverified_node_count, @@ -289,6 +295,7 @@ def diagnose_extraction( "post_build_error": build_error, "producer_suppression": scan_producer_suppression_sites(suppression_path), "examples": examples, + "degraded_passes": degraded_passes, } @@ -394,7 +401,16 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"post_build_graph_type: {summary['post_build_graph_type']}", f"post_build_edges: {summary['post_build_edge_count']}", f"producer_suppression_sites: {suppression.get('total_sites', 0)}", + f"degraded_resolution_passes: {len(summary.get('degraded_passes', []))}", ] + if summary.get("degraded_passes"): + lines.append("degraded_resolution_pass_details:") + for item in summary["degraded_passes"]: + p = item.get("pass", "unknown") + err_type = item.get("error_type", "Exception") + err = item.get("error", "") + sfx = ", ".join(item.get("suffixes", [])) + lines.append(f" - {p} ({err_type}: {err}) suffixes=[{sfx}]") if summary.get("post_build_error"): lines.append(f"post_build_error: {summary['post_build_error']}") if suppression.get("error"): diff --git a/graphify/export.py b/graphify/export.py index 43c9b9adb7..f626735712 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -404,6 +404,15 @@ def _canonical(item: dict, lead: tuple[str, ...]) -> dict: if isinstance(data.get("graph"), dict) and "hyperedges" in data["graph"]: data["graph"]["hyperedges"] = hyperedges data["hyperedges"] = hyperedges + + # Prevent dual persistence: exclude degraded_passes from serialized + # data["graph"] without mutating G.graph, and assign to top-level + # data["degraded_passes"] only when non-empty. + degraded = getattr(G, "graph", {}).get("degraded_passes") + if isinstance(data.get("graph"), dict): + data["graph"] = {k: v for k, v in data["graph"].items() if k != "degraded_passes"} + if degraded: + data["degraded_passes"] = sorted(degraded, key=lambda d: d.get("pass", "")) # Fallback provenance comes from the repo the graph is being written INTO # (output_path lives in /graphify-out/), never the shell's cwd — # the same cwd-anchoring mistake #2316 fixed for `update`. diff --git a/graphify/extract.py b/graphify/extract.py index da66f41317..c2dc0c5e9d 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -5135,6 +5135,20 @@ def _resolve_kotlin_member_calls( }) +def _record_degraded_pass( + degraded_passes: list[dict], + name: str, + suffixes: Any, + exc: Exception, +) -> None: + degraded_passes.append({ + "pass": name, + "error": str(exc), + "error_type": type(exc).__name__, + "suffixes": sorted(suffixes), + }) + + # Kotlin import-target resolution runs EARLY (directly in extract(), before the # shared call pass builds its import-evidence index) — registering it in the # tail registry would rewrite the targets after promotion already read them. @@ -6892,6 +6906,7 @@ def extract( # cache directory's location diverges from it. cache_location = (cache_root if cache_root is not None else Path(".")).resolve() total = len(paths) + degraded_passes: list[dict] = [] # Phase 1: separate cached hits from uncached work per_file: list[dict | None] = [None] * total @@ -7557,6 +7572,7 @@ def _learn(e: dict) -> None: except Exception as exc: import logging logging.getLogger(__name__).warning("PHP type-reference resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "php_type_references", _php_exts, exc) # Java package/import disambiguation must likewise run BEFORE the rewire # (#2504): an EXTERNAL import (`org.springframework.stereotype.Component`) # leaves a bare `Component` stub that the rewire would collapse onto the only @@ -7572,6 +7588,7 @@ def _learn(e: dict) -> None: except Exception as exc: import logging logging.getLogger(__name__).warning("Java type-reference resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "java_type_references", [".java"], exc) # Resolve internal Go pkg.Type references exactly and park external ones # before the generic bare-label stub rewire can manufacture a collision. _go_sel = [(r, p) for r, p in zip(per_file, paths) if p.suffix == ".go"] @@ -7587,6 +7604,7 @@ def _learn(e: dict) -> None: logging.getLogger(__name__).warning( "Go type-reference resolution failed, skipping: %s", exc ) + _record_degraded_pass(degraded_passes, "go_type_references", [".go"], exc) # Cross-file Python import resolution and type-reference repointing (#3252) py_paths = [p for p in paths if p.suffix == ".py"] if py_paths: @@ -7597,6 +7615,7 @@ def _learn(e: dict) -> None: except Exception as exc: import logging logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "python_imports", [".py"], exc) _rewire_unique_stub_nodes(all_nodes, all_edges) # Cross-file Java import resolution @@ -7608,6 +7627,7 @@ def _learn(e: dict) -> None: except Exception as exc: import logging logging.getLogger(__name__).warning("Java cross-file import resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "java_cross_file_imports", [".java"], exc) # Cross-file C# type-reference resolution: re-point dangling inherits/implements/ # references edges left on shadow stubs, disambiguating same-named types by the @@ -7621,11 +7641,13 @@ def _learn(e: dict) -> None: except Exception as exc: import logging logging.getLogger(__name__).warning("C# type-reference resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "csharp_type_references", _DOTNET_TYPE_EXTS, exc) try: _resolve_cross_file_csharp_imports(cs_results, cs_paths, all_nodes, all_edges) except Exception as exc: import logging logging.getLogger(__name__).warning("C# cross-file import resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "csharp_cross_file_imports", _DOTNET_TYPE_EXTS, exc) # Cross-file Bash source-backed call resolution: a call to a function defined # in a file this one `source`s is left unresolved by the per-file extractor @@ -7670,6 +7692,7 @@ def _looks_like_bash(result: object) -> bool: except Exception as exc: import logging logging.getLogger(__name__).warning("Bash cross-file call resolution failed, skipping: %s", exc) + _record_degraded_pass(degraded_passes, "bash_source_edges", [".bash", ".sh"], exc) # Cross-file call resolution for all languages # Each extractor saved unresolved calls in raw_calls. Now that we have all @@ -7739,6 +7762,7 @@ def _looks_like_bash(result: object) -> bool: run_language_resolvers( paths, per_file, all_nodes, all_edges, resolvers=[_KOTLIN_IMPORT_TARGET_RESOLVER], + degraded_passes=degraded_passes, ) # Build evidence index from import edges so cross-file calls backed by an @@ -8034,11 +8058,17 @@ def _has_import_evidence(candidate_id: str) -> bool: _rl_nodes = list(resolution_nodes) _rl_edges = all_edges + list(resolution_context_edges or []) _n0, _e0 = len(_rl_nodes), len(_rl_edges) - run_language_resolvers(paths, per_file, _rl_nodes, _rl_edges) + run_language_resolvers( + paths, per_file, _rl_nodes, _rl_edges, + degraded_passes=degraded_passes, + ) all_nodes.extend(_rl_nodes[_n0:]) all_edges.extend(_rl_edges[_e0:]) else: - run_language_resolvers(paths, per_file, all_nodes, all_edges) + run_language_resolvers( + paths, per_file, all_nodes, all_edges, + degraded_passes=degraded_passes, + ) # Relativize source_file fields so paths are portable across machines (#555). # When the node's id was itself minted from the absolute path, remap it to a @@ -8244,6 +8274,7 @@ def _canon(nid: str) -> str: # merge_raw_extraction know which files were genuinely re-extracted # rather than guessing ownership from node["source_file"] (#3411). "extracted_sources": [str(p) for p in paths], + "degraded_passes": degraded_passes, } diff --git a/graphify/report.py b/graphify/report.py index ac854a2e64..2fbbd394a3 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -276,6 +276,21 @@ def _real_count(nodes) -> int: conf_tag = f"{conf} {cscore:.2f}" if cscore is not None else conf lines.append(f"- **{h.get('label', h.get('id', ''))}** — {node_labels} [{conf_tag}]") + degraded_passes = G.graph.get("degraded_passes", []) + if degraded_passes: + lines += [ + "", + "## Degraded Resolution Passes", + "> [!WARNING]", + "> One or more cross-file resolution passes failed during extraction. Call edges or type references for the affected languages may be incomplete.", + ] + for dp in degraded_passes: + pass_name = dp.get("pass", "unknown") + err = dp.get("error", "") + err_type = dp.get("error_type", "Exception") + sfx = ", ".join(dp.get("suffixes", [])) + lines.append(f"- **{pass_name}** ({err_type}: {err}): suffixes `{sfx}`") + lines += ["", f"## Communities ({len(communities)} total, {thin_count_summary} thin omitted)"] for cid, nodes in communities.items(): label = community_labels.get(cid, f"Community {cid}") diff --git a/graphify/resolver_registry.py b/graphify/resolver_registry.py index b17478a78a..c1b20577e2 100644 --- a/graphify/resolver_registry.py +++ b/graphify/resolver_registry.py @@ -63,6 +63,7 @@ def run_language_resolvers( all_edges: list[dict], *, resolvers: Sequence[LanguageResolver] | None = None, + degraded_passes: list[dict] | None = None, ) -> None: """Run every resolver whose suffix appears in ``paths``. @@ -73,6 +74,7 @@ def run_language_resolvers( ``resolvers`` defaults to the global registry; tests pass an explicit list to exercise the driver in isolation. + If ``degraded_passes`` is provided (a list), failures are appended to it. """ active = _REGISTRY if resolvers is None else resolvers suffixes_present = {p.suffix for p in paths} @@ -83,3 +85,10 @@ def run_language_resolvers( resolver.resolve(per_file, all_nodes, all_edges) except Exception as exc: _LOG.warning("%s resolution failed, skipping: %s", resolver.name, exc) + if degraded_passes is not None: + degraded_passes.append({ + "pass": resolver.name, + "error": str(exc), + "error_type": type(exc).__name__, + "suffixes": sorted(resolver.suffixes), + }) diff --git a/graphify/watch.py b/graphify/watch.py index 3105f322a5..3900a6844a 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -798,6 +798,11 @@ def _reconcile_existing_graph( # staying fail-closed. existing = json.loads(existing_graph.read_text(encoding="utf-8")) existing_graph_data = existing + existing_degraded = existing.get("degraded_passes") + if existing_degraded is None and isinstance(existing.get("graph"), dict): + existing_degraded = existing["graph"].get("degraded_passes") + if not isinstance(existing_degraded, list): + existing_degraded = [] # Backfill tier provenance on legacy items (#2334), mirroring # build._load_existing_graph (this reconcile path loads the raw dict @@ -1033,13 +1038,38 @@ def _ignored_now(identity: str) -> bool: for item in preserved_nodes + preserved_edges + preserved_hyperedges: source_paths.rebase_preserved(item) - return { + fresh_degraded = list(result.get("degraded_passes", [])) + if full_rebuild: + reconciled_degraded = fresh_degraded + else: + fresh_passes = {d["pass"] for d in fresh_degraded if isinstance(d, dict) and "pass" in d} + reconciled_degraded = [] + for dp in existing_degraded: + if not isinstance(dp, dict) or "pass" not in dp: + continue + if dp["pass"] in fresh_passes: + continue + dp_suffixes = {s.lower() for s in dp.get("suffixes", [])} + # Check if any live code file matching suffix was NOT in extract_targets + has_unextracted = any( + p.suffix.lower() in dp_suffixes and p not in extract_targets + for p in code_files + ) + if has_unextracted: + reconciled_degraded.append(dp) + reconciled_degraded.extend(fresh_degraded) + reconciled_degraded.sort(key=lambda d: d.get("pass", "")) + + out_dict = { "nodes": result["nodes"] + preserved_nodes, "edges": result["edges"] + preserved_edges, "hyperedges": result.get("hyperedges", []) + preserved_hyperedges, "input_tokens": 0, "output_tokens": 0, - }, existing_graph_data + } + if reconciled_degraded: + out_dict["degraded_passes"] = reconciled_degraded + return out_dict, existing_graph_data except Exception as exc: # Post-load reconciliation failure: fall back to the fresh extraction # while keeping the loaded baseline, so _check_shrink still guards the @@ -1145,6 +1175,15 @@ def _canonical_topology_for_compare(graph_data: dict) -> dict: key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), ) + degraded_passes = canonical.get("degraded_passes") + if isinstance(degraded_passes, list): + canonical["degraded_passes"] = sorted( + degraded_passes, + key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), + ) + if isinstance(canonical.get("graph"), dict) and "degraded_passes" in canonical["graph"]: + canonical["graph"] = {k: v for k, v in canonical["graph"].items() if k != "degraded_passes"} + return canonical @@ -1155,6 +1194,11 @@ def _topology_from_graph(G) -> dict: except TypeError: data = json_graph.node_link_data(G) data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) + degraded = getattr(G, "graph", {}).get("degraded_passes") + if isinstance(data.get("graph"), dict): + data["graph"] = {k: v for k, v in data["graph"].items() if k != "degraded_passes"} + if degraded: + data["degraded_passes"] = sorted(degraded, key=lambda d: d.get("pass", "")) return data diff --git a/tests/test_build.py b/tests/test_build.py index c496873cbe..669dbf85c2 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1985,6 +1985,87 @@ def test_build_merge_explicit_ast_sources_argument(tmp_path): assert "nuget_pkg" in G1, "ast_sources explicit arg must protect undispatched A.csproj" +def test_build_merge_reconciles_degraded_passes(tmp_path): + root = tmp_path / "corpus" + root.mkdir() + graph_path = root / "graph.json" + existing_data = { + "nodes": [ + {"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}, + {"id": "b_g", "label": "g", "source_file": "b.py", "_origin": "ast"}, + ], + "edges": [], + "degraded_passes": [ + {"pass": "python_imports", "error": "err", "error_type": "RuntimeError", "suffixes": [".py"]} + ], + } + graph_path.write_text(json.dumps(existing_data), encoding="utf-8") + + # Re-extract only a.py: b.py survives, so python_imports is preserved + chunk_partial = { + "nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], + "extracted_sources": ["a.py"], + "degraded_passes": [], + } + G_partial = build_merge([chunk_partial], graph_path, dedup=False, root=root) + assert "degraded_passes" in G_partial.graph + assert G_partial.graph["degraded_passes"][0]["pass"] == "python_imports" + + # Re-extract both a.py and b.py: all .py sources re-extracted cleanly, so degraded_passes cleared + chunk_full = { + "nodes": [ + {"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}, + {"id": "b_g", "label": "g", "source_file": "b.py", "_origin": "ast"}, + ], + "extracted_sources": ["a.py", "b.py"], + "degraded_passes": [], + } + G_full = build_merge([chunk_full], graph_path, dedup=False, root=root) + assert "degraded_passes" not in G_full.graph + + +def test_merge_raw_extraction_reconciles_degraded_passes(tmp_path): + from graphify.build import merge_raw_extraction + root = tmp_path / "corpus" + root.mkdir() + graph_path = root / "graph.json" + existing_data = { + "nodes": [ + {"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}, + {"id": "b_g", "label": "g", "source_file": "b.py", "_origin": "ast"}, + ], + "edges": [], + "degraded_passes": [ + {"pass": "python_imports", "error": "err", "error_type": "RuntimeError", "suffixes": [".py"]} + ], + } + graph_path.write_text(json.dumps(existing_data), encoding="utf-8") + + # Partial re-extract keeps degraded_passes + new_partial = { + "nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], + "edges": [], + "extracted_sources": ["a.py"], + "degraded_passes": [], + } + merged_partial = merge_raw_extraction(new_partial, graph_path, root=root) + assert "degraded_passes" in merged_partial + assert merged_partial["degraded_passes"][0]["pass"] == "python_imports" + + # Full re-extract clears degraded_passes + new_full = { + "nodes": [ + {"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}, + {"id": "b_g", "label": "g", "source_file": "b.py", "_origin": "ast"}, + ], + "edges": [], + "extracted_sources": ["a.py", "b.py"], + "degraded_passes": [], + } + merged_full = merge_raw_extraction(new_full, graph_path, root=root) + assert "degraded_passes" not in merged_full + + def test_build_annotations_all_resolve(): # build.py imports every annotated name at module scope -- no TYPE_CHECKING-only # names -- so an unresolvable hint here means a missing import, not a lazy one. diff --git a/tests/test_export.py b/tests/test_export.py index 725f458431..5d76e158e0 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1197,3 +1197,37 @@ def test_to_html_spiral_seed_uses_a_real_map_index(): "spiral seed references `i` but the node map has no index param " "-> ReferenceError: i is not defined (#3699)" ) + + +def test_to_json_persists_degraded_passes_at_top_level_and_omits_from_graph(tmp_path): + import networkx as nx + G = nx.Graph() + G.add_node("a", label="A") + G.graph["degraded_passes"] = [ + {"pass": "python_imports", "error": "syntax err", "error_type": "SyntaxError", "suffixes": [".py"]} + ] + out_file = tmp_path / "graph.json" + to_json(G, {0: ["a"]}, str(out_file)) + + data = json.loads(out_file.read_text(encoding="utf-8")) + assert "degraded_passes" in data + assert data["degraded_passes"] == [ + {"pass": "python_imports", "error": "syntax err", "error_type": "SyntaxError", "suffixes": [".py"]} + ] + assert "degraded_passes" not in data.get("graph", {}) + + # Verify round-trip through build_from_json + G2 = build_from_json(data) + assert G2.graph.get("degraded_passes") == data["degraded_passes"] + + +def test_to_json_omits_degraded_passes_when_clean(tmp_path): + import networkx as nx + G = nx.Graph() + G.add_node("a", label="A") + out_file = tmp_path / "graph.json" + to_json(G, {0: ["a"]}, str(out_file)) + + data = json.loads(out_file.read_text(encoding="utf-8")) + assert "degraded_passes" not in data + assert "degraded_passes" not in data.get("graph", {}) diff --git a/tests/test_extract.py b/tests/test_extract.py index 13a64dc457..d3ca68b064 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -4698,3 +4698,29 @@ def test_3252_metadata_preservation(tmp_path): assert param_ref["source_location"] == "L2" assert node_by_id[param_ref["target"]]["label"] == "User" assert node_by_id[param_ref["target"]]["source_file"] == "models.py" + + +def test_extract_captures_degraded_passes_on_hand_wired_failure(monkeypatch, tmp_path): + import graphify.extract as extractmod + + def _broken_resolver(*args, **kwargs): + raise RuntimeError("fake resolver crash") + + monkeypatch.setattr(extractmod, "_resolve_cross_file_imports", _broken_resolver) + f = tmp_path / "hello.py" + f.write_text("x = 1\n", encoding="utf-8") + res = extractmod.extract([f], root=tmp_path, cache_root=tmp_path) + assert len(res.get("degraded_passes", [])) == 1 + dp = res["degraded_passes"][0] + assert dp["pass"] == "python_imports" + assert dp["error"] == "fake resolver crash" + assert dp["error_type"] == "RuntimeError" + assert dp["suffixes"] == [".py"] + + +def test_extract_clean_has_empty_degraded_passes(tmp_path): + import graphify.extract as extractmod + f = tmp_path / "hello.py" + f.write_text("x = 1\n", encoding="utf-8") + res = extractmod.extract([f], root=tmp_path, cache_root=tmp_path) + assert res.get("degraded_passes") == [] diff --git a/tests/test_language_resolvers.py b/tests/test_language_resolvers.py index 787c1d505d..072535e51c 100644 --- a/tests/test_language_resolvers.py +++ b/tests/test_language_resolvers.py @@ -72,3 +72,29 @@ def _add_edge(per_file, all_nodes, all_edges): edges: list[dict] = [] run_language_resolvers([Path("a.rb")], [], [], edges, resolvers=resolvers) assert edges == [{"source": "x", "target": "y", "relation": "calls"}] + + +def test_failing_resolver_records_degraded_pass() -> None: + def _boom(per_file, all_nodes, all_edges): + raise TypeError("resolver type error") + + resolvers = [LanguageResolver("boom", frozenset({".rb", ".rake"}), _boom)] + degraded: list[dict] = [] + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=resolvers, degraded_passes=degraded) + assert len(degraded) == 1 + assert degraded[0] == { + "pass": "boom", + "error": "resolver type error", + "error_type": "TypeError", + "suffixes": [".rake", ".rb"], + } + + +def test_successful_resolver_does_not_record_degraded_pass() -> None: + def _ok(per_file, all_nodes, all_edges): + pass + + resolvers = [LanguageResolver("ok", frozenset({".rb"}), _ok)] + degraded: list[dict] = [] + run_language_resolvers([Path("a.rb")], [], [], [], resolvers=resolvers, degraded_passes=degraded) + assert degraded == [] diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 0206a0c13b..a496cfdc02 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -489,3 +489,32 @@ def test_diagnose_multigraph_cli_rejects_conflicting_direction_flags( assert exc_info.value.code == 1 assert "--directed and --undirected are mutually exclusive" in capsys.readouterr().err + + +def test_diagnose_extraction_captures_degraded_passes(): + payload = _diagnostic_fixture() + payload["degraded_passes"] = [ + { + "pass": "go_type_references", + "error": "panic in resolver", + "error_type": "RuntimeError", + "suffixes": [".go"], + } + ] + summary = diagnose_extraction(payload) + assert len(summary["degraded_passes"]) == 1 + assert summary["degraded_passes"][0]["pass"] == "go_type_references" + + report = format_diagnostic_report(summary) + assert "degraded_resolution_passes: 1" in report + assert "degraded_resolution_pass_details:" in report + assert "go_type_references (RuntimeError: panic in resolver) suffixes=[.go]" in report + + +def test_diagnose_extraction_handles_clean_degraded_passes(): + payload = _diagnostic_fixture() + summary = diagnose_extraction(payload) + assert summary["degraded_passes"] == [] + report = format_diagnostic_report(summary) + assert "degraded_resolution_passes: 0" in report + assert "degraded_resolution_pass_details:" not in report diff --git a/tests/test_report.py b/tests/test_report.py index 98f256907d..77321ec85b 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -201,3 +201,25 @@ def test_report_hubs_use_wikilinks_when_obsidian(): labels = {cid: f"Widget {cid}" for cid in communities} report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1, obsidian=True) assert "[[_COMMUNITY_" in report + + +def test_report_renders_degraded_resolution_passes_warning(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + G.graph["degraded_passes"] = [ + { + "pass": "python_imports", + "error": "syntax failure in module", + "error_type": "SyntaxError", + "suffixes": [".py"], + } + ] + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Degraded Resolution Passes" in report + assert "> [!WARNING]" in report + assert "**python_imports** (SyntaxError: syntax failure in module): suffixes `.py`" in report + + +def test_report_omits_degraded_resolution_passes_when_clean(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Degraded Resolution Passes" not in report diff --git a/tests/test_watch.py b/tests/test_watch.py index 7e1b01b58d..bb06525f1f 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -4932,3 +4932,134 @@ def test_requires_symlinks_rebuild_symlink_worker(requires_symlinks, tmp_path, c data = json.loads(graph_path.read_text(encoding="utf-8")) labels = {n.get("label") for n in data["nodes"]} assert "run_worker()" in labels + + +def test_reconcile_existing_graph_preserves_degraded_passes_when_unreextracted(tmp_path): + from graphify.watch import _reconcile_existing_graph + + existing_graph = tmp_path / "graph.json" + existing_payload = { + "nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], + "links": [], + "degraded_passes": [ + {"pass": "python_imports", "error": "err", "error_type": "RuntimeError", "suffixes": [".py"]} + ], + } + existing_graph.write_text(json.dumps(existing_payload), encoding="utf-8") + + code_files = [tmp_path / "a.py", tmp_path / "b.js"] + extract_targets = [tmp_path / "b.js"] + fresh_result = {"nodes": [{"id": "b_g", "label": "g", "source_file": "b.js", "_origin": "ast"}], "edges": []} + + reconciled, _ = _reconcile_existing_graph( + existing_graph, + fresh_result, + out=tmp_path, + project_root=tmp_path, + watch_root=tmp_path, + code_files=code_files, + extract_targets=extract_targets, + full_rebuild=False, + deleted_paths=set(), + deleted_source_identities=set(), + ) + assert "degraded_passes" in reconciled + assert reconciled["degraded_passes"][0]["pass"] == "python_imports" + + +def test_reconcile_existing_graph_clears_recovered_degraded_passes(tmp_path): + from graphify.watch import _reconcile_existing_graph + + existing_graph = tmp_path / "graph.json" + existing_payload = { + "nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], + "links": [], + "degraded_passes": [ + {"pass": "python_imports", "error": "err", "error_type": "RuntimeError", "suffixes": [".py"]} + ], + } + existing_graph.write_text(json.dumps(existing_payload), encoding="utf-8") + + code_files = [tmp_path / "a.py"] + extract_targets = [tmp_path / "a.py"] + fresh_result = {"nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], "edges": []} + + reconciled, _ = _reconcile_existing_graph( + existing_graph, + fresh_result, + out=tmp_path, + project_root=tmp_path, + watch_root=tmp_path, + code_files=code_files, + extract_targets=extract_targets, + full_rebuild=False, + deleted_paths=set(), + deleted_source_identities=set(), + ) + assert "degraded_passes" not in reconciled + + +def test_reconcile_existing_graph_full_rebuild_clears_stale_degraded_passes(tmp_path): + from graphify.watch import _reconcile_existing_graph + + existing_graph = tmp_path / "graph.json" + existing_payload = { + "nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], + "links": [], + "degraded_passes": [ + {"pass": "python_imports", "error": "err", "error_type": "RuntimeError", "suffixes": [".py"]} + ], + } + existing_graph.write_text(json.dumps(existing_payload), encoding="utf-8") + + code_files = [tmp_path / "a.py", tmp_path / "b.py"] + extract_targets = [tmp_path / "a.py", tmp_path / "b.py"] + fresh_result = {"nodes": [{"id": "a_f", "label": "f", "source_file": "a.py", "_origin": "ast"}], "edges": []} + + reconciled, _ = _reconcile_existing_graph( + existing_graph, + fresh_result, + out=tmp_path, + project_root=tmp_path, + watch_root=tmp_path, + code_files=code_files, + extract_targets=extract_targets, + full_rebuild=True, + deleted_paths=set(), + deleted_source_identities=set(), + ) + assert "degraded_passes" not in reconciled + + +def test_topology_from_graph_and_canonical_topology_match_existing_degraded_graph(tmp_path): + """Unchanged degraded graphs must match candidate topology to short-circuit rebuilds.""" + import networkx as nx + from graphify.export import to_json + from graphify.watch import _canonical_topology_for_compare, _topology_from_graph + + degraded_entry = [{"pass": "foo", "error": "bar", "error_type": "ValueError", "suffixes": [".py"]}] + G = nx.Graph() + G.add_node("a", label="A") + G.graph["degraded_passes"] = degraded_entry + + candidate = _topology_from_graph(G) + assert "degraded_passes" in candidate + assert candidate["degraded_passes"] == degraded_entry + assert "degraded_passes" not in candidate.get("graph", {}) + + graph_file = tmp_path / "graph.json" + to_json(G, {}, str(graph_file)) + existing_graph_data = json.loads(graph_file.read_text(encoding="utf-8")) + + assert ( + json.dumps(_canonical_topology_for_compare(existing_graph_data), sort_keys=True) + == json.dumps(_canonical_topology_for_compare(candidate), sort_keys=True) + ) + + # If existing graph was clean, they must not match so rebuild triggers + clean_existing = dict(existing_graph_data) + del clean_existing["degraded_passes"] + assert ( + json.dumps(_canonical_topology_for_compare(clean_existing), sort_keys=True) + != json.dumps(_canonical_topology_for_compare(candidate), sort_keys=True) + )