Merge pull request #112 from voidborne-d/fix/large-graph-quadratic-aggregations

fix(dashboard): O(N+K) per-layer aggregations, kill quadratic Array.includes (#102)

Author: Lum1104

understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs

@@ -0,0 +1,99 @@
+// Per-layer aggregation perf benchmark.
+//
+// Mirrors the BEFORE shape (graph.nodes.filter(n => layer.nodeIds.includes(n.id))
+// per layer) and the AFTER shape (single nodesById Map + iterate layer.nodeIds)
+// from `useOverviewGraph` in `src/components/GraphView.tsx`. Issue #102 reported
+// a 4.8 MB knowledge graph that froze the dashboard on overview render — the
+// quadratic Array.includes pass was the dominant synchronous cost.
+//
+// We can't import the dashboard helper directly (Vite-bundled, no
+// per-module dist), so the new shape is reproduced here in lockstep with
+// `src/utils/layerStats.ts::computeLayerStats`.
+//
+// Usage:
+//   node understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs
+
+import { performance } from "node:perf_hooks";
+
+function makeGraph(layerCount, nodesPerLayer) {
+  const nodes = [];
+  const layers = [];
+  for (let li = 0; li < layerCount; li++) {
+    const ids = [];
+    for (let ni = 0; ni < nodesPerLayer; ni++) {
+      const id = `n-${li}-${ni}`;
+      const complexity = ["simple", "moderate", "complex"][(li + ni) % 3];
+      nodes.push({ id, complexity });
+      ids.push(id);
+    }
+    layers.push({ id: `L${li}`, nodeIds: ids });
+  }
+  return { nodes, layers };
+}
+
+// --- BEFORE: O(N × K × L) per overview render ----------------------------
+function aggregateBefore(graph) {
+  const out = [];
+  for (const layer of graph.layers) {
+    const memberNodes = graph.nodes.filter((n) => layer.nodeIds.includes(n.id));
+    const c = { simple: 0, moderate: 0, complex: 0 };
+    for (const n of memberNodes) c[n.complexity]++;
+    const aggregate =
+      c.complex > memberNodes.length * 0.3
+        ? "complex"
+        : c.moderate > memberNodes.length * 0.3
+          ? "moderate"
+          : "simple";
+    out.push({ id: layer.id, aggregateComplexity: aggregate });
+  }
+  return out;
+}
+
+// --- AFTER: O(N + Σ K_i) per overview render ----------------------------
+function aggregateAfter(graph, nodesById) {
+  const out = [];
+  for (const layer of graph.layers) {
+    const c = { simple: 0, moderate: 0, complex: 0 };
+    let resolved = 0;
+    for (const nid of layer.nodeIds) {
+      const node = nodesById.get(nid);
+      if (!node) continue;
+      resolved++;
+      c[node.complexity]++;
+    }
+    const aggregate =
+      c.complex > resolved * 0.3
+        ? "complex"
+        : c.moderate > resolved * 0.3
+          ? "moderate"
+          : "simple";
+    out.push({ id: layer.id, aggregateComplexity: aggregate });
+  }
+  return out;
+}
+
+function bench(label, layerCount, nodesPerLayer) {
+  const graph = makeGraph(layerCount, nodesPerLayer);
+  const nodesById = new Map(graph.nodes.map((n) => [n.id, n]));
+
+  const t0 = performance.now();
+  const before = aggregateBefore(graph);
+  const t1 = performance.now();
+  const after = aggregateAfter(graph, nodesById);
+  const t2 = performance.now();
+
+  const beforeMs = t1 - t0;
+  const afterMs = t2 - t1;
+  const speedup = afterMs > 0 ? beforeMs / afterMs : Infinity;
+  const parity = JSON.stringify(before) === JSON.stringify(after);
+  console.log(
+    `${label} (${layerCount} layers × ${nodesPerLayer} nodes = ${graph.nodes.length} total): ` +
+      `BEFORE ${beforeMs.toFixed(1)}ms | AFTER ${afterMs.toFixed(1)}ms | ` +
+      `${speedup.toFixed(1)}× faster | parity ${parity}`,
+  );
+}
+
+bench("small", 10, 50);
+bench("medium", 30, 100);
+bench("large", 50, 200);
+bench("issue#102 shape", 100, 200);

understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx

@@ -20,6 +20,7 @@ function downloadBlob(blob: Blob, filename: string) {
 
 export default function ExportMenu() {
   const graph = useDashboardStore((s) => s.graph);
+  const nodeIdToLayerIds = useDashboardStore((s) => s.nodeIdToLayerIds);
   const filters = useDashboardStore((s) => s.filters);
   const exportMenuOpen = useDashboardStore((s) => s.exportMenuOpen);
   const toggleExportMenu = useDashboardStore((s) => s.toggleExportMenu);
@@ -187,7 +188,7 @@ export default function ExportMenu() {
         ? graph.nodes.filter((n) => !subFileTypes.has(n.type))
         : graph.nodes;
 
-      filteredGraphNodes = filterNodes(filteredGraphNodes, graph.layers ?? [], filters);
+      filteredGraphNodes = filterNodes(filteredGraphNodes, nodeIdToLayerIds, filters);
       const filteredNodeIds = new Set(filteredGraphNodes.map((n) => n.id));
 
       let filteredGraphEdges = graph.edges.filter(

understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx

@@ -51,6 +51,7 @@ import {
 } from "../utils/edgeAggregation";
 import { deriveContainers } from "../utils/containers";
 import type { DerivedContainer } from "../utils/containers";
+import { computeLayerStats } from "../utils/layerStats";
 
 const nodeTypes = {
   custom: CustomNode,
@@ -139,6 +140,8 @@ function SelectedNodeFitView() {
 
 function useOverviewGraph() {
   const graph = useDashboardStore((s) => s.graph);
+  const nodesById = useDashboardStore((s) => s.nodesById);
+  const nodeIdToLayerId = useDashboardStore((s) => s.nodeIdToLayerId);
   const searchResults = useDashboardStore((s) => s.searchResults);
   const drillIntoLayer = useDashboardStore((s) => s.drillIntoLayer);
 
@@ -154,36 +157,25 @@ function useOverviewGraph() {
       return null;
     }
 
-    // Build search match counts per layer
+    // Build search match counts per layer using the precomputed
+    // nodeIdToLayerId index. Reusing the store-level index avoids an extra
+    // O(N) pass when search results change frequently.
     const searchMatchByLayer = new Map<string, number>();
     if (searchResults.length > 0) {
-      const nodeToLayer = new Map<string, string>();
-      for (const layer of layers) {
-        for (const nid of layer.nodeIds) {
-          nodeToLayer.set(nid, layer.id);
-        }
-      }
       for (const result of searchResults) {
-        const lid = nodeToLayer.get(result.nodeId);
+        const lid = nodeIdToLayerId.get(result.nodeId);
         if (lid) {
           searchMatchByLayer.set(lid, (searchMatchByLayer.get(lid) ?? 0) + 1);
         }
       }
     }
 
-    // Create cluster nodes
+    // Create cluster nodes. Per-layer aggregation goes through
+    // `computeLayerStats`, which iterates `layer.nodeIds` against the
+    // `nodesById` index — O(K) per layer instead of the previous
+    // O(N) Array.filter that ran `layer.nodeIds.includes(n.id)` (#102).
     const clusterNodes: LayerClusterFlowNode[] = layers.map((layer, i) => {
-      const memberNodes = graph.nodes.filter((n) => layer.nodeIds.includes(n.id));
-      const complexCounts = { simple: 0, moderate: 0, complex: 0 };
-      for (const n of memberNodes) {
-        complexCounts[n.complexity]++;
-      }
-      const aggregateComplexity =
-        complexCounts.complex > memberNodes.length * 0.3
-          ? "complex"
-          : complexCounts.moderate > memberNodes.length * 0.3
-            ? "moderate"
-            : "simple";
+      const { aggregateComplexity } = computeLayerStats(layer, nodesById);
 
       return {
         id: layer.id,
@@ -222,7 +214,7 @@ function useOverviewGraph() {
     }
 
     return { clusterNodes, flowEdges, dims };
-  }, [graph, searchResults, drillIntoLayer]);
+  }, [graph, nodesById, nodeIdToLayerId, searchResults, drillIntoLayer]);
 
   const [overview, setOverview] = useState<{ nodes: Node[]; edges: Edge[] }>({
     nodes: [],

understand-anything-plugin/packages/dashboard/src/store.ts

@@ -3,6 +3,7 @@ import { SearchEngine } from "@understand-anything/core/search";
 import type { SearchResult } from "@understand-anything/core/search";
 import type { GraphIssue } from "@understand-anything/core/schema";
 import type {
+  GraphNode,
   KnowledgeGraph,
   TourStep,
 } from "@understand-anything/core/types";
@@ -49,19 +50,60 @@ const DEFAULT_FILTERS: FilterState = {
 /** Categories used for node type filter toggles. Single source of truth for NodeCategory. */
 export type NodeCategory = "code" | "config" | "docs" | "infra" | "data" | "domain" | "knowledge";
 
-/** Find which layer a node belongs to. Returns layerId or null. */
-function findNodeLayer(graph: KnowledgeGraph, nodeId: string): string | null {
+/**
+ * Build the (id → node) and (id → layerId) lookup maps that the rest of
+ * the dashboard reads via store selectors. Centralised so `setGraph` and
+ * any future graph-replacement path stay in sync.
+ *
+ * Two layer indexes, intentionally distinct:
+ *
+ * - `nodeIdToLayerId` preserves the prior `findNodeLayer` "first matching
+ *   layer wins" semantics — if a node id appears in multiple layers
+ *   (rare but legal in the schema), the first occurrence in `graph.layers`
+ *   order is the one we map to. Drives navigation (drillIntoLayer, tour
+ *   step → layer, sidebar history) where a single canonical layer is the
+ *   right answer.
+ *
+ * - `nodeIdToLayerIds` records *every* layer a node belongs to. Drives
+ *   membership queries (filterNodes) where the prior `Layer[] +
+ *   layer.nodeIds.includes` shape was any-layer-wins — a node in L1 and
+ *   L2 with only L2 selected must still pass. Collapsing to first-wins
+ *   for filtering would be a silent regression.
+ */
+function buildGraphIndexes(graph: KnowledgeGraph): {
+  nodesById: Map<string, GraphNode>;
+  nodeIdToLayerId: Map<string, string>;
+  nodeIdToLayerIds: Map<string, Set<string>>;
+} {
+  const nodesById = new Map<string, GraphNode>();
+  for (const node of graph.nodes) nodesById.set(node.id, node);
+  const nodeIdToLayerId = new Map<string, string>();
+  const nodeIdToLayerIds = new Map<string, Set<string>>();
   for (const layer of graph.layers) {
-    if (layer.nodeIds.includes(nodeId)) return layer.id;
+    for (const nid of layer.nodeIds) {
+      if (!nodeIdToLayerId.has(nid)) nodeIdToLayerId.set(nid, layer.id);
+      let set = nodeIdToLayerIds.get(nid);
+      if (!set) {
+        set = new Set<string>();
+        nodeIdToLayerIds.set(nid, set);
+      }
+      set.add(layer.id);
+    }
   }
-  return null;
+  return { nodesById, nodeIdToLayerId, nodeIdToLayerIds };
 }
 
 /** Maximum number of entries in the sidebar navigation history. */
 const MAX_HISTORY = 50;
 
 interface DashboardStore {
   graph: KnowledgeGraph | null;
+  /** id → node lookup, rebuilt by setGraph. Empty before any graph loads. */
+  nodesById: Map<string, GraphNode>;
+  /** id → layer id (first-matching-layer wins), rebuilt by setGraph. Empty before any graph loads. */
+  nodeIdToLayerId: Map<string, string>;
+  /** id → set of every layer the node belongs to, rebuilt by setGraph. Empty before any graph loads. */
+  nodeIdToLayerIds: Map<string, Set<string>>;
   selectedNodeId: string | null;
   searchQuery: string;
   searchResults: SearchResult[];
@@ -189,11 +231,11 @@ function getSortedTour(graph: KnowledgeGraph): TourStep[] {
 
 /** Navigate tour step to the correct layer for the first highlighted node. */
 function navigateTourToLayer(
-  graph: KnowledgeGraph,
+  nodeIdToLayerId: Map<string, string>,
   nodeIds: string[],
 ): Partial<DashboardStore> {
   if (nodeIds.length === 0) return {};
-  const layerId = findNodeLayer(graph, nodeIds[0]);
+  const layerId = nodeIdToLayerId.get(nodeIds[0]);
   if (layerId) {
     return {
       navigationLevel: "layer-detail" as const,
@@ -205,6 +247,9 @@ function navigateTourToLayer(
 
 export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   graph: null,
+  nodesById: new Map<string, GraphNode>(),
+  nodeIdToLayerId: new Map<string, string>(),
+  nodeIdToLayerIds: new Map<string, Set<string>>(),
   selectedNodeId: null,
   searchQuery: "",
   searchResults: [],
@@ -259,8 +304,12 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
     const { viewMode, domainGraph, activeDomainId } = get();
     // Preserve domain view if a domain graph is already loaded
     const keepDomainView = viewMode === "domain" && domainGraph !== null;
+    const { nodesById, nodeIdToLayerId, nodeIdToLayerIds } = buildGraphIndexes(graph);
     set({
       graph,
+      nodesById,
+      nodeIdToLayerId,
+      nodeIdToLayerIds,
       searchEngine,
       searchResults,
       navigationLevel: "overview",
@@ -296,9 +345,9 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   navigateToNodeInLayer: (nodeId) => {
-    const { graph, selectedNodeId, nodeHistory } = get();
+    const { graph, selectedNodeId, nodeHistory, nodeIdToLayerId } = get();
     if (!graph) return;
-    const layerId = findNodeLayer(graph, nodeId);
+    const layerId = nodeIdToLayerId.get(nodeId) ?? null;
     const newHistory =
       selectedNodeId && nodeId !== selectedNodeId
         ? [...nodeHistory, selectedNodeId].slice(-MAX_HISTORY)
@@ -323,11 +372,11 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   navigateToHistoryIndex: (index) => {
-    const { nodeHistory, graph } = get();
+    const { nodeHistory, graph, nodeIdToLayerId } = get();
     if (!graph || index < 0 || index >= nodeHistory.length) return;
     const targetId = nodeHistory[index];
     const newHistory = nodeHistory.slice(0, index);
-    const layerId = findNodeLayer(graph, targetId);
+    const layerId = nodeIdToLayerId.get(targetId) ?? null;
     set({
       selectedNodeId: targetId,
       nodeHistory: newHistory,
@@ -336,11 +385,11 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   goBackNode: () => {
-    const { nodeHistory, graph } = get();
+    const { nodeHistory, graph, nodeIdToLayerId } = get();
     if (nodeHistory.length === 0 || !graph) return;
     const prevNodeId = nodeHistory[nodeHistory.length - 1];
     const newHistory = nodeHistory.slice(0, -1);
-    const layerId = findNodeLayer(graph, prevNodeId);
+    const layerId = nodeIdToLayerId.get(prevNodeId) ?? null;
     if (layerId) {
       set({
         navigationLevel: "layer-detail",
@@ -483,10 +532,10 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   startTour: () => {
-    const { graph } = get();
+    const { graph, nodeIdToLayerId } = get();
     if (!graph || !graph.tour || graph.tour.length === 0) return;
     const sorted = getSortedTour(graph);
-    const layerNav = navigateTourToLayer(graph, sorted[0].nodeIds);
+    const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[0].nodeIds);
     set({
       tourActive: true,
       currentTourStep: 0,
@@ -504,11 +553,11 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
     }),
 
   setTourStep: (step) => {
-    const { graph } = get();
+    const { graph, nodeIdToLayerId } = get();
     if (!graph || !graph.tour || graph.tour.length === 0) return;
     const sorted = getSortedTour(graph);
     if (step < 0 || step >= sorted.length) return;
-    const layerNav = navigateTourToLayer(graph, sorted[step].nodeIds);
+    const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[step].nodeIds);
     set({
       currentTourStep: step,
       tourHighlightedNodeIds: sorted[step].nodeIds,
@@ -517,12 +566,12 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   nextTourStep: () => {
-    const { graph, currentTourStep } = get();
+    const { graph, currentTourStep, nodeIdToLayerId } = get();
     if (!graph || !graph.tour || graph.tour.length === 0) return;
     const sorted = getSortedTour(graph);
     if (currentTourStep < sorted.length - 1) {
       const next = currentTourStep + 1;
-      const layerNav = navigateTourToLayer(graph, sorted[next].nodeIds);
+      const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[next].nodeIds);
       set({
         currentTourStep: next,
         tourHighlightedNodeIds: sorted[next].nodeIds,
@@ -532,12 +581,12 @@ export const useDashboardStore = create<DashboardStore>()((set, get) => ({
   },
 
   prevTourStep: () => {
-    const { graph, currentTourStep } = get();
+    const { graph, currentTourStep, nodeIdToLayerId } = get();
     if (!graph || !graph.tour || graph.tour.length === 0) return;
     if (currentTourStep > 0) {
       const sorted = getSortedTour(graph);
       const prev = currentTourStep - 1;
-      const layerNav = navigateTourToLayer(graph, sorted[prev].nodeIds);
+      const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[prev].nodeIds);
       set({
         currentTourStep: prev,
         tourHighlightedNodeIds: sorted[prev].nodeIds,