feat(aiops): sub-agent orchestration with UI-driven settings and console hardening

.env.example

@@ -90,35 +90,22 @@ DEFAULT_TENANT_ID=org-default
 # Working memory (Phase 1) — in-session context compaction for long-running agents.
 # When over WORKING_MEMORY_TOKEN_BUDGET estimated tokens, older turns are folded into
 # a rolling summary + scratchpad (via the reflector model) so multi-hour runs survive
-# the context window. Set WORKING_MEMORY_ENABLED=false for identical legacy behavior.
-WORKING_MEMORY_ENABLED=true
 WORKING_MEMORY_TOKEN_BUDGET=60000
 WORKING_MEMORY_KEEP_RECENT=8
 
 # Save-time memory reconciliation (Phase 2) — dedup/contradiction resolution via an
 # LLM judge before writing. Set false to restore blind upserts (legacy behavior).
-MEMORY_RECONCILE_ENABLED=true
 
 # Episodic memory (Phase 3) — capture one distilled episode per tool-using run and
 # replay similar past episodes as few-shot experience at task start.
-EPISODIC_MEMORY_ENABLED=true
 
 # Procedural memory (Phase 4) — learn operating rules from corrections/failures and
 # inject them as '### Operating rules (learned)'; promote matured rules to Skills (human-approved).
-PROCEDURAL_MEMORY_ENABLED=true
 
 # Hermes autonomy + observability
 # MEMORY_LOG_VERBOSE: print full injected memoryContext blocks (summary lines are always on).
-# CHAT_TRIAGE_ENABLED: chat-layer router — conversational messages get a direct reply instead of the agent workflow; the same call auto-picks the skill. false = always run the workflow (legacy behavior).
 # (Auto skill selection/loading has no env flag — it is always available and toggled per-session via the "Auto skills" switch in the AIOps console.)
-# AUTO_SKILL_CREATION_ENABLED: matured procedural domains auto-become/refresh enabled system skills (read-only tier).
-# AUTO_SKILL_MATURITY_THRESHOLD: accessCount a rule needs before promoting.
 MEMORY_LOG_VERBOSE=true
-CHAT_TRIAGE_ENABLED=true
-AUTO_SKILL_CREATION_ENABLED=true
-AUTO_SKILL_MATURITY_THRESHOLD=3
-# SKILL_SYNTHESIS_MIN_RULES: matured rules a domain needs before it earns a synthesized skill.
-SKILL_SYNTHESIS_MIN_RULES=3
 
 # ============================================================================
 # Optional integrations
@@ -148,6 +135,21 @@ LANGFUSE_HOST=https://cloud.langfuse.com
 # Skip audit DB writes in local dev
 # SKIP_AUDIT_LOGGING=true
 
+# AI Ops sub-agents (Layer B). The feature is UI-driven: each tenant enables it
+# and tunes its budget from the AI Ops console settings — no env var needed.
+# SUBAGENTS_ENABLED=false is an EMERGENCY deployment-wide kill-switch (tenant
+# settings cannot re-enable while it is set); leave it unset in normal operation.
+# The SUBAGENT_MAX_* vars are CEILINGS that lower (never raise) the built-in
+# maximums a tenant may configure.
+# SUBAGENTS_ENABLED=false
+# SUBAGENT_MAX_CONCURRENCY=6
+# SUBAGENT_MAX_PER_RUN=16
+# SUBAGENT_MAX_TOKENS_PER_RUN=1000000
+# SUBAGENT_MAX_ITERATIONS=16
+# SUBAGENT_TIMEOUT_MS=300000
+# Max concurrent shell subprocesses from execute_command (Layer A).
+# TOOL_CONCURRENCY=6
+
 # Internal worker<->web-ui auth key (must match across both apps).
 # REQUIRED for the agent-ops scheduler to work: the workers process sends this
 # as x-internal-key when triggering a scheduled task, and web-ui only trusts the

apps/web-ui/app/api/chat/__tests__/stream-parts.test.ts

@@ -1,6 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { buildMemoryPart, humanizeReflection, humanizePlanning, isWorkingMemoryPayload, stripWorkingMemoryPrelude } from '@/app/api/chat/stream-parts';
 import { buildUsagePart } from '../stream-parts';
+import { buildSubagentPart, buildHeartbeatChunks, isSubagentModelEvent } from '../stream-parts';
+import { SUBAGENT_MODEL_TAG } from '@/lib/agent/subagent';
 
 const WM_JSON = JSON.stringify({
     summary: 'User asked to connect Jira; site resource retrieved.',
@@ -236,3 +238,134 @@ describe('buildUsagePart', () => {
         expect(buildUsagePart(3, 4)).toEqual({ type: 'data-usage', data: { input: 3, output: 4 } });
     });
 });
+
+describe('buildSubagentPart', () => {
+    const base = {
+        id: 'sa-1', role: 'EC2 auditor', task: 'audit account 1',
+        toolCount: 3, tokensIn: 900, tokensOut: 120,
+    };
+
+    it('builds a stable id so updates replace rather than append', () => {
+        const part = buildSubagentPart({ ...base, status: 'running' });
+        expect(part.type).toBe('data-subagent');
+        expect(part.id).toBe('subagent-sa-1');
+    });
+
+    it('carries progress counters', () => {
+        const part = buildSubagentPart({ ...base, status: 'running' });
+        expect(part.data).toMatchObject({ role: 'EC2 auditor', status: 'running', toolCount: 3, tokensIn: 900 });
+    });
+
+    it('omits the transcript from the stream payload', () => {
+        const part = buildSubagentPart({
+            ...base, status: 'done', summary: 'found things',
+            transcript: [{ kind: 'ai', text: 'internal reasoning' }],
+        });
+        expect(JSON.stringify(part.data)).not.toContain('internal reasoning');
+        expect((part.data as { summary?: string }).summary).toBe('found things');
+    });
+});
+
+// The heartbeat exists solely to keep bytes flowing so CloudFront's 60s
+// originReadTimeout (infra/compute/index.ts:962) can't kill a long run. Its tick
+// body lives here — not inline in route.ts — because nothing in the suite imports
+// route.ts, so an inline tick is untested by construction.
+describe('buildHeartbeatChunks', () => {
+    const base = {
+        id: 'sa-1', role: 'EC2 auditor', task: 'audit account 1',
+        toolCount: 3, tokensIn: 900, tokensOut: 120,
+    };
+
+    it('re-emits one data-subagent part per in-flight sub-agent and no keep-alive', () => {
+        const chunks = buildHeartbeatChunks([
+            { ...base, id: 'sa-1', status: 'running' },
+            { ...base, id: 'sa-2', status: 'running' },
+        ]);
+        expect(chunks).toHaveLength(2);
+        expect(chunks.map((c) => c.type)).toEqual(['data-subagent', 'data-subagent']);
+        expect(chunks.map((c: any) => c.id)).toEqual(['subagent-sa-1', 'subagent-sa-2']);
+        expect(chunks.some((c) => c.type === 'data-keepalive')).toBe(false);
+    });
+
+    // REGRESSION GUARD for F1. `liveSubagents` empties as sub-agents reach their
+    // terminal event, so a tick that only walks that map writes ZERO bytes during
+    // the orchestrator's post-fan-out call — the longest silence and the largest
+    // context in the run, exactly when the 60s origin timeout bites. A tick MUST
+    // write something. This test fails if the empty-case branch is removed.
+    it('emits exactly one keep-alive chunk when no sub-agent is in flight', () => {
+        const chunks = buildHeartbeatChunks([]);
+        expect(chunks).toHaveLength(1);
+        expect(chunks[0].type).toBe('data-keepalive');
+    });
+
+    it('marks the keep-alive transient and gives it no sub-agent fields', () => {
+        const [chunk] = buildHeartbeatChunks([]) as any[];
+        // transient parts are delivered to the client but never appended to
+        // message.parts, so a keep-alive cannot bloat the message or move the reducer.
+        expect(chunk.transient).toBe(true);
+        expect(chunk.id).toBeUndefined();
+        expect(chunk.data).toEqual({});
+        expect(JSON.stringify(chunk)).not.toMatch(/sa-1|EC2 auditor|audit account|transcript/);
+    });
+
+    it('still omits the transcript for a terminal event passed through the tick', () => {
+        const chunks = buildHeartbeatChunks([{
+            ...base, status: 'done', summary: 'found things',
+            transcript: [{ kind: 'ai', text: 'internal reasoning' }],
+        }]);
+        expect(chunks).toHaveLength(1);
+        expect(JSON.stringify(chunks[0])).not.toContain('internal reasoning');
+        expect(JSON.stringify(chunks[0])).not.toContain('transcript');
+    });
+});
+
+// Sub-agent model calls run inside ToolNode, so their on_chat_model_* callback
+// events flow through the SAME streamEvents pipeline as the orchestrator's own
+// model runs. The route's text-part machinery is a single-threaded state machine;
+// N concurrent sub-agent runs driving it interleave part ids, the client receives
+// a text-delta for a part that never started, and the AI SDK kills the stream.
+// isSubagentModelEvent is the filter that keeps them out.
+describe('isSubagentModelEvent', () => {
+    it('matches chat-model events tagged by the sub-agent runtime', () => {
+        for (const name of ['on_chat_model_start', 'on_chat_model_stream', 'on_chat_model_end']) {
+            expect(isSubagentModelEvent({ event: name, tags: ['x', SUBAGENT_MODEL_TAG] })).toBe(true);
+        }
+    });
+
+    it('ignores untagged chat-model events (the orchestrator itself)', () => {
+        expect(isSubagentModelEvent({ event: 'on_chat_model_stream', tags: ['seq:step:2'] })).toBe(false);
+        expect(isSubagentModelEvent({ event: 'on_chat_model_stream' })).toBe(false);
+    });
+
+    it('ignores non-chat-model events even when tagged (tool events must still flow)', () => {
+        expect(isSubagentModelEvent({ event: 'on_tool_start', tags: [SUBAGENT_MODEL_TAG] })).toBe(false);
+        expect(isSubagentModelEvent({ event: 'on_chain_stream', tags: [SUBAGENT_MODEL_TAG] })).toBe(false);
+    });
+
+    it('tolerates malformed events', () => {
+        expect(isSubagentModelEvent({})).toBe(false);
+        expect(isSubagentModelEvent({ event: undefined, tags: undefined })).toBe(false);
+    });
+});
+
+describe('buildSubagentPart live-detail fields', () => {
+    const base = {
+        id: 'sa-1', role: 'EC2 auditor', task: 'audit account 1',
+        toolCount: 3, tokensIn: 900, tokensOut: 120,
+    };
+
+    it('stamps a ts so the client can derive startedAt from the first part', () => {
+        const before = Date.now();
+        const part = buildSubagentPart({ ...base, status: 'running' });
+        const ts = (part.data as { ts: number }).ts;
+        expect(ts).toBeGreaterThanOrEqual(before);
+        expect(ts).toBeLessThanOrEqual(Date.now());
+    });
+
+    it('passes lastTool through for the live action line, omitting it when absent', () => {
+        const withTool = buildSubagentPart({ ...base, status: 'running', lastTool: 'aws_read (+2 more)' });
+        expect((withTool.data as { lastTool?: string }).lastTool).toBe('aws_read (+2 more)');
+        const without = buildSubagentPart({ ...base, status: 'running' });
+        expect('lastTool' in (without.data as object)).toBe(false);
+    });
+});