Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/cli/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,7 @@ export function decorateWrapInlineText(result: WrapResult["result"], raw: boolea
"",
"---",
WRAP_AUTHORITATIVE_FOOTER,
...(result.rawRef?.id ? [`Raw output is available locally: tokenjuice cat ${result.rawRef.id}`] : []),
].join("\n");
return `${result.inlineText}${footer}`;
}
Expand Down
35 changes: 35 additions & 0 deletions src/core/command-match.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { CommandMatchSource, ToolExecutionInput } from "../types.js";

import {
ENV_ASSIGNMENT_PATTERN,
hasSequentialShellCommands,
isCompoundShellCommand,
splitTopLevelCommandChain,
stripLeadingEnvAssignmentsFromCommand,
Expand Down Expand Up @@ -523,6 +524,40 @@ export function resolveEffectiveCommand(input: Pick<ToolExecutionInput, "argv" |
return null;
}

export function hasMultipleSubstantiveShellCommands(input: Pick<ToolExecutionInput, "argv" | "command">): boolean {
const shellBody = unwrapShellRunner(input);
const command = (shellBody ?? input.command ?? "").trim();
if (!command || !hasSequentialShellCommands(command)) {
return false;
}

let effectiveCommand = command;
for (let iteration = 0; iteration < 16; iteration += 1) {
const setupIfTail = stripLeadingSetupIfBlock(effectiveCommand);
const setupSegmentTail = setupIfTail ?? stripLeadingSetupSegment(effectiveCommand);
if (!setupSegmentTail) {
break;
}
effectiveCommand = setupSegmentTail;
}

let substantiveCount = 0;
for (const sequenceSegment of splitTopLevelCommandChain(effectiveCommand)) {
for (const segment of splitTopLevelOrChain(sequenceSegment)) {
const candidate = buildEffectiveCandidate(tokenizeCommand(segment), true, segment);
if (!candidate) {
continue;
}
substantiveCount += 1;
if (substantiveCount > 1) {
return true;
}
}
}

return false;
}

export function getEffectiveCommandArgv(input: Pick<ToolExecutionInput, "argv" | "command">): string[] {
return resolveEffectiveCommand(input)?.argv ?? getCandidateArgv(input);
}
Expand Down
1 change: 1 addition & 0 deletions src/core/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ export type { CommandMatchCandidate } from "./command-match.js";
export {
deriveCommandMatchCandidates,
getEffectiveCommandArgv,
hasMultipleSubstantiveShellCommands,
isSetupWrapperSegment,
resolveEffectiveCommand,
stripLeadingEnvAssignments,
Expand Down
2 changes: 1 addition & 1 deletion src/core/compaction-metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export const NO_COMPACTION_METADATA: CompactionMetadata = {
kinds: [],
};

export const WRAP_AUTHORITATIVE_FOOTER = "[tokenjuice] This is the complete, authoritative output for this command. It was deterministically compacted to remove low-signal noise; the omitted content is not retrievable. Do not re-run the command, vary flags, or switch tools to try to recover it. Proceed with the task using this output.";
export const WRAP_AUTHORITATIVE_FOOTER = "[tokenjuice] This is the complete, authoritative output for this command. It was deterministically compacted to remove low-signal noise. Do not re-run the command, vary flags, or switch tools to try to recover omitted content; use a raw-artifact recovery command below when one is provided. Proceed with the task using this output.";

function buildCompactionMetadata(authoritative: boolean, ...kinds: CompactionKind[]): CompactionMetadata {
if (kinds.length === 0) {
Expand Down
22 changes: 17 additions & 5 deletions src/core/reduce.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { loadRules } from "./rules.js";
import { hasMultipleSubstantiveShellCommands } from "./command-match.js";
import { classifyExecution, resolveRuleMatch } from "./classify.js";
import { isFileContentInspectionCommand } from "./command-identity.js";
import { normalizeExecutionInput } from "./execution-input.js";
Expand Down Expand Up @@ -323,12 +324,21 @@ export async function reduceExecutionWithRules(
reducedChars,
ratio: measuredRawChars === 0 ? 1 : reducedChars / measuredRawChars,
});
const resolvedMatch = opts.classifier
const multipleSubstantiveCommands = !opts.classifier && hasMultipleSubstantiveShellCommands(input);
const resolvedMatch = opts.classifier || multipleSubstantiveCommands
? undefined
: resolveRuleMatch(input, rules);
const classification = resolvedMatch?.classification
?? classifyExecution(input, rules, opts.classifier);
const reducerInput = resolvedMatch?.candidateInput ?? normalizedInput;
const classification = multipleSubstantiveCommands
? {
family: "generic",
confidence: 0.2,
matchedReducer: "generic/fallback",
}
: resolvedMatch?.classification
?? classifyExecution(input, rules, opts.classifier);
const reducerInput = multipleSubstantiveCommands
? normalizedInput
: resolvedMatch?.candidateInput ?? normalizedInput;
const trace = opts.trace
? {
...(normalizedInput.command ? { normalizedCommand: normalizedInput.command } : {}),
Expand Down Expand Up @@ -390,7 +400,9 @@ export async function reduceExecutionWithRules(
};
}

const inspectionSummary = buildInspectionSummary(normalizedInput, rawText, opts.noOmit);
const inspectionSummary = multipleSubstantiveCommands
? null
: buildInspectionSummary(normalizedInput, rawText, opts.noOmit);
if (inspectionSummary) {
const summaryText = inspectionSummary.lines.join("\n").trim();
const selectedText = clampTextMiddleWithMetadata(summaryText, maxInlineChars, opts.noOmit);
Expand Down
27 changes: 27 additions & 0 deletions test/cli/main.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,33 @@ describe("decorateWrapInlineText", () => {
expect(decorateWrapInlineText(result, false)).toContain(WRAP_AUTHORITATIVE_FOOTER);
});

it("provides the raw-artifact recovery command when output is stored", () => {
const result: CompactResult = {
inlineText: "summary",
compaction: {
authoritative: true,
kinds: ["head-tail-omission"],
},
rawRef: {
id: "tj_0123456789ab",
path: "/tmp/tokenjuice/raw.txt",
metadataPath: "/tmp/tokenjuice/meta.json",
},
stats: {
rawChars: 4_000,
reducedChars: 40,
ratio: 0.01,
},
classification: {
family: "generic",
confidence: 0.9,
matchedReducer: "generic/fallback",
},
};

expect(decorateWrapInlineText(result, false)).toContain("tokenjuice cat tj_0123456789ab");
});

it("suppresses the authoritative footer for lossless rewrites", () => {
const result: CompactResult = {
inlineText: "summary",
Expand Down
21 changes: 21 additions & 0 deletions test/core/command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
import {
deriveCommandMatchCandidates,
getGitSubcommand,
hasMultipleSubstantiveShellCommands,
hasSequentialShellCommands,
isFileContentInspectionCommand,
isRepositoryInspectionCommand,
Expand Down Expand Up @@ -371,6 +372,26 @@ describe("hasSequentialShellCommands", () => {
});
});

describe("hasMultipleSubstantiveShellCommands", () => {
it.each([
"grep -i github /etc/hosts; echo '---dig:'; dig +short api.github.com @1.1.1.1; scutil --dns",
"cd repo && swift test && rg -n failure src",
"command -v rg || cargo install ripgrep; rg --files src",
"bash -lc 'grep -i github /etc/hosts; dig +short api.github.com @1.1.1.1'",
])("detects `%s` as multiple substantive commands", (command) => {
expect(hasMultipleSubstantiveShellCommands({ command })).toBe(true);
});

it.each([
"cd repo && pnpm test",
"source .env && cargo test",
"if command -v tt >/dev/null 2>&1; then tt title 'tests'; else tmux select-pane -T 'tests' 2>/dev/null || true; fi; pnpm test",
"bash -lc 'cd repo && pnpm test'",
])("keeps setup-wrapped `%s` as one substantive command", (command) => {
expect(hasMultipleSubstantiveShellCommands({ command })).toBe(false);
});
});

describe("getGitSubcommand", () => {
it.each([
{ command: "git ls-files src", subcommand: "ls-files" },
Expand Down
71 changes: 60 additions & 11 deletions test/core/reduce.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ describe("reduceExecution", () => {
}
});

it("matches wrapped rg search commands and prefers the first substantive command in a chain", async () => {
it("matches wrapped rg search commands after setup commands", async () => {
const searchResult = await reduceExecution({
toolName: "exec",
command: "pwd && rg -n AssertionError src",
Expand All @@ -721,20 +721,69 @@ describe("reduceExecution", () => {
expect(searchResult.classification.matchedReducer).toBe("search/rg");
expect(searchResult.classification.matchedVia).toBe("effective");
expect(searchResult.classification.matchedCommand).toBe("rg -n AssertionError src");
});

it("preserves all short output from a multi-command sequence", async () => {
const rawText = [
"127.0.0.1 github.com",
"---dig:",
"140.82.121.4",
"---scutil:",
"DNS configuration",
].join("\n");

const firstCommandWins = await reduceExecution({
const result = await reduceExecution({
toolName: "exec",
command: "cd repo && swift test && rg -n failure src",
combinedText: [
"Test Case 'FooTests.testExample' failed (0.12 seconds).",
"Executed 1 test, with 1 failure (0 unexpected) in 0.12 (0.12) seconds",
].join("\n"),
exitCode: 1,
command: "grep -i github /etc/hosts; echo '---dig:'; dig +short api.github.com @1.1.1.1; echo '---scutil:'; scutil --dns | head -1",
combinedText: rawText,
exitCode: 0,
});

expect(firstCommandWins.classification.matchedReducer).toBe("tests/swift-test");
expect(firstCommandWins.classification.matchedVia).toBe("effective");
expect(firstCommandWins.classification.matchedCommand).toBe("swift test");
expect(result.classification.matchedReducer).toBe("generic/fallback");
expect(result.inlineText).toBe(rawText);
expect(result.stats.ratio).toBe(1);
});

it("uses authoritative generic compaction for large multi-command output", async () => {
const rawText = Array.from({ length: 80 }, (_, index) => `output ${index + 1} ${"x".repeat(48)}`).join("\n");
const result = await reduceExecution({
toolName: "exec",
command: "grep -i github /etc/hosts; dig +short api.github.com @1.1.1.1",
combinedText: rawText,
exitCode: 0,
}, {
maxInlineChars: 240,
});

expect(result.classification.matchedReducer).toBe("generic/fallback");
expect(result.inlineText).toContain("output 1");
expect(result.inlineText).toContain("output 80");
expect(result.inlineText).not.toContain("output 40");
expect(result.compaction).toEqual({
authoritative: true,
kinds: expect.arrayContaining(["head-tail-omission"]),
});
});

it("does not summarize file inspection output from a multi-command sequence", async () => {
const rawText = [
"{",
" \"name\": \"example\",",
" \"lockfileVersion\": 3,",
" \"packages\": {}",
"}",
"DONE",
].join("\n");
const result = await reduceExecution({
toolName: "exec",
command: "cat package-lock.json; echo DONE",
combinedText: rawText,
exitCode: 0,
});

expect(result.classification.matchedReducer).toBe("generic/fallback");
expect(result.inlineText).toBe(rawText);
expect(result.stats.ratio).toBe(1);
});

it("keeps wrapped file inspection output verbatim under generic fallback", async () => {
Expand Down
Loading