diff --git a/src/engine/engine.go b/src/engine/engine.go index d5c00d9..f22148f 100644 --- a/src/engine/engine.go +++ b/src/engine/engine.go @@ -161,13 +161,13 @@ func (e *Engine) RunWorkflow(ctx context.Context, wf *Workflow, cfg RunConfig) ( // Parallel dispatcher step if step.Prompt == "" && len(step.Parallel) > 0 { cost, err := e.runParallel(ctx, step, wf.Steps, stepIndex, session, cfg.Input, outputs, opts, &iterNum) + totalUSD += cost if err != nil { e.db.UpdateSessionStatus(session.ID, "failed") failed = true stepErr = err break } - totalUSD += cost if step.End { break } @@ -194,6 +194,7 @@ func (e *Engine) RunWorkflow(ctx context.Context, wf *Workflow, cfg RunConfig) ( } else { cost, output, err := e.runStep(ctx, step, session, cfg.Input, outputs, opts, &iterNum) if err != nil { + totalUSD += cost e.db.UpdateSessionStatus(session.ID, "failed") failed = true stepErr = err @@ -373,10 +374,12 @@ func (e *Engine) runStep(ctx context.Context, step *WfStep, session *lib.Session result, err := e.runner.Run(ctx, prompt, opts) if err != nil { + // Runners may report spend on failure (e.g. claude max-turns). dbStep.Status = "failed" + dbStep.CostUSD = result.CostUSD dbStep.ChangeSummary = err.Error() e.db.UpdateStep(dbStep) - return 0, "", fmt.Errorf("step %s failed: %w", step.ID, err) + return result.CostUSD, "", fmt.Errorf("step %s failed: %w", step.ID, err) } if err := ValidateRequiredOutput(*step, result.Output); err != nil { dbStep.Status = "failed" @@ -406,6 +409,16 @@ func (e *Engine) runLoop(ctx context.Context, step *WfStep, session *lib.Session maxIter = 1 } + // Snapshot the tree before the first iteration so a failed iteration + // rolls back only its own edits, keeping uncommitted output from + // earlier steps. A snapshot (not a commit) runs no hooks, so a repair + // loop still starts when pre-commit would reject the current state. + snapshot, err := e.git.Snapshot() + if err != nil { + return 0, fmt.Errorf("loop %s: snapshot failed: %w", step.ID, err) + } + rollback := func() error { return e.git.RestoreSnapshot(snapshot) } + succeeded := false consecutiveFailures := 0 stuckDetected := false @@ -433,11 +446,23 @@ func (e *Engine) runLoop(ctx context.Context, step *WfStep, session *lib.Session result, err := e.runner.Run(ctx, prompt, opts) if err != nil { + // Count spend from failed runs so overBudget stays accurate. + // The failed run may still have edited files; those edits never + // faced the gate, so revert them before retrying — otherwise a + // later commit (iteration or final) would keep unvalidated work. dbStep.Status = "failed" + dbStep.CostUSD = result.CostUSD dbStep.ChangeSummary = err.Error() e.db.UpdateStep(dbStep) + totalCost += result.CostUSD + if addCost != nil { + addCost(result.CostUSD) + } + if revertErr := rollback(); revertErr != nil { + return totalCost, fmt.Errorf("loop %s: revert failed after runner error: %w", step.ID, revertErr) + } consecutiveFailures++ - fmt.Printf(" failed, retrying\n\n") + fmt.Printf(" failed, reverted, retrying\n\n") continue } if err := ValidateRequiredOutput(*step, result.Output); err != nil { @@ -445,8 +470,15 @@ func (e *Engine) runLoop(ctx context.Context, step *WfStep, session *lib.Session dbStep.CostUSD = result.CostUSD dbStep.ChangeSummary = err.Error() e.db.UpdateStep(dbStep) + totalCost += result.CostUSD + if addCost != nil { + addCost(result.CostUSD) + } + if revertErr := rollback(); revertErr != nil { + return totalCost, fmt.Errorf("loop %s: revert failed after required-output failure: %w", step.ID, revertErr) + } consecutiveFailures++ - fmt.Printf(" failed required output check, retrying\n\n") + fmt.Printf(" failed required output check, reverted, retrying\n\n") continue } @@ -475,7 +507,7 @@ func (e *Engine) runLoop(ctx context.Context, step *WfStep, session *lib.Session if exitCode != 0 { reason := fmt.Sprintf("gate failed (exit %d)", exitCode) fmt.Printf(" → %s, reverting iteration\n\n", reason) - if revertErr := e.git.RevertAll(); revertErr != nil { + if revertErr := rollback(); revertErr != nil { fmt.Printf(" → revert failed: %s\n", revertErr) dbStep.Status = "failed" dbStep.ChangeSummary = fmt.Sprintf("%s; revert failed: %s", reason, revertErr) @@ -521,6 +553,12 @@ func (e *Engine) runLoop(ctx context.Context, step *WfStep, session *lib.Session if commitErr := e.git.CommitAll(fmt.Sprintf("%s: %s iteration %d", session.Workflow, step.ID, attempt)); commitErr != nil { fmt.Printf(" → commit failed: %s\n", commitErr) } + // Kept work is the new rollback point even if its commit failed. + snap, snapErr := e.git.Snapshot() + if snapErr != nil { + return totalCost, fmt.Errorf("loop %s: snapshot failed: %w", step.ID, snapErr) + } + snapshot = snap // Check until condition. Line-anchored (see MatchUntil) — // sentinel must appear on its own line to avoid matching @@ -599,9 +637,10 @@ func (e *Engine) runParallel(ctx context.Context, dispatcher *WfStep, allSteps [ if err != nil { dbStep.Status = "failed" + dbStep.CostUSD = result.CostUSD dbStep.ChangeSummary = err.Error() e.db.UpdateStep(dbStep) - results[j] = parallelResult{id: pid, err: err} + results[j] = parallelResult{id: pid, err: err, cost: result.CostUSD} return } if err := ValidateRequiredOutput(*step, result.Output); err != nil { @@ -609,7 +648,7 @@ func (e *Engine) runParallel(ctx context.Context, dispatcher *WfStep, allSteps [ dbStep.CostUSD = result.CostUSD dbStep.ChangeSummary = err.Error() e.db.UpdateStep(dbStep) - results[j] = parallelResult{id: pid, err: err} + results[j] = parallelResult{id: pid, err: err, cost: result.CostUSD} return } @@ -629,6 +668,8 @@ func (e *Engine) runParallel(ctx context.Context, dispatcher *WfStep, allSteps [ var failCount int var firstErr error for _, r := range results { + // Failed children still spent money; count it for the budget. + totalCost += r.cost if r.err != nil { fmt.Printf(" %s: failed (%v)\n", r.id, r.err) failCount++ @@ -638,7 +679,6 @@ func (e *Engine) runParallel(ctx context.Context, dispatcher *WfStep, allSteps [ continue } outputs[r.id] = r.output - totalCost += r.cost fmt.Printf(" %s: done ($%.4f)\n", r.id, r.cost) } fmt.Println() diff --git a/src/engine/engine_test.go b/src/engine/engine_test.go index fb6e930..c7eaa53 100644 --- a/src/engine/engine_test.go +++ b/src/engine/engine_test.go @@ -2,12 +2,14 @@ package engine import ( "context" + "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "sync" + "sync/atomic" "testing" "github.com/5uck1ess/devkit/lib" @@ -1714,6 +1716,108 @@ func TestInterpolateDeterministic(t *testing.T) { } } +// erroringCostRunner fails every call but reports spend, like claude -p +// hitting max-turns after real work. +type erroringCostRunner struct{ calls atomic.Int32 } + +func (r *erroringCostRunner) Name() string { return "err-cost" } +func (r *erroringCostRunner) Available() bool { return true } +func (r *erroringCostRunner) Run(ctx context.Context, prompt string, opts runners.RunOpts) (runners.RunResult, error) { + r.calls.Add(1) + return runners.RunResult{CostUSD: 0.50}, errors.New("max turns") +} + +func TestRunWorkflowFailedRunsCountTowardBudget(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + runner := &erroringCostRunner{} + eng := mustEngine(t, db, git, runner, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "fix", Prompt: "Fix", Loop: &Loop{Max: 10, Until: "DONE"}}, + }} + res, _ := eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x", BudgetUSD: 1.00}) + if n := runner.calls.Load(); n > 2 { + t.Errorf("runner called %d times; failed runs' cost must count toward the $1.00 budget (want <= 2)", n) + } + if res != nil && res.TotalUSD < 1.00 { + t.Errorf("TotalUSD = %.2f, want >= 1.00 (failed-run spend dropped)", res.TotalUSD) + } +} + +func TestRunWorkflowParallelFailedRunsCountCost(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + eng := mustEngine(t, db, git, &erroringCostRunner{}, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "a", Prompt: "A"}, + {ID: "b", Prompt: "B"}, + {ID: "fan", Parallel: []string{"a", "b"}}, + }} + res, _ := eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x"}) + if res == nil || res.TotalUSD < 1.00 { + t.Fatalf("failed parallel children's spend dropped: %+v", res) + } +} + +func TestRunWorkflowLoopRequireFailureCountsTowardBudget(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + responses := make([]runners.RunResult, 10) + for i := range responses { + responses[i] = runners.RunResult{Output: "no marker", CostUSD: 0.50} + } + runner := newMockRunner(responses, nil) + eng := mustEngine(t, db, git, runner, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "fix", Prompt: "Fix", Loop: &Loop{Max: 10, Until: "DONE"}, Require: &Require{Contains: []string{"MARKER"}}}, + }} + eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x", BudgetUSD: 1.00}) + if runner.callIdx > 2 { + t.Errorf("runner called %d times; require-failed iterations must count toward the budget", runner.callIdx) + } +} + +// editThenFailRunner succeeds on the first call and, on the second, edits a +// tracked file and then errors — like claude -p writing code and hitting +// max-turns. +type editThenFailRunner struct { + dir string + calls int +} + +func (r *editThenFailRunner) Name() string { return "edit-fail" } +func (r *editThenFailRunner) Available() bool { return true } +func (r *editThenFailRunner) Run(ctx context.Context, prompt string, opts runners.RunOpts) (runners.RunResult, error) { + r.calls++ + if r.calls == 1 { + return runners.RunResult{Output: "progress"}, nil + } + if err := os.WriteFile(filepath.Join(r.dir, "state.txt"), []byte("bad\n"), 0o644); err != nil { + return runners.RunResult{}, err + } + return runners.RunResult{}, errors.New("error_max_turns") +} + +func TestRunWorkflowLoopRunnerErrorRevertsEdits(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + eng := mustEngine(t, db, git, &editThenFailRunner{dir: dir}, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "fix", Prompt: "Fix", Loop: &Loop{Max: 2, Until: "DONE"}}, + }} + eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x"}) + if _, err := os.Stat(filepath.Join(dir, "state.txt")); err == nil { + t.Fatal("edits from a failed runner call survived; they must be reverted, never committed") + } + out, err := exec.Command("git", "-C", dir, "log", "--all", "--name-only", "--format=").Output() + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "state.txt") { + t.Fatal("unvalidated edit from failed runner call was committed") + } +} + func TestRunWorkflowEndStopsBeforeTrailingSteps(t *testing.T) { db := tempDB(t) dir, git := initGitRepo(t) @@ -1871,3 +1975,94 @@ func TestValidateRejectsBranchOnParallelDispatcher(t *testing.T) { t.Error("branch on a parallel dispatcher should be rejected (CLI never evaluates it)") } } + +// A failed loop iteration must roll back only its own edits, not +// uncommitted output from steps that ran before the loop. +func TestRunWorkflowLoopRollbackKeepsEarlierStepWork(t *testing.T) { + cases := map[string]struct { + runner *mockRunner + gate string + }{ + "runner error": { + runner: newMockRunner([]runners.RunResult{{}, result("DONE")}, []error{errors.New("exit 1"), nil}), + }, + "gate failure": { + runner: newMockRunner([]runners.RunResult{result("try"), result("DONE")}, nil), + gate: `test -f gate-ok || { touch gate-ok; exit 1; }`, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("gate-ok\n.devkit/\n"), 0o644); err != nil { + t.Fatal(err) + } + eng := mustEngine(t, db, git, tc.runner, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "prep", Command: "printf good > state.txt"}, + {ID: "fix", Prompt: "Fix", Loop: &Loop{Max: 2, Until: "DONE", Gate: tc.gate}}, + }} + if _, err := eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x"}); err != nil { + t.Fatalf("RunWorkflow: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, "state.txt")) + if err != nil || string(got) != "good" { + t.Fatalf("earlier step's state.txt lost by loop rollback: %q, %v", got, err) + } + }) + } +} + +// funcRunner adapts a function to runners.Runner. +type funcRunner func() (runners.RunResult, error) + +func (f funcRunner) Name() string { return "func" } +func (f funcRunner) Available() bool { return true } +func (f funcRunner) Run(ctx context.Context, prompt string, opts runners.RunOpts) (runners.RunResult, error) { + return f() +} + +// A repair loop must still start when a pre-commit hook rejects the state +// it is meant to repair — rollback protection must not depend on commits. +func TestRunWorkflowRepairLoopRunsDespiteRejectingPreCommitHook(t *testing.T) { + db := tempDB(t) + dir, git := initGitRepo(t) + gitIn := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %s", args, out) + } + } + write := func(name, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), mode); err != nil { + t.Fatal(err) + } + } + write(".gitignore", ".devkit/\n", 0o644) + write("state.txt", "good", 0o644) + gitIn("add", "-A") + gitIn("commit", "-m", "state") + write(filepath.Join(".git", "hooks", "pre-commit"), "#!/bin/sh\ntest \"$(cat state.txt)\" = good\n", 0o755) + + calls := 0 + repair := funcRunner(func() (runners.RunResult, error) { + calls++ + write("state.txt", "good", 0o644) + return result("DONE"), nil + }) + eng := mustEngine(t, db, git, repair, dir) + wf := &Workflow{Name: "test", Steps: []WfStep{ + {ID: "break", Command: "printf bad > state.txt"}, + {ID: "repair", Prompt: "Fix", Loop: &Loop{Max: 2, Until: "DONE"}}, + }} + if _, err := eng.RunWorkflow(context.Background(), wf, RunConfig{Input: "x"}); err != nil { + t.Fatalf("RunWorkflow: %v", err) + } + if calls == 0 { + t.Fatal("repair loop never ran") + } +} diff --git a/src/lib/git.go b/src/lib/git.go index 0afc839..9e196c9 100644 --- a/src/lib/git.go +++ b/src/lib/git.go @@ -53,8 +53,24 @@ func (g *Git) CommitAll(msg string) error { return err } -func (g *Git) RevertAll() error { - if _, err := g.run("checkout", "--", "."); err != nil { +// Snapshot records the current working tree (tracked and untracked, +// honoring .gitignore) as a git tree object and returns its id. Unlike a +// commit it runs no hooks, so it works even when the tree would fail a +// pre-commit check. +func (g *Git) Snapshot() (string, error) { + if _, err := g.run("add", "-A"); err != nil { + return "", err + } + return g.run("write-tree") +} + +// RestoreSnapshot makes the working tree match a tree from Snapshot: +// modified and deleted files come back, files created since are removed. +func (g *Git) RestoreSnapshot(tree string) error { + if _, err := g.run("read-tree", tree); err != nil { + return err + } + if _, err := g.run("checkout-index", "-a", "-f"); err != nil { return err } _, err := g.run("clean", "-fd") diff --git a/src/runners/claude.go b/src/runners/claude.go index 401bffb..2ab070d 100644 --- a/src/runners/claude.go +++ b/src/runners/claude.go @@ -14,10 +14,12 @@ import ( type ClaudeRunner struct{} type claudeResponse struct { - Result string `json:"result"` - SessionID string `json:"session_id"` - IsError bool `json:"is_error"` - TotalCostUSD float64 `json:"total_cost_usd"` + Result string `json:"result"` + SessionID string `json:"session_id"` + IsError bool `json:"is_error"` + Subtype string `json:"subtype"` + Errors []string `json:"errors"` + TotalCostUSD float64 `json:"total_cost_usd"` Usage struct { InputTokens int `json:"input_tokens"` OutputTokens int `json:"output_tokens"` @@ -85,12 +87,31 @@ func (r *ClaudeRunner) Run(ctx context.Context, prompt string, opts RunOpts) (Ru fmt.Errorf("claude returned non-JSON output: %s", TruncStr(stdout.String(), 200)) } - return RunResult{ + result := RunResult{ Output: resp.Result, SessionID: resp.SessionID, TokensIn: resp.Usage.InputTokens, TokensOut: resp.Usage.OutputTokens, CostUSD: resp.TotalCostUSD, ExitCode: exitCode, - }, nil + } + // claude -p reports API errors, auth failures and max-turns + // exhaustion as valid JSON with is_error=true (often with a non-zero + // exit). Treat either signal as a failed run so the engine does not + // record the error text as a kept step output. Output and cost are + // kept on the result for reporting/budget accounting. + if resp.IsError || exitCode != 0 { + msg := strings.TrimSpace(resp.Result) + if msg == "" { + msg = strings.Join(resp.Errors, "; ") + } + if msg == "" { + msg = strings.TrimSpace(stderr.String()) + } + if resp.Subtype != "" && resp.Subtype != "success" { + msg = resp.Subtype + ": " + msg + } + return result, fmt.Errorf("claude error (exit %d, is_error=%t): %s", exitCode, resp.IsError, TruncStr(msg, 500)) + } + return result, nil } diff --git a/src/runners/claude_run_test.go b/src/runners/claude_run_test.go new file mode 100644 index 0000000..747a977 --- /dev/null +++ b/src/runners/claude_run_test.go @@ -0,0 +1,72 @@ +//go:build !windows + +package runners + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// fakeClaude puts a `claude` script on PATH that prints stdout and exits with code. +func fakeClaude(t *testing.T, stdout string, code int) { + t.Helper() + dir := t.TempDir() + script := "#!/bin/sh\ncat <<'JSON'\n" + stdout + "\nJSON\nexit " + itoa(code) + "\n" + if err := os.WriteFile(filepath.Join(dir, "claude"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func itoa(n int) string { return strconv.Itoa(n) } + +func TestClaudeRun_IsErrorReturnsError(t *testing.T) { + fakeClaude(t, `{"is_error":true,"result":"Credit balance is too low","total_cost_usd":0.01}`, 0) + res, err := (&ClaudeRunner{}).Run(context.Background(), "hi", RunOpts{}) + if err == nil { + t.Fatalf("expected error for is_error=true, got nil (output %q)", res.Output) + } + if !strings.Contains(err.Error(), "Credit balance is too low") { + t.Errorf("error should carry the result text, got %v", err) + } + if res.CostUSD != 0.01 { + t.Errorf("cost should be preserved on error, got %v", res.CostUSD) + } +} + +func TestClaudeRun_NonZeroExitWithJSONReturnsError(t *testing.T) { + fakeClaude(t, `{"is_error":false,"result":"partial","total_cost_usd":0.02}`, 1) + res, err := (&ClaudeRunner{}).Run(context.Background(), "hi", RunOpts{}) + if err == nil { + t.Fatal("expected error for non-zero exit, got nil") + } + if res.ExitCode != 1 { + t.Errorf("exit code = %d, want 1", res.ExitCode) + } + if res.Output != "partial" { + t.Errorf("output should be preserved, got %q", res.Output) + } +} + +func TestClaudeRun_Success(t *testing.T) { + fakeClaude(t, `{"is_error":false,"result":"ok","total_cost_usd":0.03}`, 0) + res, err := (&ClaudeRunner{}).Run(context.Background(), "hi", RunOpts{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Output != "ok" || res.CostUSD != 0.03 { + t.Errorf("got %+v", res) + } +} + +func TestClaudeRun_MaxTurnsUsesErrorsField(t *testing.T) { + fakeClaude(t, `{"subtype":"error_max_turns","is_error":true,"total_cost_usd":0.8,"errors":["Reached maximum number of turns (30)"]}`, 1) + _, err := (&ClaudeRunner{}).Run(context.Background(), "hi", RunOpts{}) + if err == nil || !strings.Contains(err.Error(), "Reached maximum number of turns") { + t.Fatalf("error should carry the errors[] text, got %v", err) + } +}