diff --git a/crates/tinyagents-harness/src/lib.rs b/crates/tinyagents-harness/src/lib.rs index 1474a22c6..9bdd4108f 100644 --- a/crates/tinyagents-harness/src/lib.rs +++ b/crates/tinyagents-harness/src/lib.rs @@ -125,9 +125,9 @@ pub use error::{Result, TinyAgentsError}; pub use ids::*; pub use model_registry::{ModelRegistry, ModelSelection, ResolvedModelBinding}; pub use no_progress::{ - DEFAULT_IDENTICAL_HALT_THRESHOLD, DEFAULT_REPEAT_CALL_THRESHOLD, - DEFAULT_REPEAT_OUTPUT_THRESHOLD, NoProgress, NoProgressTracker, SuccessfulRepeat, - SuccessfulRepeatTracker, ToolAttempt, + ClassifiedFailure, ClassifiedFailureTracker, DEFAULT_IDENTICAL_HALT_THRESHOLD, + DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD, NoProgress, NoProgressTracker, + SuccessfulRepeat, SuccessfulRepeatTracker, ToolAttempt, }; pub use observability::{ AgentCallLatency, AgentLatencyMetrics, AgentObservation, FanOutSink, HarnessEventJournal, diff --git a/crates/tinyagents-harness/src/no_progress/README.md b/crates/tinyagents-harness/src/no_progress/README.md index ed90179b6..a8069fe94 100644 --- a/crates/tinyagents-harness/src/no_progress/README.md +++ b/crates/tinyagents-harness/src/no_progress/README.md @@ -31,6 +31,13 @@ follow-up; see the "Driving this from an `after_tool` hook" section in state for one turn. `new(identical_halt_threshold)` builds it, `record(step, &ToolAttempt) -> NoProgress` feeds one outcome and returns the verdict, `reset()` clears all counters (called internally after a halt). +- [`ClassifiedFailureTracker`] — an additive ledger for equivalent failures + keyed by class, operation, and resource or permission scope. `record` accepts + a class-specific recovery budget; `clear` removes one group only after an + observation shows its blocker changed. Intervening tool calls leave it intact. + Hold one tracker per turn, or call `reset()` at a new turn boundary. A + `NoProgress::Halt` verdict does not reset classified counts: retrying the same + unchanged blocker in a resumed turn would halt again. - [`ToolAttempt`] — one observed outcome, built with `success`/`failure` plus the `hard_reject()`/`recoverable_miss()` modifiers. - [`NoProgress`] — the verdict enum: `Continue`, `Nudge(String)`, diff --git a/crates/tinyagents-harness/src/no_progress/classified.rs b/crates/tinyagents-harness/src/no_progress/classified.rs new file mode 100644 index 000000000..2b4811af1 --- /dev/null +++ b/crates/tinyagents-harness/src/no_progress/classified.rs @@ -0,0 +1,93 @@ +//! Failure equivalence and recovery budgets independent of literal call arguments. + +use std::collections::HashMap; +use std::sync::Mutex; + +use super::NoProgress; + +/// A trusted classification of one failed operation. The caller supplies stable +/// operation and scope identifiers; neither error prose nor literal query text +/// belongs in this key. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ClassifiedFailure { + pub class: String, + pub operation: String, + pub scope: String, +} + +impl ClassifiedFailure { + pub fn new( + class: impl Into, + operation: impl Into, + scope: impl Into, + ) -> Self { + Self { + class: class.into(), + operation: operation.into(), + scope: scope.into(), + } + } +} + +/// Counts equivalent failures across intervening calls. Only an observation +/// that the same blocker changed should call [`Self::clear`]. +#[derive(Default)] +pub struct ClassifiedFailureTracker { + counts: Mutex>, +} + +impl ClassifiedFailureTracker { + /// `recovery_budget` is the number of retry calls allowed after the first + /// failure. A budget of one permits one retry and halts if that retry also + /// fails; zero stops on the first observation. + pub fn record(&self, key: &ClassifiedFailure, recovery_budget: usize) -> NoProgress { + let mut counts = self.counts.lock().unwrap(); + let attempts = counts.entry(key.clone()).or_default(); + *attempts += 1; + if *attempts > recovery_budget { + NoProgress::Halt(format!( + "Stopping after {} attempt(s): failure class `{}` still blocks operation `{}` on `{}`. Resolve this blocker before retrying.", + attempts, key.class, key.operation, key.scope + )) + } else { + NoProgress::Continue + } + } + + /// Clear one blocker after an observation proves it changed or recovered. + pub fn clear(&self, key: &ClassifiedFailure) { + self.counts.lock().unwrap().remove(key); + } + + /// Clear all groups when a new turn begins. + pub fn reset(&self) { + self.counts.lock().unwrap().clear(); + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn equivalent_failures_survive_intervening_calls_and_clear_by_scope() { + let tracker = ClassifiedFailureTracker::default(); + let a = ClassifiedFailure::new("permission", "search", "account-a"); + let b = ClassifiedFailure::new("permission", "search", "account-b"); + assert_eq!(tracker.record(&a, 1), NoProgress::Continue); + assert_eq!(tracker.record(&b, 1), NoProgress::Continue); + assert!( + matches!(tracker.record(&a, 1), NoProgress::Halt(message) if message.contains("2 attempt(s)")) + ); + tracker.clear(&a); + assert_eq!(tracker.record(&a, 1), NoProgress::Continue); + assert!(matches!(tracker.record(&b, 1), NoProgress::Halt(_))); + } + + #[test] + fn zero_budget_stops_immediately() { + let tracker = ClassifiedFailureTracker::default(); + let key = ClassifiedFailure::new("policy", "write", "project"); + assert!(matches!(tracker.record(&key, 0), NoProgress::Halt(_))); + } +} diff --git a/crates/tinyagents-harness/src/no_progress/mod.rs b/crates/tinyagents-harness/src/no_progress/mod.rs index 2bf534374..41f44f204 100644 --- a/crates/tinyagents-harness/src/no_progress/mod.rs +++ b/crates/tinyagents-harness/src/no_progress/mod.rs @@ -60,9 +60,11 @@ //! and [`NoProgress::as_str`] exist so step 4 needs no enum match, and //! `as_str()` gives a stable telemetry label. +mod classified; mod successful_repeat; mod types; +pub use classified::{ClassifiedFailure, ClassifiedFailureTracker}; pub use successful_repeat::{DEFAULT_REPEAT_CALL_THRESHOLD, DEFAULT_REPEAT_OUTPUT_THRESHOLD}; use types::LadderState; pub use types::{