From 1eb71cf0c8dee6e5736b32237365636129122968 Mon Sep 17 00:00:00 2001 From: onatozmenn Date: Fri, 14 Aug 2026 15:05:42 +0300 Subject: [PATCH] fix: retry errored instances on resume --- benchmarks/utils/critics.py | 5 +++-- tests/test_iterative_resume.py | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/benchmarks/utils/critics.py b/benchmarks/utils/critics.py index b53cdd30a..6fcd8e28d 100644 --- a/benchmarks/utils/critics.py +++ b/benchmarks/utils/critics.py @@ -147,8 +147,7 @@ def evaluate_output(critic: CriticBase, eval_output: EvalOutput) -> bool: def get_completed_instances(output_file: str) -> Set[EvalInstanceID]: """ - Get all instance IDs present in output file - (completed, regardless of success/failure). + Get instance IDs that produced output without a runner error. Reads ``instance_id`` directly from each JSON line WITHOUT validating the full ``EvalOutput`` model. Resume must recognise prior completion across @@ -184,6 +183,8 @@ def get_completed_instances(output_file: str) -> Set[EvalInstanceID]: f"Missing 'instance_id' on line {line_num} in {output_file}" ) continue + if isinstance(data, dict) and data.get("error") is not None: + continue completed_instances.add(instance_id) except Exception as e: diff --git a/tests/test_iterative_resume.py b/tests/test_iterative_resume.py index d9358fc86..6693a0cd1 100644 --- a/tests/test_iterative_resume.py +++ b/tests/test_iterative_resume.py @@ -445,7 +445,7 @@ def _make_output(instance_id, has_patch): ) -def test_get_completed_instances_tolerates_stale_archive_schema(): +def test_completed_instances_tolerate_stale_schema_and_skip_runner_errors(): """ Resume must skip instances that were completed by an older SDK whose EvalOutput schema has since drifted (e.g. browser tool/observation @@ -454,6 +454,8 @@ def test_get_completed_instances_tolerates_stale_archive_schema(): Reading instance_id from raw JSON avoids treating those rows as "never completed" and re-running them from scratch. Regression test for evaluation#515. + + Rows with a top-level runner error must remain eligible for retry. """ from benchmarks.utils.critics import get_completed_instances @@ -492,13 +494,24 @@ def test_get_completed_instances_tolerates_stale_archive_schema(): ) f.write(fresh.model_dump_json() + "\n") - # 3) Blank line: must be tolerated, not counted. + # 3) Runner error: must be retried rather than counted as complete. + errored = EvalOutput( + instance_id="instance_error", + test_result={}, + instruction=None, + error="Runtime timed out", + history=[], + instance={"x": 3}, + ) + f.write(errored.model_dump_json() + "\n") + + # 4) Blank line: must be tolerated, not counted. f.write("\n") - # 4) Malformed JSON: must be skipped with a warning, not raise. + # 5) Malformed JSON: must be skipped with a warning, not raise. f.write("{not valid json}\n") - # 5) JSON object missing instance_id: must be skipped. + # 6) JSON object missing instance_id: must be skipped. f.write(json.dumps({"foo": "bar"}) + "\n") completed = get_completed_instances(path)