From 71fbc796b22e1ace14272308e7f0ea49b243df6b Mon Sep 17 00:00:00 2001 From: Lumen Yang Date: Wed, 20 May 2026 23:15:54 +0200 Subject: [PATCH] fix: dedupe replayed context tail after compression --- api/streaming.py | 60 +++++++++++++++++++ tests/test_issue1217_transcript_compaction.py | 55 +++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/api/streaming.py b/api/streaming.py index 20a78e47..5b7de02b 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -2199,6 +2199,52 @@ def _messages_have_prefix(messages, prefix): return True +def _message_replay_key(msg): + """Return a stable comparison key for replay/overlap de-duplication.""" + identity = _message_identity(msg) + if identity is not None: + return identity + if not isinstance(msg, dict): + return None + return ( + str(msg.get('role') or ''), + _message_text(msg.get('content', '')), + str(msg.get('tool_call_id') or ''), + json.dumps(msg.get('tool_calls') or [], sort_keys=True, ensure_ascii=False), + ) + + +def _strip_replayed_prefix(existing_messages, candidates): + """Drop a candidate prefix that is already the suffix of existing_messages. + + Compression/continuation can replay the active tail from state.db after the + previous WebUI context/display already contains it. Prefix-only merge logic + then treats that replayed tail as a fresh delta and duplicates a whole turn. + Strip the largest exact suffix/prefix overlap before appending. + """ + existing_messages = list(existing_messages or []) + candidates = list(candidates or []) + max_overlap = min(len(existing_messages), len(candidates)) + for overlap in range(max_overlap, 0, -1): + left = [_message_replay_key(m) for m in existing_messages[-overlap:]] + right = [_message_replay_key(m) for m in candidates[:overlap]] + if left == right: + return candidates[overlap:] + return candidates + + +def _dedupe_replayed_active_context(previous_context, result_messages): + """Keep model context append-only without re-appending a replayed tail.""" + previous_context = list(previous_context or []) + result_messages = list(result_messages or []) + if not previous_context or not result_messages: + return result_messages + if not _messages_have_prefix(result_messages, previous_context): + return result_messages + candidates = result_messages[len(previous_context):] + return previous_context + _strip_replayed_prefix(previous_context, candidates) + + def _is_context_compression_marker(msg): if not isinstance(msg, dict): return False @@ -2443,6 +2489,8 @@ def _merge_display_messages_after_agent_result(previous_display, previous_contex if _messages_have_prefix(result_messages, previous_context): candidates = result_messages[len(previous_context):] + candidates = _strip_replayed_prefix(previous_display, candidates) + candidates = _strip_replayed_prefix(previous_context, candidates) else: current_user_idx = _find_current_user_turn(result_messages, msg_text) marker_candidates = [ @@ -4322,6 +4370,10 @@ def _run_agent_streaming( _previous_context_messages, _result_messages, ) + _next_context_messages = _dedupe_replayed_active_context( + _previous_context_messages, + _next_context_messages, + ) s.context_messages = _next_context_messages s.messages = _merge_display_messages_after_agent_result( _previous_messages, @@ -4465,6 +4517,10 @@ def _run_agent_streaming( _previous_context_messages, _result_messages, ) + _next_context_messages = _dedupe_replayed_active_context( + _previous_context_messages, + _next_context_messages, + ) s.context_messages = _next_context_messages s.messages = _merge_display_messages_after_agent_result( _previous_messages, @@ -5281,6 +5337,10 @@ def _run_agent_streaming( _next_context_messages = _restore_reasoning_metadata( _previous_context_messages, _result_messages, ) + _next_context_messages = _dedupe_replayed_active_context( + _previous_context_messages, + _next_context_messages, + ) s.context_messages = _next_context_messages s.messages = _merge_display_messages_after_agent_result( _previous_messages, diff --git a/tests/test_issue1217_transcript_compaction.py b/tests/test_issue1217_transcript_compaction.py index c7ebe4bd..4549cb82 100644 --- a/tests/test_issue1217_transcript_compaction.py +++ b/tests/test_issue1217_transcript_compaction.py @@ -5,6 +5,7 @@ from types import SimpleNamespace from api.streaming import ( _assistant_reply_added_after_current_turn, _context_messages_for_new_turn, + _dedupe_replayed_active_context, _merge_display_messages_after_agent_result, _new_turn_context_from_messages, _sanitize_messages_for_api, @@ -227,6 +228,60 @@ def test_append_only_agent_result_preserves_normal_delta_behavior(): assert merged == result_messages +def test_replayed_active_tail_after_compression_is_not_duplicated(): + previous_display = [ + {"role": "assistant", "content": "old visible transcript"}, + {"role": "user", "content": "choose agent"}, + {"role": "assistant", "content": "checking agents"}, + {"role": "tool", "content": "agent list"}, + {"role": "assistant", "content": "agent answer"}, + ] + previous_context = [ + {"role": "assistant", "content": "[Session Arc Summary] compacted history"}, + {"role": "user", "content": "choose agent"}, + {"role": "assistant", "content": "checking agents"}, + {"role": "tool", "content": "agent list"}, + {"role": "assistant", "content": "agent answer"}, + ] + result_messages = previous_context + [ + # The new compressed state.db segment can replay the already-visible + # active tail before adding the next turn. + {"role": "user", "content": "choose agent"}, + {"role": "assistant", "content": "checking agents"}, + {"role": "tool", "content": "agent list"}, + {"role": "assistant", "content": "agent answer"}, + {"role": "user", "content": "choose B"}, + {"role": "assistant", "content": "using B"}, + ] + + merged = _merge_display_messages_after_agent_result( + previous_display, + previous_context, + result_messages, + "choose B", + ) + next_context = _dedupe_replayed_active_context(previous_context, result_messages) + + assert [m["content"] for m in merged] == [ + "old visible transcript", + "choose agent", + "checking agents", + "agent list", + "agent answer", + "choose B", + "using B", + ] + assert [m["content"] for m in next_context] == [ + "[Session Arc Summary] compacted history", + "choose agent", + "checking agents", + "agent list", + "agent answer", + "choose B", + "using B", + ] + + def test_repeated_user_text_after_compaction_is_not_dropped(): previous_display = [ {"role": "user", "content": "continue"},