From d702bd6bd951688c5f3e7e12fd5825aecb49e236 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 6 Oct 2026 10:14:35 -0400 Subject: [PATCH 1/5] feat: List every archived session in agentop, with no history key A restart, or max_sessions eviction, took a session off agentop's list until H was pressed, and H then showed "archived" where the time belonged. agentop now asks for ?archived=true on every fetch (a proxy without an archive ignores it), H and the marker are gone, and archived rows show their real UPDATED time. Thirty days of history is ~2,000 rows, so two walks written for a handful change with it: the title-harvest gate counts only resident rows (archived rows nothing can name held it open, re-walking the transcript tree every ~3 minutes forever), and a streamed event marks the table stale for the 1s tick instead of rebuilding it (6.4ms and 1.9MB per event at 2,000 rows). Beyond the new tests, three existing ones change: TestSessionsPane_EventCountDoesNotFlipOnAStreamedEvent and TestHandleStreamEvent_TunnelCloseDoesNotRefreshTheSession now flush the sessions table before asserting, because the rebuild waits for the tick; TestCharacterize_ServiceInstall_OverAnArchivingProxy expects a restart message that names agentop's listing, not H. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/agentop/README.md | 25 +-- cmd/agentop/cmd_service.go | 4 +- cmd/agentop/cmd_service_characterize_test.go | 4 +- cmd/agentop/tui/app.go | 109 ++++++------- cmd/agentop/tui/clear.go | 1 - cmd/agentop/tui/clear_test.go | 6 +- cmd/agentop/tui/event_retention_test.go | 3 + cmd/agentop/tui/help_overlay.go | 3 +- cmd/agentop/tui/help_overlay_test.go | 2 +- cmd/agentop/tui/history_test.go | 151 ++++++++++-------- cmd/agentop/tui/keys.go | 28 +--- cmd/agentop/tui/session_metadata.go | 37 +++-- cmd/agentop/tui/sessions_context.go | 6 - .../tui/sessions_context_bench_test.go | 5 +- cmd/agentop/tui/sessions_context_fold_test.go | 7 +- cmd/agentop/tui/sessions_pane.go | 12 +- cmd/agentop/tui/sessions_stream_bench_test.go | 64 ++++++++ cmd/agentop/tui/tunnel_close_test.go | 3 + cmd/cortex/archive.go | 2 +- docs/assets/cortex-demo.svg | 6 +- docs/laptop-service.md | 5 +- docs/session-dump.md | 2 +- 22 files changed, 267 insertions(+), 218 deletions(-) create mode 100644 cmd/agentop/tui/sessions_stream_bench_test.go diff --git a/cmd/agentop/README.md b/cmd/agentop/README.md index af93733b0..3f04d437a 100644 --- a/cmd/agentop/README.md +++ b/cmd/agentop/README.md @@ -757,17 +757,13 @@ agentop is for, and the other three are surfaces you visit and leave. that column also carried, for sessions the server has forgotten but agentop still holds events for, moved into `UPDATED`. - `UPDATED` also carries `archived`, but only with history on (`H`): a session - the proxy's memory no longer holds and its session archive still does — after - a restart, say, or once `session.max_sessions` evicted it. The two markers - answer the same question from opposite ends: `cached` is agentop remembering - what the proxy forgot, `archived` is the proxy remembering it on disk. Enter - opens an archived row like any other; its events are read back from disk a page - at a time — the latest on Enter, older ones with `o`, as for a resident - session — so a long one costs no more to open than a short one. - A proxy without an archive — any cluster sidecar, and a laptop that has turned - it off — answers `H` with the live list and a one-line note saying so. `X` - erases all of it, after asking; see the keybindings below. + With a session archive — a local install — the table lists every session the + archive holds beside the ones in memory, so neither a restart nor + `session.max_sessions` eviction takes a row off it. Enter opens an old session + like any other; its events are read back from disk a page at a time — the + latest on Enter, older ones with `o`, as for a resident session — so a long one + costs no more to open than a short one. `X` erases all of it, after asking; see + the keybindings below. **Every figure in this table is a per-session total**, summed over that session's whole history rather than over a clock window — which is why its @@ -797,13 +793,9 @@ agentop is for, and the other three are surfaces you visit and leave. default 1h ago 8 — — — — ● connected 2.1 events/sec feedback: https://github.com/rossoctl/cortex/issues/new/choose - … [H] history [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit + [↑↓] nav [↵] drill [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit ``` - The footer is 111 columns whole, so at 100 it has already given up `[↑↓] nav` - and `[↵] drill` — the two most guessable keys on the line, and the ones placed - first so they are the ones to go. `[u]` and `[$]` hold to 80. - The two money columns are dropped entirely on a terminal too narrow to show a sub-cent charge honestly — below 93 columns — rather than rounded to `$0.00` or blanked. A charge under a cent reads `<$0.01`. `TITLE` is fitted first and @@ -1258,7 +1250,6 @@ Layered on top of all of them: | `Esc` / `←` / `h` | detail, events | back out | | `Esc` | sessions | back to the agents picker when the list was reached by picking an agent there; otherwise (picker mode) tear down port-forward and back to pods. Sessions and the agents picker above it are the only panes that tear down — every key-opened surface returns to its caller instead | | `/` | sessions, events | filter (substring match; Enter commits and saves, Esc cancels the edit and saves nothing; clear the box and press Enter to remove a saved filter) | -| `H` | sessions | toggle history: also list the sessions the proxy's session archive holds and memory no longer does, marked `archived` in `UPDATED`. Capital because `h` backs out. Ahead of `[u]` in the footer so an 80-column cut drops it before the cost keys | | `X` | sessions | clear all history: every session this Cortex holds, in memory and on disk. Asks first, with the count and the size on disk; `y` erases, `n`/`esc` keeps. The cost ledger is kept. Refused — with the proxy's reason — anywhere but a loopback-only laptop install. Not in the footer, like `A`: a destructive key should not be advertised on the always-visible line | | `s` | events | toggle skip-row visibility (default: hidden; the events footer shows the hidden count) | | `c` | events | open the column picker (`↑↓`/`jk` move, `space`/`x` toggle, `s` sort, `r` reset, `Esc`/`Enter`/`c` close); the selection and sort are saved on close | diff --git a/cmd/agentop/cmd_service.go b/cmd/agentop/cmd_service.go index 324734c40..1ca1e1e12 100644 --- a/cmd/agentop/cmd_service.go +++ b/cmd/agentop/cmd_service.go @@ -1006,8 +1006,8 @@ func reportHistoryCleared(wasServing, archived bool, stdout io.Writer) { return } if archived { - fmt.Fprintln(stdout, " The proxy's memory is cleared, but its session archive is not: H in") - fmt.Fprintln(stdout, " agentop lists the sessions it holds.") + fmt.Fprintln(stdout, " The proxy's memory is cleared, but its session archive is not: agentop") + fmt.Fprintln(stdout, " still lists every session it holds.") return } fmt.Fprintln(stdout, " Captured session history is cleared: the store is in memory, so any") diff --git a/cmd/agentop/cmd_service_characterize_test.go b/cmd/agentop/cmd_service_characterize_test.go index 79b7f3908..44d5bdf34 100644 --- a/cmd/agentop/cmd_service_characterize_test.go +++ b/cmd/agentop/cmd_service_characterize_test.go @@ -528,8 +528,8 @@ func TestCharacterize_ServiceInstall_OverAnArchivingProxy(t *testing.T) { f.Close() run := sc.install(t, true, false) - const want = " The proxy's memory is cleared, but its session archive is not: H in\n" + - " agentop lists the sessions it holds.\n" + const want = " The proxy's memory is cleared, but its session archive is not: agentop\n" + + " still lists every session it holds.\n" if run.code != 0 || !strings.HasSuffix(run.out, want) || strings.Contains(run.out, "history is cleared") { t.Fatalf("exit %d, stdout:\n%s\nwant it to end with:\n%s", run.code, run.out, want) } diff --git a/cmd/agentop/tui/app.go b/cmd/agentop/tui/app.go index 8eafdc3f2..0f4b7dd13 100644 --- a/cmd/agentop/tui/app.go +++ b/cmd/agentop/tui/app.go @@ -259,11 +259,6 @@ type tickMsg time.Time type refreshTickMsg time.Time type sessionsLoadedMsg []session.SessionSummary -// historyLoadedMsg is the session list fetched with history on: sessionsLoadedMsg's rows plus the -// archive's disk-only ones, and its usage. A separate type rather than a change to -// sessionsLoadedMsg, which eight files build and match on; its handler sets the usage and then -// takes exactly the sessionsLoadedMsg path. -type historyLoadedMsg struct{ list apiclient.SessionList } type pipelineLoadedMsg *apiclient.PipelineView type snapshotLoadedMsg struct { // olderNotFetched is how many events precede the ones in this response, from the @@ -408,14 +403,12 @@ type model struct { // Data caches. sessions []session.SessionSummary + // sessionsStale says a streamed event changed the session list since the table was last built. + // The table is rebuilt on the next 1s tick (flushSessionsTable) rather than per event: with + // history listed it holds thirty days of sessions, where a rebuild costs 6.4ms and 1.9MB, and + // events arrive several a second. The poll replaces the list every 2s regardless. + sessionsStale bool - // showHistory lists the sessions the proxy's session archive holds and memory no longer does - // (H). Not persisted, after agentScope's precedent: a view of the moment, not a preference. - // archiveUsage is the archive's size and bounds from the last history list, nil without one; - // historyNoticed keeps the "no archive here" notice to once per toggle. - showHistory bool - archiveUsage *pipeline.ArchiveUsage - historyNoticed bool // clearConfirm is X's confirmation, modal while non-nil; clearGen counts completed clears, // so a snapshot answered before one is not stored after it. See clear.go. clearConfirm *clearConfirm @@ -443,16 +436,17 @@ type model struct { // AND EVERY ROW ABOVE THAT REBASES ALSO OWES rebuildSessionsTable, which is a second // obligation and not a restatement of the first. Updating contextRun makes the figure right; // the sessions table holds the gauge as a BAKED string, so nothing on screen changes until - // the rows are rebuilt. The streamed append had that call all along and the other three did - // not, so a session agentop streamed showed a gauge while a session it merely opened showed a - // dash — corrected in contextRun, and only visible when the next /v1/sessions poll happened - // to repaint. See TestSessionsTable_ASnapshotRepaintsTheGaugeItFilled. + // the rows are rebuilt. The streamed append had that rebuild all along — today by setting + // sessionsStale, so it lands on the next 1s tick — and the other three did not, so a session + // agentop streamed showed a gauge while a session it merely opened showed a dash — corrected + // in contextRun, and only visible when the next /v1/sessions poll happened to repaint. See TestSessionsTable_ASnapshotRepaintsTheGaugeItFilled. events map[string][]pipeline.SessionEvent // sessionID → every event held for it // contextRun is the CTX(1M) gauge's answer per session, folded forward as events // arrive rather than recomputed from the whole slice — see sessionContextFor. The row - // loop asks for every session on every rebuild, and a rebuild happens on every streamed - // event, so a full scan there is O(events) per session per event. It REMEMBERS THE WINNING - // TURN under the rule's ordering rather than caching the slice, and that is what lets the + // loop asks for every session on every rebuild, and a rebuild happens on the 1s tick after + // streamed events and on every 2s poll rather than per event, so a full scan there is + // O(events) per session per rebuild. It REMEMBERS THE WINNING TURN under the rule's + // ordering rather than caching the slice, and that is what lets the // events stop carrying the evidence (view=summary strips it) while the answer stays true. // Which turn wins is not a question of size — see pipeline.PromptContextFold. contextRun map[string]pipeline.PromptContextFold @@ -1212,25 +1206,17 @@ func refreshTickCmd() tea.Cmd { return tea.Tick(refreshInterval, func(t time.Time) tea.Msg { return refreshTickMsg(t) }) } -// loadSessionsCmd fetches the current session list. Used at startup and after -// each successful reconnect so we don't miss new sessions that appeared -// while the stream was down. +// loadSessionsCmd fetches the session list: every session the proxy holds in memory and, where it +// runs a session archive, every one it holds only on disk — so neither a restart nor an eviction +// takes a session off the list. A proxy without an archive ignores ?archived=true and sends its +// resident sessions, which is all it has. Used at startup, on every refresh tick and after a clear. func (m *model) loadSessionsCmd() tea.Cmd { - if m.showHistory { - return func() tea.Msg { - list, err := m.client.ListSessionsArchived(m.ctx) - if err != nil { - return errMsg{where: "list sessions", err: err} - } - return historyLoadedMsg{list: list} - } - } return func() tea.Msg { - summaries, err := m.client.ListSessions(m.ctx) + list, err := m.client.ListSessionsArchived(m.ctx) if err != nil { return errMsg{where: "list sessions", err: err} } - return sessionsLoadedMsg(summaries) + return sessionsLoadedMsg(list.Sessions) } } @@ -1286,6 +1272,7 @@ func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil case tickMsg: + m.flushSessionsTable() // In picker mode, skip the rate calculation — m.client may be nil // after a back-out. Keep the ticker alive so it's ready when the // user re-enters a session. @@ -1301,34 +1288,20 @@ func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.lastTick, m.lastCt = now, m.eventCt return m, tickCmd() - case historyLoadedMsg: - // A reply to a request made while history was on can land after H turned it off; its - // disk-only rows would then stay until the next poll. Drop it instead. - if !m.showHistory { - return m, nil - } - m.archiveUsage = msg.list.Archive - if !m.historyNoticed { - m.historyNoticed = true - if msg.list.Archive == nil { - m.setFlash("this Cortex has no session archive: showing live sessions only") - } else { - m.setFlash(fmt.Sprintf("history on: %s of sessions on disk, kept %d days", - formatBytes(msg.list.Archive.Bytes), msg.list.Archive.RetentionDays)) - } - } - return m.Update(sessionsLoadedMsg(msg.list.Sessions)) - case sessionsLoadedMsg: - // The server list says what is LIVE. It does not say what is worth - // keeping on screen. + // The server list says what the proxy holds: what is LIVE, and, where it + // runs a session archive, what is only on disk. It does not say what is + // worth keeping on screen. // // This used to drop cached events for every session the list omitted, and // bounce the user back to the sessions pane. The session store is - // in-memory and per-pod, so agentop's copy is the only copy: a proxy restart - // (or any blip that empties /v1/sessions, which arrives as a normal - // message, not an error) destroyed the events someone was mid-investigation - // on, about two seconds after they looked away. That is #870. + // in-memory and per-pod, so agentop's copy can be the only copy: a proxy + // without a session archive lists nothing after a restart (one with an + // archive lists everything it holds, and the sessions it omits are then + // few), and any blip that empties /v1/sessions arrives as a normal + // message, not an error. Either destroyed the events someone was + // mid-investigation on, about two seconds after they looked away. That is + // #870. // // Cached events are now released in exactly one place: when the user // returns to the picker and selects a different session (see keys.go). @@ -2195,7 +2168,7 @@ func (m *model) handleStreamEvent(ev apiclient.StreamEvent) { if !trailing { m.sessions[i].UpdatedAt = e.At } - goto sortAndRebuild + goto stale } } // New session → create a stub summary; next list refresh will replace it. @@ -2204,11 +2177,8 @@ func (m *model) handleStreamEvent(ev apiclient.StreamEvent) { ID: e.SessionID, CreatedAt: e.At, UpdatedAt: e.At, EventCount: 1, Active: true, }) } -sortAndRebuild: - sort.Slice(m.sessions, func(i, j int) bool { - return m.sessions[i].UpdatedAt.After(m.sessions[j].UpdatedAt) - }) - m.rebuildSessionsTable() +stale: + m.sessionsStale = true if m.pane == paneEvents && m.selectedSess == e.SessionID { // TODO: coalesce these rebuilds. Every streamed event rebuilds the whole table // for the session being watched: ~1.5ms flat plus ~0.23ms per 1000 events held @@ -2221,6 +2191,19 @@ sortAndRebuild: } } +// flushSessionsTable re-sorts the session list by recency and rebuilds its table when a streamed +// event has changed it since the last rebuild. Called from the 1s tick; see sessionsStale. +func (m *model) flushSessionsTable() { + if !m.sessionsStale { + return + } + m.sessionsStale = false + sort.Slice(m.sessions, func(i, j int) bool { + return m.sessions[i].UpdatedAt.After(m.sessions[j].UpdatedAt) + }) + m.rebuildSessionsTable() +} + // View composes the full screen. The [?] key-help overlay is layered on // top of whatever the pane rendered, so it works over the picker and the // session views alike. diff --git a/cmd/agentop/tui/clear.go b/cmd/agentop/tui/clear.go index df3b3626e..c95e9dff1 100644 --- a/cmd/agentop/tui/clear.go +++ b/cmd/agentop/tui/clear.go @@ -125,7 +125,6 @@ func (m *model) resetAfterClear() { m.usage.reqSeq++ m.untitledMisses = 0 m.untitledCounted = nil - m.archiveUsage = nil m.detailEvent = nil m.detailPlugin = nil m.selectedSess = "" diff --git a/cmd/agentop/tui/clear_test.go b/cmd/agentop/tui/clear_test.go index fea234513..6bab3e7e2 100644 --- a/cmd/agentop/tui/clear_test.go +++ b/cmd/agentop/tui/clear_test.go @@ -115,9 +115,9 @@ func TestClearConfirm_SwallowsOtherKeys(t *testing.T) { if cmd := m.handleKey(k); cmd != nil { t.Errorf("%q returned a command", k.String()) } - if m.clearConfirm == nil || m.pane != paneSessions || m.helpVisible || m.filtering || m.showHistory { - t.Fatalf("%q got past the confirmation: confirm=%v pane=%v help=%v filtering=%v history=%v", - k.String(), m.clearConfirm != nil, m.pane, m.helpVisible, m.filtering, m.showHistory) + if m.clearConfirm == nil || m.pane != paneSessions || m.helpVisible || m.filtering { + t.Fatalf("%q got past the confirmation: confirm=%v pane=%v help=%v filtering=%v", + k.String(), m.clearConfirm != nil, m.pane, m.helpVisible, m.filtering) } } if len(store.ListSessions()) != 2 { diff --git a/cmd/agentop/tui/event_retention_test.go b/cmd/agentop/tui/event_retention_test.go index 571bd0c65..8d4c7356b 100644 --- a/cmd/agentop/tui/event_retention_test.go +++ b/cmd/agentop/tui/event_retention_test.go @@ -366,6 +366,9 @@ func TestSessionsPane_EventCountDoesNotFlipOnAStreamedEvent(t *testing.T) { At: time.Now(), SessionID: id, Direction: pipeline.Outbound, Phase: pipeline.SessionRequest, Host: "h", }}) + // The table is rebuilt on the tick, not on the event, so flush it first: without + // this the cell read below is the one the poll built, and no stream write could reach it. + m.flushSessionsTable() if got := sessionsEventsCell(t, m, id); got != want { t.Fatalf("streamed event %d changed EVENTS to %q, want %q", i+1, got, want) } diff --git a/cmd/agentop/tui/help_overlay.go b/cmd/agentop/tui/help_overlay.go index cabc0c097..4e854636f 100644 --- a/cmd/agentop/tui/help_overlay.go +++ b/cmd/agentop/tui/help_overlay.go @@ -290,12 +290,11 @@ var paneKeys = map[paneID]keyGroup{ paneSessions: { title: "SESSIONS (this pane)", purpose: "one row per agent session, with its tokens, cost and context. " + - "Figures are per session; with the session archive on, H also lists sessions the proxy no longer holds in memory.", + "Where the proxy keeps a session archive, every session it holds is listed.", bindings: []keyBinding{ {"↑↓ / jk", "navigate"}, {"↵ / → / l", "drill into session"}, {"/", "filter"}, - {"H", "history: also list archived sessions"}, {"X", "clear all history, in memory and on disk (asks first; the cost ledger is kept)"}, {"esc", "back to the agents picker, else pods picker"}, }, diff --git a/cmd/agentop/tui/help_overlay_test.go b/cmd/agentop/tui/help_overlay_test.go index 2d2e614f5..2c13ddb21 100644 --- a/cmd/agentop/tui/help_overlay_test.go +++ b/cmd/agentop/tui/help_overlay_test.go @@ -553,7 +553,7 @@ func TestHelpOverlayScrollKeys(t *testing.T) { // that no longer existed. Too small fails loudly; too large fails silently. const ( helpWideTerminal = 100 - helpNoScrollHeight = 105 + helpNoScrollHeight = 103 ) // With everything visible there must be no scroll affordance — it would be noise diff --git a/cmd/agentop/tui/history_test.go b/cmd/agentop/tui/history_test.go index cb3031347..fe428c529 100644 --- a/cmd/agentop/tui/history_test.go +++ b/cmd/agentop/tui/history_test.go @@ -2,6 +2,8 @@ package tui import ( "context" + "io" + "net/http" "net/http/httptest" "slices" "strings" @@ -17,64 +19,46 @@ import ( "github.com/rossoctl/cortex/core/sessionapi" ) -// H shows sessions the proxy no longer holds — the session archive's — and only from the -// sessions pane, where that list is. -func TestHKey_TogglesHistoryOnTheSessionsPaneOnly(t *testing.T) { - m := &model{pane: paneSessions, client: deadClient()} - if cmd := m.handleKey(keyRune('H')); !m.showHistory || cmd == nil { - t.Fatalf("H on sessions: showHistory=%v cmd=%v, want on and a reload", m.showHistory, cmd != nil) - } - m.handleKey(keyRune('H')) - if m.showHistory { - t.Fatal("a second H did not turn history off") - } - m.pane = paneEvents - m.handleKey(keyRune('H')) - if m.showHistory { - t.Fatal("H toggled history from the events pane") +// Every session-list fetch asks for history. A proxy with a session archive answers with the +// sessions only its disk holds as well; one without ignores the parameter. There is no key for it: +// a restart must not take a session off the list. +func TestLoadSessions_AlwaysAsksForHistory(t *testing.T) { + var path, query string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path, query = r.URL.Path, r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"sessions":[{"id":"old","resident":false,"eventCount":3}]}`) + })) + defer ts.Close() + m := &model{ctx: context.Background(), client: apiclient.New(ts.URL)} + msg, ok := m.loadSessionsCmd()().(sessionsLoadedMsg) + if !ok { + t.Fatalf("loadSessionsCmd returned %T, want sessionsLoadedMsg", msg) + } + if path != "/v1/sessions" || query != "archived=true" { + t.Fatalf("requested %s?%s, want /v1/sessions?archived=true", path, query) + } + if len(msg) != 1 || msg[0].ID != "old" { + t.Fatalf("rows = %+v", msg) } } -// A disk-only row says so in the column where cached-only rows already say "cached": it is the -// one cell answering why the row has no live session behind it. -func TestHistoryLoaded_MarksDiskOnlyRowsArchived(t *testing.T) { +// A row only the archive holds is an ordinary row: UPDATED says when it was last active, as for any +// other, rather than a marker standing in for the time. +func TestSessionsPane_ADiskOnlyRowShowsWhenItWasUpdated(t *testing.T) { m := newTitleModel(t, nil) - m.showHistory = true no := false now := time.Now() - m.Update(historyLoadedMsg{list: apiclient.SessionList{ - Sessions: []session.SessionSummary{ - {ID: "live-1", UpdatedAt: now}, - {ID: "old-2", UpdatedAt: now.Add(-48 * time.Hour), Resident: &no, EventCount: 7}, - }, - Archive: &pipeline.ArchiveUsage{Bytes: 4 << 20, MaxBytes: 2 << 30, RetentionDays: 30}, - }}) + old := now.Add(-48 * time.Hour) + m.Update(sessionsLoadedMsg{ + {ID: "live-1", UpdatedAt: now}, + {ID: "old-2", UpdatedAt: old, Resident: &no, EventCount: 7}, + }) if got := strings.Join(m.sessionRowIDs, ","); got != "live-1,old-2" { t.Fatalf("rows %s, want live-1,old-2", got) } - rows := m.sessionsTbl.Rows() - if !rowHas(rows[1], archivedMarker) || rowHas(rows[0], archivedMarker) { - t.Fatalf("archived marker on the wrong rows: %v / %v", rows[0], rows[1]) - } - if m.archiveUsage == nil || m.archiveUsage.Bytes != 4<<20 { - t.Fatalf("archive usage not kept: %+v", m.archiveUsage) - } -} - -// Against a proxy that has no archive — or predates it, and ignores ?archived=true — the list -// comes back with no archive object. Say so once, instead of showing an unchanged list as if -// history were on. -func TestHistoryLoaded_SaysOnceWhenTheProxyHasNoArchive(t *testing.T) { - m := newTitleModel(t, nil) - m.showHistory = true - m.Update(historyLoadedMsg{list: apiclient.SessionList{Sessions: []session.SessionSummary{{ID: "live-1"}}}}) - if !strings.Contains(m.flash, "no session archive") { - t.Fatalf("flash = %q, want it to say there is no archive", m.flash) - } - m.flash = "" - m.Update(historyLoadedMsg{list: apiclient.SessionList{Sessions: []session.SessionSummary{{ID: "live-1"}}}}) - if m.flash != "" { - t.Fatalf("the notice repeated on the next poll: %q", m.flash) + if want := relTime(m.clock(), old); !rowHas(m.sessionsTbl.Rows()[1], want) { + t.Fatalf("disk-only row %v does not show %q", m.sessionsTbl.Rows()[1], want) } } @@ -87,10 +71,10 @@ func rowHas(row []string, cell string) bool { return false } -// The whole path, against a real session API with a real archive: history from a previous -// process, H to list it, Enter on its row, and the timeline arrives — through the same snapshot -// agentop uses for a live session, since the server pages into the archive for it. -func TestHistory_EnterOnAnArchivedRowOpensItsTimeline(t *testing.T) { +// The whole path against a real session API and archive: history from a previous process is +// listed on the first fetch, with no key pressed, and Enter on it opens its timeline through the +// same snapshot a live session uses, since the server pages into the archive for it. +func TestHistory_AnArchivedSessionIsListedAndOpens(t *testing.T) { resetSettingsForTest(t) // New seeds the filter from Settings; another test's must not hide the row root := t.TempDir() prev, err := archive.Open(root) @@ -119,10 +103,10 @@ func TestHistory_EnterOnAnArchivedRowOpensItsTimeline(t *testing.T) { m := New(context.Background(), apiclient.New(ts.URL)).(*model) m.width, m.height = 120, 40 m.pane = paneSessions - m.Update(m.handleKey(keyRune('H'))()) + m.Update(m.loadSessionsCmd()()) idx := slices.Index(m.sessionRowIDs, "old") if idx < 0 { - t.Fatalf("the archived session is not listed with history on: %v", m.sessionRowIDs) + t.Fatalf("the archived session is not listed: %v", m.sessionRowIDs) } m.sessionsTbl.SetCursor(idx) cmd := m.handleKey(tea.KeyMsg{Type: tea.KeyEnter}) @@ -135,9 +119,28 @@ func TestHistory_EnterOnAnArchivedRowOpensItsTimeline(t *testing.T) { } } -// Adding [H] must not cost the cost keys their place at 80 columns: helpView's comments promise -// [u] and [$] survive that cut, and fitHintLine drops hints from the front. -func TestSessionsFooter_HDoesNotPushTheCostKeysOffAt80Columns(t *testing.T) { +// A proxy without an archive — any cluster sidecar — gets the same request, ignores the parameter +// and lists what it holds, and agentop says nothing about history it was never offered. +func TestHistory_AProxyWithoutAnArchiveListsItsSessionsQuietly(t *testing.T) { + resetSettingsForTest(t) + store := session.New(0, 0, 0) + defer store.Close() + store.Append("live", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionRequest}) + ts := httptest.NewServer(sessionapi.New(":0", store).Server().Handler) + defer ts.Close() + + m := New(context.Background(), apiclient.New(ts.URL)).(*model) + m.width, m.height = 120, 40 + m.pane = paneSessions + m.Update(m.loadSessionsCmd()()) + if !slices.Contains(m.sessionRowIDs, "live") || m.flash != "" { + t.Fatalf("rows %v flash %q, want live listed and no flash", m.sessionRowIDs, m.flash) + } +} + +// The sessions footer keeps its cost keys at 80 columns and offers no history key: history is +// always on. +func TestSessionsFooter_KeepsTheCostKeysAndHasNoHistoryKey(t *testing.T) { m := &model{pane: paneSessions} line := fitHintLine(m.helpView(), 80) for _, want := range []string{"[u] usage", "[$] spend", "[?] keys", "[q] quit"} { @@ -145,11 +148,33 @@ func TestSessionsFooter_HDoesNotPushTheCostKeysOffAt80Columns(t *testing.T) { t.Errorf("at 80 columns the sessions footer lost %q: %q", want, line) } } - if wide := fitHintLine(m.helpView(), 120); !strings.Contains(wide, "[H] history") { - t.Errorf("at 120 columns the footer does not advertise [H]: %q", wide) + for _, viaAgents := range []bool{false, true} { + m.sessionsViaAgents = viaAgents + if wide := fitHintLine(m.helpView(), 140); strings.Contains(wide, "[H]") { + t.Errorf("viaAgents=%v: the footer still offers [H]: %q", viaAgents, wide) + } + } +} + +// A row only the archive holds is never what the harvest is waiting on. Thirty days of history +// carry rows nothing can name — default, pending:*, agents with no transcripts on this machine — +// and counting them held the gate open, so the transcript tree was re-walked at the backoff cap +// forever. The same row, once its session resumes, is resident and counts again. +func TestHarvestGate_IgnoresRowsOnlyTheArchiveHolds(t *testing.T) { + no := false + m := newTitleModel(t, nil) + m.sessions = []session.SessionSummary{{ID: "default", UpdatedAt: time.Now().Add(-time.Hour), Resident: &no}} + if m.untitledSettled(time.Now()) || m.untitledFresh() { + t.Fatal("an archive-only row opened the harvest gate") + } + if counted, fresh := m.countUntitled(); len(counted) != 0 || fresh { + t.Fatalf("countUntitled counted an archive-only row: %v %v", counted, fresh) + } + if m.harvestNamedSomething(map[string]SessionMetadata{"default": {Title: "named"}}) { + t.Fatal("a harvest naming an archive-only row counted as progress") } - m.sessionsViaAgents = true - if wide := fitHintLine(m.helpView(), 120); !strings.Contains(wide, "[H] history") || !strings.Contains(wide, "[esc] agents") { - t.Errorf("at 120 columns the footer reached through the agents picker does not advertise [H] and [esc] agents: %q", wide) + m.sessions[0].Resident = nil + if !m.untitledSettled(time.Now()) || !m.untitledFresh() { + t.Fatal("the same row, resident, did not open the gate") } } diff --git a/cmd/agentop/tui/keys.go b/cmd/agentop/tui/keys.go index d91980dc0..aaccd5b8c 100644 --- a/cmd/agentop/tui/keys.go +++ b/cmd/agentop/tui/keys.go @@ -805,21 +805,6 @@ func (m *model) handleKey(msg tea.KeyMsg) tea.Cmd { // detail viewport pages too. return m.pageActivePane(msg) - case "H": - // History: also list the sessions the proxy's session archive holds and memory no - // longer does. Capital because lowercase `h` backs out. Below the modal blocks for `P`'s - // reason (see there), so it needs no guards of its own. - if m.client == nil || m.pane != paneSessions { - return nil - } - m.showHistory = !m.showHistory - m.historyNoticed = false - if !m.showHistory { - m.archiveUsage = nil - m.setFlash("history off: showing live sessions") - } - return m.loadSessionsCmd() - case "X": // Clear all history, after a confirmation. Capital, and off the footer like `A`: a // destructive key should not be one stray press of a common letter, nor advertised on @@ -1184,20 +1169,13 @@ func (m *model) helpView() string { // [A] agents is absent for width: the line is already 98 columns, and with one agent — // the common case — the breakdown is a single row. The [?] overlay names it. esc names // the picker instead when this list was reached through it. - // - // [H] history sits AHEAD of [u], so it is the hint that pays for its own width. It makes - // the line 111 columns wherever it goes; placed behind [$] that cost both cost keys at - // 80, while here fitHintLine drops it before them, and every hint behind it survives to - // exactly the width it did before [H] existed. It is visible from 92 columns, and the - // [?] overlay names it at any width. (The 98 measured above is the line before [H]; and - // [P]'s 34 is unchanged, being a fit from the tail.) if m.sessionsViaAgents { - return "[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [esc] agents [p] pause [P] pipeline [?] keys [q] quit" + return "[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [esc] agents [p] pause [P] pipeline [?] keys [q] quit" } if m.parentCtx != nil { - return "[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [esc] pods [p] pause [P] pipeline [?] keys [q] quit" + return "[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [esc] pods [p] pause [P] pipeline [?] keys [q] quit" } - return "[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit" + return "[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit" case paneEvents: skipHint := "[s] hide passthru/skip" if m.hideInactive { diff --git a/cmd/agentop/tui/session_metadata.go b/cmd/agentop/tui/session_metadata.go index 5c1832d0c..dc8bb658c 100644 --- a/cmd/agentop/tui/session_metadata.go +++ b/cmd/agentop/tui/session_metadata.go @@ -10,6 +10,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/rossoctl/cortex/core/observe/claude" + "github.com/rossoctl/cortex/core/session" ) // SessionMetadata is what a coding agent knows about one of its own sessions that Cortex @@ -217,6 +218,13 @@ func harvestCmd(h HarvestFunc) tea.Cmd { } } +// awaitsHarvest reports whether s is a row the title harvest could still be waiting on: one the +// proxy holds in memory. A row only the session archive holds is history — its title came from +// the metadata cache while it was live, or there is none to find — and with thirty days of such +// rows listed, counting them would hold the gate open on rows nothing can name. A session that +// resumes is resident again, and counts again. +func awaitsHarvest(s session.SessionSummary) bool { return s.Resident == nil || *s.Resident } + // untitledSettled reports whether some session on screen has no title yet and has been // quiet long enough that its transcript is probably complete on disk. // @@ -250,18 +258,21 @@ func harvestCmd(h HarvestFunc) tea.Cmd { // // IT IS NOT FREE, THOUGH, and an earlier version of this comment claimed "one map lookup per row // per tick", which undersells it: sessionHasTitle goes through sessionTitle, which calls -// sanitizeLabel, which builds a new string. So the steady state allocates once per row per 2s -// tick and always walks the whole list — the all-titled case is the one that cannot exit early, -// because the loop is looking for a row that is not there. -// -// Left as a linear walk deliberately. The rows here are one pod's live sessions, a handful in -// practice against the ~180 in the metadata file, and the alternative — a cached "any untitled" -// flag — is a second piece of state to invalidate on every sessionsLoadedMsg and every harvest -// merge, which is how the events map grew the bugs its own comments now document. The honest -// figure is in the comment; the optimisation waits for a profile that asks for it. +// sanitizeLabel, which builds a new string. So the steady state allocates once per resident row +// per 2s tick and always walks the whole list — the all-titled case is the one that cannot exit +// early, because the loop is looking for a row that is not there. +// +// Left as a linear walk deliberately. The rows that cost anything here are one pod's live +// sessions, a handful in practice against the ~180 in the metadata file: with a session archive +// the list also holds every session the archive keeps — ~2,000 over thirty days — but +// awaitsHarvest skips those archive-only rows before the allocating sessionHasTitle call, so each +// of them costs a pointer check. And the alternative — a cached "any untitled" flag — is a second +// piece of state to invalidate on every sessionsLoadedMsg and every harvest merge, which is how +// the events map grew the bugs its own comments now document. The honest figure is in the +// comment; the optimisation waits for a profile that asks for it. func (m *model) untitledSettled(now time.Time) bool { for _, s := range m.sessions { - if m.sessionHasTitle(s.ID) { + if !awaitsHarvest(s) || m.sessionHasTitle(s.ID) { continue } // A ZERO UpdatedAt IS NOT "QUIET SINCE THE EPOCH". The field is whatever /v1/sessions @@ -325,7 +336,7 @@ func (m *model) untitledSettled(now time.Time) bool { // ordering the scoring's `fresh` arm exists to avoid. func (m *model) untitledFresh() bool { for _, sess := range m.sessions { - if m.sessionHasTitle(sess.ID) { + if !awaitsHarvest(sess) || m.sessionHasTitle(sess.ID) { continue } if !m.untitledCounted[sess.ID] { @@ -346,7 +357,7 @@ func (m *model) untitledFresh() bool { func (m *model) countUntitled() (counted map[string]bool, fresh bool) { counted = make(map[string]bool, len(m.sessions)) for _, sess := range m.sessions { - if m.sessionHasTitle(sess.ID) { + if !awaitsHarvest(sess) || m.sessionHasTitle(sess.ID) { continue } counted[sess.ID] = true @@ -452,7 +463,7 @@ func blankSanitized(sanitized string) bool { // arriving for a session nobody is looking at is not why the backoff exists. func (m *model) harvestNamedSomething(meta map[string]SessionMetadata) bool { for _, sess := range m.sessions { - if m.sessionHasTitle(sess.ID) { + if !awaitsHarvest(sess) || m.sessionHasTitle(sess.ID) { continue } if !titleIsBlank(meta[sess.ID].Title) { diff --git a/cmd/agentop/tui/sessions_context.go b/cmd/agentop/tui/sessions_context.go index 207e7a106..79bd8a448 100644 --- a/cmd/agentop/tui/sessions_context.go +++ b/cmd/agentop/tui/sessions_context.go @@ -160,9 +160,3 @@ func contextGauge(promptTokens, width int) string { // TestSessionsPicker_CachedMarkerRendersIntact, which exists because a styled cell had its // escape bytes measured against the column width and came out mangled. const cachedMarker = "cached" - -// archivedMarker names a row the proxy's session archive served because memory no longer holds -// the session (H). It takes the UPDATED cell, as cachedMarker does, because it is the one thing -// on the row saying why there is no live session behind it; the list's order still says how -// recent it is. -const archivedMarker = "archived" diff --git a/cmd/agentop/tui/sessions_context_bench_test.go b/cmd/agentop/tui/sessions_context_bench_test.go index bc0e6ef23..4e8e5014c 100644 --- a/cmd/agentop/tui/sessions_context_bench_test.go +++ b/cmd/agentop/tui/sessions_context_bench_test.go @@ -25,8 +25,9 @@ func benchContextEvents(n int) []pipeline.SessionEvent { } // THE SHAPE THAT MATTERS: one event arrives, and the sessions row loop asks every session for its -// gauge. rebuildSessionsTable runs on every streamed event and retention is unbounded, so a rescan -// here is O(events) per session per event — on the pane agentop opens on. +// gauge. rebuildSessionsTable runs on the 1s tick after streamed events and on every 2s poll, +// rather than per event, and retention is unbounded, so a rescan here is O(events) per session per +// rebuild — on the pane agentop opens on. // // This benchmark exists because that regression shipped once. The first version of the rule // made two full passes and allocated a map per call; measured at 100k events it cost 6.10ms and diff --git a/cmd/agentop/tui/sessions_context_fold_test.go b/cmd/agentop/tui/sessions_context_fold_test.go index f35f1b419..b01aa76cd 100644 --- a/cmd/agentop/tui/sessions_context_fold_test.go +++ b/cmd/agentop/tui/sessions_context_fold_test.go @@ -12,9 +12,10 @@ import ( // runs in production and the rescan is the definition. // // Why a fold at all: the sessions row loop asks for every session, rebuildSessionsTable runs on -// every streamed event, and retention is unbounded. Measured before this — 6.1ms and 3.49MB per -// call at 100k events, against ~14ns for the tail scan it replaced — ten sessions of that size -// cost 60ms and 35MB for one arriving event, on the pane agentop opens on. +// the 1s tick after streamed events and on every 2s poll rather than per event, and retention is +// unbounded. Measured before this — 6.1ms and 3.49MB per call at 100k events, against ~14ns for +// the tail scan it replaced — ten sessions of that size cost 60ms and 35MB per rebuild, on the +// pane agentop opens on. func TestSessionContextFor_FoldMatchesAFullRescan(t *testing.T) { base := time.Now() const id = "s" diff --git a/cmd/agentop/tui/sessions_pane.go b/cmd/agentop/tui/sessions_pane.go index bfd9c3bcd..98a5e4916 100644 --- a/cmd/agentop/tui/sessions_pane.go +++ b/cmd/agentop/tui/sessions_pane.go @@ -296,12 +296,8 @@ func (m *model) rebuildSessionsTable() { if agentW > 0 { row = append(row, sessionAgentCell(s, agentW)) } - updated := relTime(now, s.UpdatedAt) - if s.Resident != nil && !*s.Resident { - updated = archivedMarker - } row = append(row, - updated, + relTime(now, s.UpdatedAt), // The server's count, and only ever the server's: it is the complete one. // agentop's own cache holds what it snapshotted plus what it has streamed // since attaching, which for a session older than the connection is a @@ -324,8 +320,10 @@ func (m *model) rebuildSessionsTable() { } // Sessions whose events agentop still holds but the server no longer lists. // Retaining the events (#870) is only half a fix if there is no row to - // select them from: after a proxy restart the server lists nothing, so - // without this the picker is empty and the retained history is unreachable. + // select them from: a proxy without a session archive lists nothing after a + // restart, so without this the picker is empty and the retained history is + // unreachable. One with an archive lists everything it holds, and these rows + // are then mostly absent. adopted := m.adoptedSessionIDs() for _, id := range m.cachedOnlySessionIDs() { // A scope lists only sessions the server names an agent for, and an adopted pending bucket diff --git a/cmd/agentop/tui/sessions_stream_bench_test.go b/cmd/agentop/tui/sessions_stream_bench_test.go new file mode 100644 index 000000000..0435e4a96 --- /dev/null +++ b/cmd/agentop/tui/sessions_stream_bench_test.go @@ -0,0 +1,64 @@ +package tui + +import ( + "fmt" + "testing" + "time" + + "github.com/rossoctl/cortex/cmd/agentop/apiclient" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// streamBenchModel is a sessions pane listing n sessions, newest first, its table built. +func streamBenchModel(n int) *model { + now := time.Now() + m := &model{pane: paneSessions, width: 200, height: 40, + events: map[string][]pipeline.SessionEvent{}, sessionsTbl: newSessionsTable()} + for i := range n { + m.sessions = append(m.sessions, session.SessionSummary{ + ID: fmt.Sprintf("%08d-0000-4000-8000-000000000000", i), Title: fmt.Sprintf("session %d", i), + UpdatedAt: now.Add(-time.Duration(i) * time.Minute), EventCount: 100 + i, TotalTokens: 1000 * i, + Agent: "claude-code", + }) + } + m.rebuildSessionsTable() + return m +} + +// BenchmarkHandleStreamEvent is one streamed event against a list of n sessions. With history +// listed n is ~2,000 on a laptop (thirty days), where re-sorting and rebuilding the table on every +// event cost 6.4ms and 1.9MB, against 0.55ms at 10 rows (measured 2026-10-05). The rebuild now +// waits for the 1s tick, so its cost is gone from here. What still grows with n is the linear +// lookup of the event's session in the list, and the target sits at n/2. +func BenchmarkHandleStreamEvent(b *testing.B) { + for _, n := range []int{10, 2000} { + b.Run(fmt.Sprint(n), func(b *testing.B) { + m := streamBenchModel(n) + ev := &pipeline.SessionEvent{SessionID: m.sessions[n/2].ID, Phase: pipeline.SessionRequest} + b.ReportAllocs() + for b.Loop() { + ev.At = time.Now() + m.handleStreamEvent(apiclient.StreamEvent{Event: ev}) + // Truncated, capacity kept: left to grow over millions of iterations it is GBs of heap. + m.events[ev.SessionID] = m.events[ev.SessionID][:0] + } + }) + } +} + +// A streamed event marks the list stale instead of rebuilding it, and the next tick re-sorts and +// rebuilds once, however many events arrived in between. +func TestStreamEvent_RebuildsTheSessionsTableOnTheNextTick(t *testing.T) { + m := streamBenchModel(3) + last := m.sessions[2].ID + m.handleStreamEvent(apiclient.StreamEvent{Event: &pipeline.SessionEvent{ + SessionID: last, At: time.Now(), Phase: pipeline.SessionRequest}}) + if m.sessionRowIDs[0] == last { + t.Fatal("the table was rebuilt on the event itself") + } + m.Update(tickMsg(time.Now())) + if m.sessionRowIDs[0] != last { + t.Fatalf("after the tick the newest session is not first: %v", m.sessionRowIDs) + } +} diff --git a/cmd/agentop/tui/tunnel_close_test.go b/cmd/agentop/tui/tunnel_close_test.go index ede7b3832..a3d754790 100644 --- a/cmd/agentop/tui/tunnel_close_test.go +++ b/cmd/agentop/tui/tunnel_close_test.go @@ -95,6 +95,9 @@ func TestHandleStreamEvent_TunnelCloseDoesNotRefreshTheSession(t *testing.T) { At: time.Now(), SessionID: "sess-A", Direction: pipeline.Outbound, Phase: pipeline.SessionResponse, Host: "kube:6443", Tunnel: true, RequestID: "tun", StatusCode: 200, }}) + // The list is re-sorted on the tick, not on the event. Without the flush nothing sorts it + // and the order below would hold whatever the close did. + m.flushSessionsTable() if m.sessions[0].ID != "sess-B" { t.Errorf("session list leads with %q after a tunnel close; want sess-B, which spoke last", m.sessions[0].ID) diff --git a/cmd/cortex/archive.go b/cmd/cortex/archive.go index 2cd56cf46..628f53a0f 100644 --- a/cmd/cortex/archive.go +++ b/cmd/cortex/archive.go @@ -26,7 +26,7 @@ func sessionArchiveDir() (string, error) { // sessionArchiveRuns decides whether the session archive runs, and says why. // // ONLY ON A LOCAL INSTALL. A laptop is where a restart costs a user their history and where the -// user can read it back (agentop's H) and erase it (X, or DELETE /v1/sessions). Elsewhere it is +// user can read it back (agentop lists it) and erase it (X, or DELETE /v1/sessions). Elsewhere it is // off even when asked for — raw prompts on a cluster's volume need a decision of their own, and a // mounted path is not one. On a local install, config.ArchiveRunsOnLocalInstall decides. func sessionArchiveRuns(cfg *config.Config, configPath string) (bool, string) { diff --git a/docs/assets/cortex-demo.svg b/docs/assets/cortex-demo.svg index deffbe212..bcc4226d3 100644 --- a/docs/assets/cortex-demo.svg +++ b/docs/assets/cortex-demo.svg @@ -282,7 +282,7 @@ clipPath rect{width:100000px} ● connected0.0 events/sec feedback: https://github.com/rossoctl/cortex/issues/new/choose -[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit +[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit agentop · http://127.0.0.1:9094 @@ -319,7 +319,7 @@ clipPath rect{width:100000px} ● connected0.0 events/sec feedback: https://github.com/rossoctl/cortex/issues/new/choose -[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit +[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit agentop · http://127.0.0.1:9094 · usage · all @@ -493,6 +493,6 @@ clipPath rect{width:100000px} ● connected0.0 events/sec feedback: https://github.com/rossoctl/cortex/issues/new/choose -[↑↓] nav [↵] drill [H] history [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit +[↑↓] nav [↵] drill [u] usage [$] spend [/] filter [p] pause [P] pipeline [?] keys [q] quit diff --git a/docs/laptop-service.md b/docs/laptop-service.md index 1a14ab032..49b4ee4f2 100644 --- a/docs/laptop-service.md +++ b/docs/laptop-service.md @@ -300,9 +300,8 @@ Two other knobs, same restart rule: **A local install keeps every session on disk, on by default.** Each event the session store records is also written to `~/.cortex/sessions`, so a session outlives a restart and the store's -own eviction (`session.max_sessions`, a `session.ttl`). In `agentop`, `H` on the sessions pane -lists the sessions only the archive still holds, marked `archived`, and Enter opens one like any -other. +own eviction (`session.max_sessions`, a `session.ttl`). `agentop` lists every session the archive +holds beside the ones in memory, and Enter opens one like any other. **Unlike the cost ledger, this is the content.** Prompts, completions, tool arguments and tool results — everything the session API serves — sit in diff --git a/docs/session-dump.md b/docs/session-dump.md index 1e981cb1a..5cb58cccf 100644 --- a/docs/session-dump.md +++ b/docs/session-dump.md @@ -3,7 +3,7 @@ > **On a laptop the proxy now keeps sessions itself.** A local install writes every > session to `~/.cortex/sessions` ([#901](https://github.com/rossoctl/cortex/issues/901); > see [the laptop guide](laptop-service.md#session-history-is-kept-in-cortexsessions)), -> so a restart no longer loses them and agentop's `H` lists them. What this script is +> so a restart no longer loses them and agentop keeps listing them. What this script is > still for is getting sessions *out* as files — and it is the only way to keep anything > from a cluster sidecar, which has no archive. A whole dump covers the sessions in > memory; `--session ` also reaches one only the archive holds, because the From 3db2cf886d3d4ecca4d76a370b4d172e89eb3d88 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 6 Oct 2026 14:36:20 -0400 Subject: [PATCH 2/5] feat: Continue a resumed session's figures from the session archive After a restart a resumed session listed only what memory held since: EVENTS 4 for a session of 82, its cost and tokens from zero, and a new title from its first post-restart message. The archive already folds, for each store entry, the session's history before that entry began (begin's before, captured in queue order, so exact). It now publishes it, and answers a new session.PriorKeeper lookup; ListSessions merges it into each resident row by each figure's own rule: counts and tokens add, cost through CostSum, currencies over the unit maps, the title through applyTitle, CTX through MergePromptContext, createdAt as the earlier. Where the archive cannot say exactly what came before, the row keeps the entry's own figures: an undercount, never a double count. The archive's fold now carries CTX, so archive-only rows show it too. With a claim owner, the owner's label is looked up in the earlier history first, since it is that agent's first label across the whole session. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- CLAUDE.md | 4 +- cmd/agentop/README.md | 12 +- cmd/agentop/tui/app.go | 11 +- cmd/agentop/tui/help_overlay.go | 17 ++- cmd/agentop/tui/help_overlay_test.go | 6 +- cmd/agentop/tui/sessions_context.go | 7 +- cmd/agentop/tui/sessions_pane.go | 7 +- core/session/archive/archive.go | 17 ++- core/session/archive/prior_test.go | 134 +++++++++++++++++ core/session/archive/read.go | 21 +++ core/session/continuity_test.go | 208 +++++++++++++++++++++++++++ core/session/fold.go | 65 ++++++--- core/session/fold_test.go | 52 ++++++- core/session/history.go | 92 +++++++++++- core/session/store.go | 55 ++++--- core/sessionapi/archive.go | 3 +- core/sessionapi/archive_test.go | 26 ++++ docs/laptop-service.md | 4 +- docs/tool-prune-plugin.md | 5 +- 19 files changed, 670 insertions(+), 76 deletions(-) create mode 100644 core/session/archive/prior_test.go create mode 100644 core/session/continuity_test.go diff --git a/CLAUDE.md b/CLAUDE.md index b568822df..495945eb2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -505,7 +505,7 @@ When `session.enabled` is true (default) and `listener.session_api_addr` is non- | Method & Path | Format | Purpose | |---|---|---| | `GET /` | text | One-line-per-endpoint index. Answers "is this the session API, and on the right port?" — the reason a 404 here was worth replacing. | -| `GET /v1/sessions` | `application/json` | List active sessions: `{sessions: [{id, createdAt, updatedAt, eventCount, title, agent, adopted, totalTokens, costMicros, avoidedMicros, saturated, active, promptContext}]}`. `id`, `createdAt`, `updatedAt`, `eventCount` and `active` are always present; every other field is `omitempty` — absent rather than zero, on the standing rule that an unknown value must not render as a real one. (Do not read that off the position of `active`: it sits second-to-last, between two `omitempty` fields.) **`title` is a suggestion, not an identifier:** the proxy derives it from the session's own events (a `/rename`, else a ``, else ordinary user prose, with `` blocks excised), so it is a display convenience and nothing addresses a session by it. Absent when nothing in the events named it. **Folded at append time and FIRST-WINS, except that a `/rename` always overrides** — so ordinary conversation does not re-title a session on every turn, and a `/rename` survives eviction of the event that carried it. agentop reads this field as a FALLBACK: its TITLE column prefers a harvested Claude Code transcript title and uses the served title only for a session the harvest cannot name. That precedence is fixed rather than a judgement about which string is better — both sides rank candidates their own way and do not agree on every session. That is the case worth having: an agent with no transcript tree on the operator's disk still routes through the proxy, so a row that used to render blank now has a name. agentop deliberately still treats such a row as unnamed for its own re-harvest backoff, so a served title does not stop it looking for a harvested one. **`?archived=true` adds history** where the proxy runs a session archive (`session.archive`, laptop only — see the chatty-traffic gotcha): every session the archive holds and memory does not, marked `resident: false`, plus a top-level `archive` object — `{bytes, maxBytes, retentionDays}` and, only when nonzero, `droppedEvents`, `writeErrors`, `droppedRenames`, `paused`, any of which means the history has gaps. A session in both is listed once, as its resident row. Without an archive the parameter is ignored and no `archive` object is sent, which is how a client tells "no history here" from "no history yet". | +| `GET /v1/sessions` | `application/json` | List active sessions: `{sessions: [{id, createdAt, updatedAt, eventCount, title, agent, adopted, totalTokens, costMicros, avoidedMicros, saturated, active, promptContext}]}`. `id`, `createdAt`, `updatedAt`, `eventCount` and `active` are always present; every other field is `omitempty` — absent rather than zero, on the standing rule that an unknown value must not render as a real one. (Do not read that off the position of `active`: it sits second-to-last, between two `omitempty` fields.) **`title` is a suggestion, not an identifier:** the proxy derives it from the session's own events (a `/rename`, else a ``, else ordinary user prose, with `` blocks excised), so it is a display convenience and nothing addresses a session by it. Absent when nothing in the events named it. **Folded at append time and FIRST-WINS, except that a `/rename` always overrides** — so ordinary conversation does not re-title a session on every turn, and a `/rename` survives eviction of the event that carried it. agentop reads this field as a FALLBACK: its TITLE column prefers a harvested Claude Code transcript title and uses the served title only for a session the harvest cannot name. That precedence is fixed rather than a judgement about which string is better — both sides rank candidates their own way and do not agree on every session. That is the case worth having: an agent with no transcript tree on the operator's disk still routes through the proxy, so a row that used to render blank now has a name. agentop deliberately still treats such a row as unnamed for its own re-harvest backoff, so a served title does not stop it looking for a harvested one. **`?archived=true` adds history** where the proxy runs a session archive (`session.archive`, laptop only — see the chatty-traffic gotcha): every session the archive holds and memory does not, marked `resident: false`, plus a top-level `archive` object — `{bytes, maxBytes, retentionDays}` and, only when nonzero, `droppedEvents`, `writeErrors`, `droppedRenames`, `paused`, any of which means the history has gaps. A session in both is listed once, as its resident row. **That row covers the session's whole history**: `eventCount`, `totalTokens`, `costMicros`, `avoidedMicros`, `currencies`, `saturated`, `title`, `agent`, `createdAt` and `promptContext` continue from the archive's fold of what came before this process's entry, each by its own rule, in the default list too — so with an archive and `session.max_events` unset (the default) a restart changes no figure, and without an archive they start over. Without an archive the parameter is ignored and no `archive` object is sent, which is how a client tells "no history here" from "no history yet". | | `GET /v1/sessions/{id}` | `application/json` | The session's most recent events. `?limit=N` (default 500, max 2000) sets the window; `?before=` returns the page ending just before that event, so the whole session is reachable by paging backward from the tail. `totalEvents` is the session's true length and `oldestSeq` the oldest event the store still holds — both present only when this response is not the whole session, so a client can tell "this is the beginning" from "there is more behind me" without a second request. 404 if unknown/expired. **One response is still not a full snapshot:** with `session.max_events` unset a session can hold thousands of events, and one real session's whole history encoded to 1.1GB — 17s to write, against clients that time out in 10. That cap is why `before` exists — until it did, a session past 2000 events had a beginning no request could reach at any limit, while still costing memory. The response is written one event at a time rather than encoded whole, so serving it costs the proxy heap proportional to one event; see the chatty-traffic gotcha below. **With a session archive, a page continues from disk** below the oldest event memory holds: a session resumed after a restart pages back through its whole history, one evicted from memory is served instead of 404'd, and `totalEvents`/`oldestSeq` describe the merged history. `GET /v1/sessions/{id}/events/{seq}` falls back to the archive the same way, and answers 404 rather than a neighbour when neither holds that seq. | | `DELETE /v1/sessions` | `application/json` | Clear every session: the store, and the session archive where one runs. **The cost ledger is kept** — spend totals, no content. Answers `{sessions, archivedSessions, bytes}`; a failed disk half is a 500 adding `archiveError`, with memory cleared regardless (the store clears under its lock before the archive is waited on, up to 10s). **The one state-changing route on this unauthenticated API, so it is guarded against a browser**, each guard pinned by its own test in `core/sessionapi/clear_test.go`: DELETE only (a cross-site DELETE needs a preflight this server never approves; a simple cross-site POST needs none); a loopback `Host` — exactly `localhost`, `127.0.0.1` or `::1`, which defeats DNS rebinding; no `Origin` header; and `WithClearAllowed`, which `cmd/cortex` sets from `listener.bind_loopback_only`. A refusal is a 403 `{"error": ...}`. agentop's `X` drives it. | | `GET /v1/events` | `text/event-stream` | SSE stream of new events. Optional `?session=` filters to one session. Heartbeat every 30s. | @@ -1095,7 +1095,7 @@ resulting `/shared/client-id.txt` and `/shared/client-secret.txt`. **And the client is the other end of the same problem.** Measured on a laptop, `agentop` sat at 1.64GB against the proxy's 1.03GB — larger than the process it was watching, and the only one of the two still climbing. It made both mistakes the server had just stopped making: it decoded each snapshot as one document (`json.Decoder` only bounds its buffer between *values*, and a whole snapshot is one value), and it kept every string the server's encoder had expanded back out of the store's sharing. Both are fixed in `cmd/agentop/apiclient/snapshot.go`: the response is decoded one event at a time and fed through the same `session.Interner` the store uses, exported for the purpose rather than reimplemented. On a 23.5MB document, `+32.0MB HeapSys` and 26.49MB retained became +0.0MB and 2.31MB (`BenchmarkDecodeSessionViewHeap`). When a memory figure here looks wrong, measure **both** processes — the proxy has not been the larger one for a while. - **The session archive is disk, not memory** (`core/session/archive`, `session.archive`, on by default for a local install bound to loopback only and off everywhere else — `config.ArchiveRunsOnLocalInstall` says why; `DELETE /v1/sessions` erases it with the store). It relieves the store of nothing: every bound above applies unchanged with it on. What it changes is what eviction *costs* — an evicted session is still listed under `?archived=true` and paged from disk, and a `max_events` trim drops nothing a reader can no longer reach — so with it on, a cap or a `ttl` stops being a trade against history. It records through the same `Recorder` hook as the cost pipeline, off the request path: a burst the writer cannot keep up with is dropped from the archive and counted, never from memory. On disk it interns the same five fields the store does, one zstd segment at a time, and it is bounded by `retention_days` (30) and `max_bytes` (2 GiB), not by anything here. `docs/laptop-service.md` has the user-facing version, including what the files contain. + **The session archive is disk, not memory** (`core/session/archive`, `session.archive`, on by default for a local install bound to loopback only and off everywhere else — `config.ArchiveRunsOnLocalInstall` says why; `DELETE /v1/sessions` erases it with the store). It relieves the store of nothing: every bound above applies unchanged with it on. What it changes is what eviction *costs* — an evicted session is still listed under `?archived=true` and paged from disk, a resumed one lists its whole history, and a `max_events` trim drops nothing a reader can no longer reach — so with it on, a cap or a `ttl` stops being a trade against history. It records through the same `Recorder` hook as the cost pipeline, off the request path: a burst the writer cannot keep up with is dropped from the archive and counted, never from memory. On disk it interns the same five fields the store does, one zstd segment at a time, and it is bounded by `retention_days` (30) and `max_bytes` (2 GiB), not by anything here. `docs/laptop-service.md` has the user-facing version, including what the files contain. Two layered defenses keep the inbound A2A user intent visible to IBAC even when an agent generates dozens of outbound events per turn: diff --git a/cmd/agentop/README.md b/cmd/agentop/README.md index 3f04d437a..40aa4cd07 100644 --- a/cmd/agentop/README.md +++ b/cmd/agentop/README.md @@ -772,11 +772,13 @@ agentop is for, and the other three are surfaces you visit and leave. is a check on the other. The table is also **not** a longer span than the band, which is the reading - worth heading off. The session store is in memory, so a session's figures only - reach back as far as the current proxy process — on a freshly restarted proxy - the whole COST column can sum to less than `TODAY`, because `TODAY` comes from - the durable cost ledger and survives restarts. `[?]` states both facts; the - title deliberately does not, since no single span is true of every row. + worth heading off. With a session archive (a local install) a session's figures + reach back through restarts to its first archived event. Without one — any + cluster sidecar — the store is in memory, so they reach back only as far as the + current proxy process, and on a freshly restarted proxy the whole COST column + can sum to less than `TODAY`, because `TODAY` comes from the durable cost ledger + and survives restarts. `[?]` states both facts; the title deliberately does + not, since no single span is true of every row. Rendered at 100 columns, where every column has its declared width; 93 is the narrowest terminal that carries them all at once: diff --git a/cmd/agentop/tui/app.go b/cmd/agentop/tui/app.go index 0f4b7dd13..b2551b9c3 100644 --- a/cmd/agentop/tui/app.go +++ b/cmd/agentop/tui/app.go @@ -2298,11 +2298,12 @@ func (m *model) paneView() string { // - "lifetime" NAMES A SPAN, and this table has no single span. Each row covers its own // session, first event to last, and no two rows need cover the same duration. There // was no one duration for the word to be true about. - // - WHERE IT DID IMPLY A SPAN, it implied the wrong one. The session store is in memory - // and resets when the proxy restarts, so a session's lifetime cannot exceed proxy - // uptime — measured on a freshly restarted local proxy, the COST column summed to - // $4.04, matching the band's rolling hour, while the band's day read $18.80. The word - // that sounds like "everything ever" was labelling the SHORTEST span on screen. + // - WHERE IT DID IMPLY A SPAN, it implied the wrong one. Where no session archive runs, + // the store is in memory and resets when the proxy restarts, so a session's lifetime + // cannot exceed proxy uptime — measured on a freshly restarted local proxy before it + // kept one, the COST column summed to $4.04, matching the band's rolling hour, while + // the band's day read $18.80. The word that sounds like "everything ever" was + // labelling the SHORTEST span on screen. // - It sat at the end of the title, one line above a band whose nearest cells are // explicitly clock-windowed, so it read as covering those too. // diff --git a/cmd/agentop/tui/help_overlay.go b/cmd/agentop/tui/help_overlay.go index 4e854636f..d0a478a8b 100644 --- a/cmd/agentop/tui/help_overlay.go +++ b/cmd/agentop/tui/help_overlay.go @@ -123,15 +123,20 @@ var spendDrawerKeys = keyGroup{ // title could only name a span, and the table has no single span to name (see paneView's // sessions case). // - // "resets on proxy restart" rather than "lifetime", which is the correction that motivated - // dropping the title note: the store is in memory, so a session's figures cover only as far - // back as the current proxy process. A reader comparing the table against the band's day - // figure and finding it smaller is seeing that, not a bug. + // "continues across proxy restarts with a session archive, resets without one" rather than + // "lifetime", which is the correction that motivated dropping the title note. Without an + // archive the store is in memory, so a session's figures cover only as far back as the + // current proxy process. With one, every row reaches back through restarts — a resident row + // adds the archive's fold of what came before its entry, and a row only the archive holds is + // that fold — but a clear or retention can still drop a session whose cost the ledger keeps. + // A reader comparing the table against the band's day figure and finding it smaller is + // seeing that, not a bug. // // A NOTE NOW, not a binding with keys:"". The same sentence, no longer pretending to be a key. notes: []string{ - "Every band cell names its own span. The sessions table is per session and resets " + - "on proxy restart, so its figures can read smaller than the band's.", + "Every band cell names its own span. The sessions table is per session: with a " + + "session archive it continues across proxy restarts, without one it resets, so its " + + "figures can read smaller than the band's.", }, } diff --git a/cmd/agentop/tui/help_overlay_test.go b/cmd/agentop/tui/help_overlay_test.go index 2c13ddb21..781b0a2bb 100644 --- a/cmd/agentop/tui/help_overlay_test.go +++ b/cmd/agentop/tui/help_overlay_test.go @@ -532,8 +532,8 @@ func TestHelpOverlayScrollKeys(t *testing.T) { // helpNoScrollHeight is a terminal tall enough to show the whole reference at // helpWideTerminal columns, so the no-affordance case is testable. // -// IT IS BIG, AND THAT IS THE POINT. The body is 99 lines once every pane carries -// its purpose and its descriptions, and 102 rows is the exact floor. The previous +// IT IS BIG, AND THAT IS THE POINT. The body is 101 lines once every pane carries +// its purpose and its descriptions, and 104 rows is the exact floor. The previous // version of this test asked for 60 and t.Skip()ed when the content did not fit — // which, the moment the body grew, silently took the three short-terminal // assertions below with it and reported PASS. A number that has to track the body's @@ -553,7 +553,7 @@ func TestHelpOverlayScrollKeys(t *testing.T) { // that no longer existed. Too small fails loudly; too large fails silently. const ( helpWideTerminal = 100 - helpNoScrollHeight = 103 + helpNoScrollHeight = 104 ) // With everything visible there must be no scroll affordance — it would be noise diff --git a/cmd/agentop/tui/sessions_context.go b/cmd/agentop/tui/sessions_context.go index 79bd8a448..606fe0b9a 100644 --- a/cmd/agentop/tui/sessions_context.go +++ b/cmd/agentop/tui/sessions_context.go @@ -12,9 +12,10 @@ import ( // with whatever the server published for that row. // // NEITHER SOURCE DOMINATES, which is why this merges rather than preferring one. The server has -// seen everything since the PROXY started; agentop only since IT attached, which is usually less — -// but agentop's copy survives a proxy restart, and destroying a figure it still holds because the -// server forgot is #870's shape. pipeline.MergePromptContext resolves it by the rule rather than by +// seen everything since the PROXY started — and, where a session archive runs, what the archive +// holds from before that; agentop only since IT attached, which is usually less. But where no +// archive runs, agentop's copy survives a proxy restart and the server's does not, and destroying +// a figure it still holds because the server forgot is #870's shape. pipeline.MergePromptContext resolves it by the rule rather than by // size: a stated figure beats an unstated one at any magnitude. // // AND MERGING MOVES THE FIGURE EITHER WAY, which "neither source dominates" must not be read as diff --git a/cmd/agentop/tui/sessions_pane.go b/cmd/agentop/tui/sessions_pane.go index 98a5e4916..55877cf00 100644 --- a/cmd/agentop/tui/sessions_pane.go +++ b/cmd/agentop/tui/sessions_pane.go @@ -65,9 +65,10 @@ func sessionsColumns() []table.Column { // Both come from the server's own sum over the session's events // (session.SessionSummary.CostMicros / .AvoidedMicros), not from the strip's ring // window: the durable ledger's row key carries no session dimension by design, and - // the ring covers only its rolling span. So these RESET when the proxy restarts - // while the strip's "today" figure does not — both are correct, and neither is a - // check on the other. + // the ring covers only its rolling span. So these continue across a proxy restart + // where it keeps a session archive and reset where it does not, while the strip's + // "today" figure always continues — both are correct, and neither is a check on the + // other. {Title: "COST", Width: 10}, {Title: "SAVED", Width: 10}, // CTX(1M) replaces an ACTIVE column that carried a ● for a flag nobody acted on. diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 6266ba3f9..25f9ceccc 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -158,7 +158,8 @@ type sessionState struct { lastWrite time.Time metaDirty bool // startAfter is where the store's current entry began numbering, once its first event - // arrived, and before is the summary of what the session held then. + // arrived, and before is the summary of what the session held then. before is never folded + // into: the reader index shares it (see indexEntry). startAfter uint64 before *session.SummaryFold } @@ -453,6 +454,9 @@ func (a *Archive) begin(id string, after uint64) { } a.closeWriter(s) s.startAfter, s.before = after, s.meta.Summary.Clone() + // Published at once rather than with the event's write: a paused or failing write publishes + // nothing, and the store's list asks for before from the moment its entry exists. + a.publish(s) } // write appends one event to its session's open segment, opening one if needed. @@ -663,7 +667,8 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte } ns.meta.Segments = append(ns.meta.Segments, seg) } - s.meta.Segments, s.meta.Summary, s.meta.UpdatedAt = keep, before, latestWrite(keep) + // A CLONE: before was published, and the old id's live fold is Added to from here on. + s.meta.Segments, s.meta.Summary, s.meta.UpdatedAt = keep, before.Clone(), latestWrite(keep) a.writeMeta(s) if ns.metaDirty { a.writeMeta(ns) @@ -856,6 +861,13 @@ type indexEntry struct { segments []SegmentInfo open bool summary session.SessionSummary + // startAfter and before are the writer's sessionState fields of the same names: where the + // store's current entry for this id began numbering, and the fold of what the session held + // then. before is SHARED with the writer rather than copied, because it is immutable: begin + // makes it as a fresh clone, nothing folds into it, and renameEntry clones it before reusing it + // as a live fold. + startAfter uint64 + before *session.SummaryFold } // publish replaces s's entry in the reader index. Writer goroutine only. @@ -867,6 +879,7 @@ func (a *Archive) publish(s *sessionState) { } e.summary = s.meta.Summary.Summary(s.meta.ID) e.summary.CreatedAt, e.summary.UpdatedAt = s.meta.CreatedAt, s.meta.UpdatedAt + e.startAfter, e.before = s.startAfter, s.before a.idxMu.Lock() a.index[s.meta.ID] = e a.idxMu.Unlock() diff --git a/core/session/archive/prior_test.go b/core/session/archive/prior_test.go new file mode 100644 index 000000000..e66c4218d --- /dev/null +++ b/core/session/archive/prior_test.go @@ -0,0 +1,134 @@ +package archive + +import ( + "testing" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +var _ session.PriorKeeper = (*Archive)(nil) + +func listed(t *testing.T, st *session.Store, id string) session.SessionSummary { + t.Helper() + for _, s := range st.ListSessions() { + if s.ID == id { + return s + } + } + t.Fatalf("%s is not listed", id) + return session.SessionSummary{} +} + +// THE POINT OF THE CHANGE, end to end: a session recorded through a store and its archive, both +// closed, then resumed under a new store over the reopened archive, lists as the whole session — +// its events, tokens and title from before the restart, plus what it did after. +func TestPrior_ARestartedSessionListsAsTheWholeSession(t *testing.T) { + evs := synthSession(71, 4, 512) + ref := session.New(0, 0, 0) + defer ref.Close() + for _, e := range evs { + ref.Append("s1", e) + } + want := listed(t, ref, "s1") + if want.Title == "" || want.TotalTokens == 0 { + t.Fatalf("fixture exercises nothing: %+v", want) + } + + a, st := restarted(t, t.TempDir(), newClock(), "s1", evs[:len(evs)/2]) + defer a.Close() + defer st.Close() + for _, e := range evs[len(evs)/2:] { + st.Append("s1", e) + } + settle(a) + got := listed(t, st, "s1") + if got.EventCount != want.EventCount || got.TotalTokens != want.TotalTokens || got.Title != want.Title { + t.Fatalf("after a restart the session lists as\n %+v\nnot as one that never restarted\n %+v", got, want) + } +} + +// Prior answers for exactly the entry whose start the writer has marked, and for no other: not +// before the writer takes the entry's first event, not for a different seq, not for an id it does +// not hold. +func TestPrior_AnswersOnlyForTheEntryTheWriterMarked(t *testing.T) { + a, st := restarted(t, t.TempDir(), newClock(), "s1", synthSession(72, 2, 256)) + defer a.Close() + defer st.Close() + after := a.LastSeq("s1") + if _, ok := a.Prior("s1", after); ok { + t.Fatal("Prior answered before the entry existed") + } + a.gate = make(chan struct{}) // hold the writer: the store has the entry, the writer has not taken it + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + if _, ok := a.Prior("s1", after); ok { + t.Fatal("Prior answered before the writer marked the entry's start") + } + close(a.gate) + settle(a) + p, ok := a.Prior("s1", after) + if !ok || p.Fold.Summary("s1").EventCount != int(after) || p.CreatedAt.IsZero() { + t.Fatalf("Prior = %+v, %v; want the %d events before the entry, with a creation time", p, ok, after) + } + if _, ok := a.Prior("s1", after+1); ok { + t.Fatal("Prior answered for an entry numbered after a different seq") + } + if _, ok := a.Prior("nobody", 3); ok { + t.Fatal("Prior answered for an id the archive does not hold") + } +} + +// A paused writer still marks the entry's start: begin runs before write's pause check, and +// publishes for itself. +func TestPrior_APausedWriterStillMarksTheEntry(t *testing.T) { + a, st := restarted(t, t.TempDir(), newClock(), "s1", synthSession(73, 2, 256)) + defer a.Close() + defer st.Close() + after := a.LastSeq("s1") + a.paused.Store(true) + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + settle(a) + if _, ok := a.Prior("s1", after); !ok { + t.Fatal("a paused writer left the entry's start unmarked") + } +} + +// After a clear the session starts over, and so does its row: nothing from before comes back. +func TestPrior_AClearedSessionListsOnlyWhatCameAfter(t *testing.T) { + a, st := restarted(t, t.TempDir(), newClock(), "s1", synthSession(74, 2, 256)) + defer a.Close() + defer st.Close() + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + settle(a) + st.Clear() + settle(a) + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + settle(a) + if got := listed(t, st, "s1").EventCount; got != 1 { + t.Fatalf("EventCount = %d after a clear, want 1", got) + } +} + +// A fold Prior hands out never changes afterwards. renameEntry turns the old id's before back into +// its live fold, which later events Add to; it must clone it first, or a reader still holding the +// published copy would watch it move under the store's read lock. +func TestPrior_AFoldHandedOutNeverChanges(t *testing.T) { + a, st := restarted(t, t.TempDir(), newClock(), "X", synthSession(75, 1, 256)) + defer a.Close() + defer st.Close() + after := a.LastSeq("X") + st.Append("X", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + settle(a) + p, ok := a.Prior("X", after) + if !ok { + t.Fatal("no prior for the resumed entry") + } + held := p.Fold.Summary("X").EventCount + st.Rekey("X", "Y") + settle(a) + st.Append("X", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + settle(a) + if got := p.Fold.Summary("X").EventCount; got != held { + t.Fatalf("a handed-out fold moved from %d to %d events", held, got) + } +} diff --git a/core/session/archive/read.go b/core/session/archive/read.go index d388f3346..77c046f62 100644 --- a/core/session/archive/read.go +++ b/core/session/archive/read.go @@ -130,6 +130,27 @@ func (a *Archive) Summaries() []session.SessionSummary { return out } +// Prior implements session.PriorKeeper: the fold of what id held before the store entry +// numbering after `after` began. ok only when the writer has marked that entry's start, because +// exactly then is before the history up to `after` and nothing of the entry's: begin runs in queue +// order, after every earlier event and before any of the entry's. Until the writer takes the +// entry's first event, and wherever it could mark none (begin found no state for the id: after a +// clear, a prune, a whole-directory rename), the answer is false, and the store shows the entry's +// own figures — an undercount, never a double count. +// +// A memory lookup under idxMu, safe under the store's read lock because idxMu is a LEAF lock: it +// may be taken while the store's read lock is held, as here, but nothing is ever acquired while +// idxMu is held, so no lock cycle can pass through it. +func (a *Archive) Prior(sessionID string, after uint64) (session.Prior, bool) { + a.idxMu.RLock() + e := a.index[sessionID] + a.idxMu.RUnlock() + if e == nil || e.before == nil || e.startAfter != after { + return session.Prior{}, false + } + return session.Prior{Fold: e.before, CreatedAt: e.summary.CreatedAt}, true +} + // lastN keeps the newest n events pushed into it. type lastN struct { buf []*pipeline.SessionEvent diff --git a/core/session/continuity_test.go b/core/session/continuity_test.go new file mode 100644 index 000000000..0a41fbb22 --- /dev/null +++ b/core/session/continuity_test.go @@ -0,0 +1,208 @@ +package session + +import ( + "fmt" + "slices" + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/event" + "github.com/rossoctl/cortex/core/pipeline" +) + +// priorKeeper is a session archive reduced to what ListSessions asks of one: the last seq it holds +// under each id, and the fold of that history. refuse makes it a keeper that cannot say. +type priorKeeper struct { + last map[string]uint64 + prior map[string]Prior + refuse bool +} + +func (k *priorKeeper) Record(string, *pipeline.SessionEvent) {} +func (k *priorKeeper) LastSeq(id string) uint64 { return k.last[id] } +func (k *priorKeeper) Prior(id string, after uint64) (Prior, bool) { + if k.refuse || after != k.last[id] { + return Prior{}, false + } + p, ok := k.prior[id] + return p, ok +} + +// continuityFixture is foldFixture with a /rename and then a later prose turn, so a split can fall +// on either side of every change in the title's rank. +func continuityFixture(t *testing.T) []pipeline.SessionEvent { + t.Helper() + evs := foldFixture(t) + rename := titleEvent("/renameflaky store test") + rename.Phase = pipeline.SessionRequest + later := titleEvent("one more question") + later.Phase = pipeline.SessionRequest + return append(evs, rename, later) +} + +// THE REASON FOR PriorKeeper: a store that lost a session — to a restart, a ttl, max_sessions — and +// is handed its earlier history lists the session exactly as a store that never lost it. Split at +// every event, so every figure's merge is tried on both sides of every change in it. +func TestListSessions_ContinuesAnEarlierHistory(t *testing.T) { + evs := continuityFixture(t) + whole := New(0, 0, 0) + defer whole.Close() + for _, e := range evs { + whole.Append("s1", e) + } + want := summaryOf(t, whole, "s1") + if want.Title != "flaky store test" || want.PromptContext == nil || len(want.Currencies) != 2 || + want.CostMicros == 0 || want.AvoidedMicros == 0 || want.Agent == "" { + t.Fatalf("fixture no longer exercises every figure: %+v", want) + } + created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC) + for split := 1; split < len(evs); split++ { + earlier := NewSummaryFold() + for i := range evs[:split] { + earlier.Add("s1", &evs[i]) + } + k := &priorKeeper{last: map[string]uint64{"s1": uint64(split)}, + prior: map[string]Prior{"s1": {Fold: earlier, CreatedAt: created}}} + st := New(0, 0, 0) + st.AddRecorder(k) + for _, e := range evs[split:] { + st.Append("s1", e) + } + got := summaryOf(t, st, "s1") + st.Close() + if got.EventCount != want.EventCount || got.Title != want.Title || got.Agent != want.Agent || + got.TotalTokens != want.TotalTokens || got.CostMicros != want.CostMicros || + got.AvoidedMicros != want.AvoidedMicros || got.Saturated != want.Saturated || + !slices.Equal(got.Currencies, want.Currencies) || !sameContext(got.PromptContext, want.PromptContext) { + t.Errorf("split %d:\n got %+v\n want %+v", split, got, want) + } + if !got.CreatedAt.Equal(created) { + t.Errorf("split %d: CreatedAt = %v, want the earlier history's %v", split, got.CreatedAt, created) + } + } +} + +// A keeper that cannot say exactly what came before leaves the row as the entry's own figures: an +// undercount, never the entry counted twice. +func TestListSessions_AnUnknownEarlierHistoryLeavesTheEntrysOwnFigures(t *testing.T) { + st := New(0, 0, 0) + defer st.Close() + st.AddRecorder(&priorKeeper{last: map[string]uint64{"s1": 5}, refuse: true}) + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + if got := summaryOf(t, st, "s1"); got.EventCount != 1 { + t.Fatalf("EventCount = %d, want the entry's own 1", got.EventCount) + } +} + +// The claim owner's label wins as it always has, even when only the earlier history carries it: +// the owner's first event came before the restart, and nothing since named a client. +func TestListSessions_TheOwnersLabelMayComeFromTheEarlierHistory(t *testing.T) { + claude := &pipeline.EventClient{Name: "claude-code", Version: "2.1.284"} + earlier := NewSummaryFold() + earlier.Add("s1", &pipeline.SessionEvent{Phase: pipeline.SessionRequest, Client: claude}) + st := New(0, 0, 0) + defer st.Close() + st.AddRecorder(&priorKeeper{last: map[string]uint64{"s1": 1}, prior: map[string]Prior{"s1": {Fold: earlier}}}) + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionRequest}) + st.Claim("s1", "claude-code") + if got, want := summaryOf(t, st, "s1").Agent, earlier.Summary("s1").Agent; got != want || want == "" { + t.Fatalf("Agent = %q, want the owner's label from the earlier history, %q", got, want) + } +} + +// A claim owner's label is the first label that agent sent across the session's whole history, so +// where the earlier history and the entry carry the owner under two different labels, the earlier +// one wins — as it does in a store that never lost the session. Two versions alone cannot show it: +// AgentName folds a name/version label to the bare name, so the entry's label here is a WebFetch's +// raw User-Agent, which names claude-code only inside a comment. +func TestListSessions_TheOwnersEarlierLabelWinsOverTheEntrys(t *testing.T) { + first := pipeline.SessionEvent{Phase: pipeline.SessionRequest, + Client: &pipeline.EventClient{Name: "claude-code", Version: "2.1.284"}} + later := pipeline.SessionEvent{Phase: pipeline.SessionRequest, + Client: &pipeline.EventClient{Raw: "Claude-User (claude-code/2.1.290; +https://support.anthropic.com/)"}} + + whole := New(0, 0, 0) + defer whole.Close() + whole.Append("s1", first) + whole.Append("s1", later) + whole.Claim("s1", "claude-code") + want := summaryOf(t, whole, "s1").Agent + + earlier := NewSummaryFold() + earlier.Add("s1", &first) + entryOnly := NewSummaryFold() + entryOnly.Add("s1", &later) + if want == "" || want != earlier.Summary("s1").Agent || want == entryOnly.Summary("s1").Agent { + t.Fatalf("fixture: the owner's two labels do not differ, or the whole store does not pick the earlier "+ + "(whole %q, earlier %q, entry %q)", want, earlier.Summary("s1").Agent, entryOnly.Summary("s1").Agent) + } + + st := New(0, 0, 0) + defer st.Close() + st.AddRecorder(&priorKeeper{last: map[string]uint64{"s1": 1}, prior: map[string]Prior{"s1": {Fold: earlier}}}) + st.Append("s1", later) + st.Claim("s1", "claude-code") + if got := summaryOf(t, st, "s1").Agent; got != want { + t.Fatalf("Agent = %q, want the owner's earlier label %q, as a store that never lost the session reports", + got, want) + } +} + +// A total that clamped on either side is a bound, so the merged row says so: one saturated figure +// before the restart marks the row after it. +func TestListSessions_ASaturatedEarlierHistoryMarksTheRow(t *testing.T) { + earlier := NewSummaryFold() + // Enough near-maximal costs that their sum overflows int64 micros: saturation is the ADDITION + // overflowing. pricing.MicrosFromUSD refuses any one figure at or past MaxCostMicros (2^53) + // rather than clamping it, so it takes over a thousand of them, as in + // TestSumCost_SaturatesRatherThanWrapping. + const n = 2048 + huge := pipeline.SessionEvent{Phase: pipeline.SessionResponse, + Plugins: costRecord(t, event.Event{CostUSD: 9e9, Settled: true, Provenance: "configured"})} + for range n { + earlier.Add("s1", &huge) + } + if !earlier.Summary("s1").Saturated { + t.Fatal("fixture: the earlier fold did not saturate") + } + st := New(0, 0, 0) + defer st.Close() + st.AddRecorder(&priorKeeper{last: map[string]uint64{"s1": n}, prior: map[string]Prior{"s1": {Fold: earlier}}}) + st.Append("s1", pipeline.SessionEvent{Phase: pipeline.SessionResponse, + Plugins: costRecord(t, event.Event{CostUSD: 0.01, Settled: true, Provenance: "configured"})}) + if got := summaryOf(t, st, "s1"); !got.Saturated { + t.Fatalf("Saturated = false after a saturated earlier history: %+v", got) + } +} + +// BenchmarkListSessions_WithPrior is agentop's two-second poll over a full store of resumed sessions: +// the default 100, each with an earlier history to merge. Against the same store with none, it +// prices the lookup and merge. +func BenchmarkListSessions_WithPrior(b *testing.B) { + t := &testing.T{} + for _, withPrior := range []bool{false, true} { + b.Run(fmt.Sprintf("prior=%v", withPrior), func(b *testing.B) { + k := &priorKeeper{last: map[string]uint64{}, prior: map[string]Prior{}} + st := New(0, 0, 0) + defer st.Close() + if withPrior { + st.AddRecorder(k) + } + for i := range 100 { + id := fmt.Sprintf("s%03d", i) + earlier := NewSummaryFold() + for _, e := range foldFixture(t) { + earlier.Add(id, &e) + } + k.last[id], k.prior[id] = uint64(earlier.events), Prior{Fold: earlier} + for _, e := range foldFixture(t) { + st.Append(id, e) + } + } + b.ReportAllocs() + for b.Loop() { + _ = st.ListSessions() + } + }) + } +} diff --git a/core/session/fold.go b/core/session/fold.go index 7095f0a98..0aaea48bc 100644 --- a/core/session/fold.go +++ b/core/session/fold.go @@ -76,12 +76,14 @@ func agentLabelIn(agents []sessionAgent, name string) string { // the session archive's running summary for a session the store may no longer hold. // // It uses the store's own rules — moneyOf through prepareAppend, sumTokens' response-only -// token rule, applyTitle, applyAgent — so a fold over an untrimmed session's events reports -// what ListSessions does. TestSummaryFold_AgreesWithListSessions holds that. +// token rule, applyTitle, applyAgent, and entry.context's CTX fold — so a fold over an +// untrimmed session's events reports what ListSessions does. TestSummaryFold_AgreesWithListSessions +// holds that. // // ONE DIFFERENCE IS INHERENT: Agent is the first known agent's label, while the store prefers // the label of the agent that CLAIMED the session, which only the store knows. A resident -// session's row therefore always comes from the store. +// session's row therefore always comes from the store, which adds the archive's fold of the +// history before its entry (PriorKeeper). // // The zero value is not ready — rankRename is 0, so a zero titleRank would claim the session // had been renamed. Use NewSummaryFold. Not safe for concurrent use. @@ -94,6 +96,13 @@ type SummaryFold struct { titleRank int title string agents []sessionAgent + // context is the CTX figure folded so far, by the store's own rule (entry.context), and + // restoredContext the figure a fold resumed from session.json started from. TWO FIELDS BECAUSE + // pipeline.PromptContextFold HAS NO RESTORE PATH: what persists is its published figure, and + // since the published order is the fold's order, merging the restored figure with what was + // folded since gives what one uninterrupted fold would have. + context pipeline.PromptContextFold + restoredContext *pipeline.PromptContext } // NewSummaryFold returns an empty fold. @@ -126,11 +135,17 @@ func (f *SummaryFold) Add(sessionID string, e *pipeline.SessionEvent) { } f.titleRank, f.title = applyTitle(f.titleRank, f.title, p.titleRank, p.titleText) f.agents = applyAgent(f.agents, sessionID, p.agentName, e.Client) + f.context.Add(e) } -// Summary sets EventCount, Title, Agent, TotalTokens, CostMicros, AvoidedMicros, Currencies -// and Saturated on a SessionSummary for sessionID, and nothing else: the timestamps and the -// live-only fields (Active, Adopted, PromptContext) are the caller's. +// promptContext is the fold's CTX figure, nil when nothing can be said. +func (f *SummaryFold) promptContext() *pipeline.PromptContext { + return pipeline.MergePromptContext(f.restoredContext, f.context.Publish()) +} + +// Summary sets EventCount, Title, Agent, TotalTokens, CostMicros, AvoidedMicros, Currencies, +// Saturated and PromptContext on a SessionSummary for sessionID, and nothing else: the +// timestamps and the live-only fields (Active, Adopted) are the caller's. func (f *SummaryFold) Summary(sessionID string) SessionSummary { sum := SessionSummary{ ID: sessionID, @@ -141,6 +156,7 @@ func (f *SummaryFold) Summary(sessionID string) SessionSummary { AvoidedMicros: f.avoided.Micros, Currencies: currenciesOf(f.units), Saturated: f.cost.Saturated || f.avoided.Saturated, + PromptContext: f.promptContext(), } if sessionID != DefaultSessionID && !strings.HasPrefix(sessionID, PendingPrefix) && len(f.agents) > 0 { sum.Agent = f.agents[0].label @@ -154,16 +170,17 @@ func (f *SummaryFold) Summary(sessionID string) SessionSummary { // TitleRank IS A POINTER because rankRename is 0: a document without it must decode to // rankNone, not to a rename that would lock the title against every later candidate. type foldJSON struct { - EventCount int `json:"eventCount"` - Title string `json:"title,omitempty"` - TitleRank *int `json:"titleRank,omitempty"` - Agents []foldAgent `json:"agents,omitempty"` - TotalTokens int `json:"totalTokens,omitempty"` - CostMicros int64 `json:"costMicros,omitempty"` - CostSaturated bool `json:"costSaturated,omitempty"` - AvoidedMicros int64 `json:"avoidedMicros,omitempty"` - AvoidedSaturated bool `json:"avoidedSaturated,omitempty"` - Units map[string]int `json:"units,omitempty"` + EventCount int `json:"eventCount"` + Title string `json:"title,omitempty"` + TitleRank *int `json:"titleRank,omitempty"` + Agents []foldAgent `json:"agents,omitempty"` + TotalTokens int `json:"totalTokens,omitempty"` + CostMicros int64 `json:"costMicros,omitempty"` + CostSaturated bool `json:"costSaturated,omitempty"` + AvoidedMicros int64 `json:"avoidedMicros,omitempty"` + AvoidedSaturated bool `json:"avoidedSaturated,omitempty"` + Units map[string]int `json:"units,omitempty"` + PromptContext *pipeline.PromptContext `json:"promptContext,omitempty"` } type foldAgent struct { @@ -184,6 +201,7 @@ func (f *SummaryFold) MarshalJSON() ([]byte, error) { AvoidedMicros: f.avoided.Micros, AvoidedSaturated: f.avoided.Saturated, Units: f.units, + PromptContext: f.promptContext(), } for _, a := range f.agents { j.Agents = append(j.Agents, foldAgent{Name: a.name, Label: a.label}) @@ -198,13 +216,14 @@ func (f *SummaryFold) UnmarshalJSON(b []byte) error { return err } *f = SummaryFold{ - events: j.EventCount, - tokens: j.TotalTokens, - cost: usage.CostSum{Micros: j.CostMicros, Saturated: j.CostSaturated}, - avoided: usage.CostSum{Micros: j.AvoidedMicros, Saturated: j.AvoidedSaturated}, - units: j.Units, - titleRank: rankNone, - title: j.Title, + events: j.EventCount, + tokens: j.TotalTokens, + cost: usage.CostSum{Micros: j.CostMicros, Saturated: j.CostSaturated}, + avoided: usage.CostSum{Micros: j.AvoidedMicros, Saturated: j.AvoidedSaturated}, + units: j.Units, + titleRank: rankNone, + title: j.Title, + restoredContext: j.PromptContext, } if j.TitleRank != nil { f.titleRank = *j.TitleRank diff --git a/core/session/fold_test.go b/core/session/fold_test.go index cf6f2d6e2..7433b4238 100644 --- a/core/session/fold_test.go +++ b/core/session/fold_test.go @@ -52,6 +52,11 @@ func foldFixture(t *testing.T) []pipeline.SessionEvent { evs = append(evs, turn("and the third thing", 0, event.Event{Source: event.SourceUsageFallback, Avoided: []event.Saving{{Component: "tool-prune", TokensAvoided: 200, USD: 0.02, Tier: "input"}}}, claude)...) + // A main-agent turn with a tool manifest, so the fixture carries a CTX figure. + evs = append(evs, pipeline.SessionEvent{Phase: pipeline.SessionResponse, Client: claude, + At: time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC), + Inference: &pipeline.InferenceExtension{TotalTokens: 300, PromptTokens: 9000, + Tools: []pipeline.InferenceTool{{Name: "read"}}}}) return evs } @@ -74,11 +79,11 @@ func TestSummaryFold_AgreesWithListSessions(t *testing.T) { if got.EventCount != want.EventCount || got.Title != want.Title || got.Agent != want.Agent || got.TotalTokens != want.TotalTokens || got.CostMicros != want.CostMicros || got.AvoidedMicros != want.AvoidedMicros || got.Saturated != want.Saturated || - !slices.Equal(got.Currencies, want.Currencies) { + !slices.Equal(got.Currencies, want.Currencies) || !sameContext(got.PromptContext, want.PromptContext) { t.Fatalf("fold = %+v\nstore = %+v", got, want) } if want.CostMicros == 0 || want.AvoidedMicros == 0 || want.Title == "" || want.Agent == "" || - len(want.Currencies) != 2 { + len(want.Currencies) != 2 || want.PromptContext == nil { t.Fatalf("fixture no longer exercises every figure: %+v", want) } } @@ -140,3 +145,46 @@ func TestSummaryFold_AMissingTitleRankIsNotARename(t *testing.T) { } func ptr[T any](v T) *T { return &v } + +// sameContext compares two CTX figures by value, the time by instant. +func sameContext(a, b *pipeline.PromptContext) bool { + if a == nil || b == nil { + return a == b + } + return a.Tokens == b.Tokens && a.Stated == b.Stated && a.Msgs == b.Msgs && a.At.Equal(b.At) +} + +// CTX survives session.json. PromptContextFold has no restore path, so the fold keeps the figure +// it read and merges it with what it folds afterwards — which is the max one uninterrupted fold +// takes, since the published order is the fold's order. Split at every event. +func TestSummaryFold_PromptContextResumesFromJSON(t *testing.T) { + evs := foldFixture(t) + whole := NewSummaryFold() + for i := range evs { + whole.Add("s1", &evs[i]) + } + want := whole.Summary("s1").PromptContext + if want == nil { + t.Fatal("fixture carries no CTX figure") + } + for split := range len(evs) + 1 { + f := NewSummaryFold() + for i := range evs[:split] { + f.Add("s1", &evs[i]) + } + b, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + g := NewSummaryFold() + if err := json.Unmarshal(b, g); err != nil { + t.Fatal(err) + } + for i := split; i < len(evs); i++ { + g.Add("s1", &evs[i]) + } + if got := g.Summary("s1").PromptContext; !sameContext(got, want) { + t.Errorf("split %d: PromptContext = %+v, want %+v", split, got, want) + } + } +} diff --git a/core/session/history.go b/core/session/history.go index 7412b6dbe..2c00b0869 100644 --- a/core/session/history.go +++ b/core/session/history.go @@ -1,6 +1,10 @@ package session -import "time" +import ( + "time" + + "github.com/rossoctl/cortex/core/pipeline" +) // SeqSeeder is optionally implemented by a Recorder that keeps a session's events for longer // than the store does — the session archive. When the store creates an entry it asks every @@ -81,3 +85,89 @@ func (s *Store) Clear() int { } return n } + +// Prior is a session's history from before a store entry began: what the session archive folded +// under the id up to the seq the entry numbers after. +type Prior struct { + // Fold is the earlier history's figures. SHARED AND IMMUTABLE: the keeper hands out the copy it + // published, so a caller reads it and never Adds to it. + Fold *SummaryFold + // CreatedAt is when the keeper first saw the session; zero when it does not know. + CreatedAt time.Time +} + +// PriorKeeper is optionally implemented by a Recorder that keeps a session's history across the +// store's restarts and evictions — the session archive. For every entry numbered after earlier +// history (a SeqSeeder answered above zero when the entry was created), ListSessions asks for that +// history and reports the session's row as the two together, so that — with session.max_events +// unset, the default — a restart, a ttl or max_sessions changes nothing a list shows. With a cap +// the entry's own figures shed what a trim drops while the archive's fold keeps it, so a restart +// can raise them. +// +// ok is false when the keeper cannot say EXACTLY what came before, and the row then shows the +// entry's own figures. That undercount is the safe side: the keeper records every event the store +// appends, so anything but the history from before the entry would count the entry twice. +// +// Prior is called under the store's READ lock, on agentop's two-second poll: it must be a memory +// lookup, must not block, and must not call back into the store. +type PriorKeeper interface { + Prior(sessionID string, after uint64) (Prior, bool) +} + +// priorLocked is the first PriorKeeper's answer for id's entry numbered after `after`. s.mu held. +func (s *Store) priorLocked(id string, after uint64) (Prior, bool) { + for _, r := range s.recorders { + if pk, ok := r.(PriorKeeper); ok { + if p, ok := pk.Prior(id, after); ok && p.Fold != nil { + return p, true + } + } + } + return Prior{}, false +} + +// withPriorLocked adds to sum — sess's row as ListSessions built it — the history id held before +// sess began. Each figure merges by the rule that folds it, so the row is what a store that never +// lost the session would report. s.mu held. +func (s *Store) withPriorLocked(sum *SessionSummary, id string, sess *entry) { + if sess.after == 0 { + return + } + p, ok := s.priorLocked(id, sess.after) + if !ok { + return + } + f := p.Fold + sum.EventCount += f.events + sum.TotalTokens += f.tokens + // CostSum's addition for the sums, OR for the flags: a total that clamped on either side is a + // bound, whichever side it was. + cost, avoided := sess.cost, sess.avoided + cost.Add(f.cost.Micros) + avoided.Add(f.avoided.Micros) + cost.Saturated = cost.Saturated || f.cost.Saturated + avoided.Saturated = avoided.Saturated || f.avoided.Saturated + sum.CostMicros, sum.AvoidedMicros = cost.Micros, avoided.Micros + sum.Saturated = cost.Saturated || avoided.Saturated + // THE UNIT COUNTS, NOT THE RENDERED LISTS: currenciesOf renders dollars-only as nil, so a union + // of two rendered lists would drop "USD" from a session priced in dollars on one side only. + if len(f.units) > 0 { + units := make(map[string]int, len(f.units)+len(sess.units)) + for u, n := range sess.units { + units[u] += n + } + for u, n := range f.units { + units[u] += n + } + sum.Currencies = currenciesOf(units) + } + // The earlier history's title as the one held and the entry's as the candidate. applyTitle + // then gives what folding both in order would, because the entry's title is itself the first + // at its best rank, or its latest /rename. + _, sum.Title = applyTitle(f.titleRank, f.title, sess.titleRank, sess.Title) + sum.Agent = s.agentOfLocked(id, sess, f.agents) + if !p.CreatedAt.IsZero() && p.CreatedAt.Before(sum.CreatedAt) { + sum.CreatedAt = p.CreatedAt + } + sum.PromptContext = pipeline.MergePromptContext(f.promptContext(), sum.PromptContext) +} diff --git a/core/session/store.go b/core/session/store.go index c9c4e79ff..764181608 100644 --- a/core/session/store.go +++ b/core/session/store.go @@ -1,6 +1,8 @@ // Package session provides an in-memory session store for correlating // inbound user intents with outbound tool calls across request boundaries. -// The store is per-pod (AuthBridge sidecar) and does not persist across restarts. +// The store is per-pod (AuthBridge sidecar) and does not persist across restarts; +// where a session archive runs, ListSessions continues each session from it (see +// PriorKeeper). package session import ( @@ -51,6 +53,10 @@ type entry struct { // rather than Seq for exactly this reason — see agentop's applyOlderPage. See also // pipeline.SessionEvent.Seq. nextSeq uint64 + // after is the seq this entry numbers its events after: the highest a SeqSeeder held for the id + // when the entry was created, 0 when none held any. Unlike nextSeq it never moves. Above 0 the + // id had history before this entry, and ListSessions asks a PriorKeeper for it. + after uint64 // cost and avoided are RUNNING TOTALS over Events, maintained by Append and read by // ListSessions. Micros, in the units usage.Counts uses. @@ -73,7 +79,8 @@ type entry struct { // subtracted, so these stay equal to sumCost(Events) — the invariant // TestAppend_RunningTotalsMatchAFullRecomputation exists to hold. That is what keeps // the COST column scoped exactly like the TOKENS column beside it, which is the claim - // SessionSummary.CostMicros makes. + // SessionSummary.CostMicros makes. ListSessions adds the history before this entry on top + // where a PriorKeeper has it (withPriorLocked); these running totals stay the entry's own. cost usage.CostSum avoided usage.CostSum // units counts the priced events behind cost by the unit they were priced in, "" for USD. @@ -472,6 +479,7 @@ func (s *Store) appendLocked(sessionID string, b *Bucket, event pipeline.Session CreatedAt: now, // After whatever the archive already numbered under this id; see SeqSeeder. nextSeq: seed, + after: seed, // NOT THE ZERO VALUE: rankRename is 0, so a zero-valued titleRank would claim this // session had already been renamed and no candidate could ever beat it — the first // prose message would be unnameable. rankNone is the "nothing has named it" rank. @@ -867,7 +875,8 @@ type SessionSummary struct { Adopted []string `json:"adopted,omitempty"` TotalTokens int `json:"totalTokens,omitempty"` // sum of Inference.TotalTokens across response events // CostMicros is what this session's events cost, in millionths of a dollar, summed from - // the records the session itself holds. + // the records the session itself holds — where a session archive runs, including those from + // before this entry. // // FROM THE EVENTS, not from the ledger, and that is the only honest source available: the // ledger's row key is (endpoint, model, agent, provenance) with no session dimension by @@ -875,10 +884,11 @@ type SessionSummary struct { // only over its rolling window, which would put a partial figure beside the LIFETIME // TotalTokens above and understate the row by whatever fell off the ring. // - // SCOPED TO WHAT THE STORE STILL HOLDS, and it therefore RESETS ON RESTART while a ledger - // window does not. Both are correct: this is "the cost of the events in this session", the - // ledger is "the cost of the day". A client showing them together must not present one as - // a check on the other. + // SCOPED TO WHAT THE STORE STILL HOLDS, PLUS — WHERE A SESSION ARCHIVE RUNS — THE SESSION'S + // HISTORY FROM BEFORE THIS ENTRY (see PriorKeeper). So with an archive it continues across a + // restart, and without one it resets; a ledger window always continues. Both are correct: + // this is "the cost of the events in this session", the ledger is "the cost of the day". A + // client showing them together must not present one as a check on the other. // // Omitted when zero rather than sent as 0, on this codebase's standing rule that an // unknown cost must never render as $0.00: a session whose traffic nothing could price is @@ -938,7 +948,8 @@ type SessionSummary struct { // // RETENTION-INDEPENDENT ALL THE SAME — unlike TotalTokens and CostMicros above, and the // asymmetry is deliberate rather than an inconsistency. Those are sums over the events the - // store still holds, so a trim sheds exactly what left the slice, which is why CostMicros can + // store still holds (plus, where a session archive runs, the session's history from before + // this entry), so a trim sheds exactly what left the slice, which is why CostMicros can // honestly call itself the cost of the events in this session. This is an extremum under a // total order: nothing about it is accumulated, so a trim has nothing to subtract from it and // it outlives the events it was read from. Recomputing it over the survivors would not @@ -951,9 +962,10 @@ type SessionSummary struct { // one-shot completions has no conversation to measure and is indistinguishable here from one // nobody observed; both must reach a client as an absent field so it can draw an em dash. // - // RESETS ON PROXY RESTART, like CostMicros and unlike a cost-ledger window, because the store - // is in-memory per-pod. A client that has been watching longer than this proxy has been up may - // hold a larger figure legitimately — see pipeline.MergePromptContext, which is how the two combine. + // CONTINUES ACROSS A RESTART WHERE A SESSION ARCHIVE RUNS, like CostMicros, and resets where + // none does, because the store is in-memory per-pod. A client that has been watching longer + // than this proxy has been up may hold a larger figure legitimately — see + // pipeline.MergePromptContext, which is how the two combine. PromptContext *pipeline.PromptContext `json:"promptContext,omitempty"` // Resident is false on a row the session archive served because the store no longer holds @@ -971,15 +983,22 @@ func (e *entry) agentLabel(name string) string { return agentLabelIn(e.agents, name) } -// agentOfLocked is SessionSummary.Agent: the label of the owner that claimed the session, else -// of the first known agent in it. -func (s *Store) agentOfLocked(id string, sess *entry) string { +// agentOfLocked is SessionSummary.Agent: the label of the owner that claimed the session, else of +// the first known agent in it. earlier is the agents of the history before this entry (see +// PriorKeeper), nil when there is none; being earlier, it is searched first. +func (s *Store) agentOfLocked(id string, sess *entry, earlier []sessionAgent) string { if id == DefaultSessionID || strings.HasPrefix(id, PendingPrefix) { return "" } if owner := s.owners[id]; owner != "" { + if l := agentLabelIn(earlier, owner); l != "" { + return l + } return sess.agentLabel(owner) } + if len(earlier) > 0 { + return earlier[0].label + } if len(sess.agents) == 0 { return "" } @@ -1014,7 +1033,7 @@ func (s *Store) ListSessions() []SessionSummary { if s.isExpired(sess, now) { continue } - out = append(out, SessionSummary{ + sum := SessionSummary{ ID: id, CreatedAt: sess.CreatedAt, UpdatedAt: sess.UpdatedAt, @@ -1030,7 +1049,7 @@ func (s *Store) ListSessions() []SessionSummary { // reappearing here — they would diverge immediately if one did. This obeys the rule // PromptContext states below, which rejects an O(events) walk under the read lock. Title: sess.Title, - Agent: s.agentOfLocked(id, sess), + Agent: s.agentOfLocked(id, sess, nil), Adopted: adoptedBy[id], // Still a walk, and deliberately left as one: it is a pointer deref per event // with no allocation, where the money figures below needed a JSON unmarshal. @@ -1049,7 +1068,9 @@ func (s *Store) ListSessions() []SessionSummary { // under the read lock, on agentop's two-second poll, in front of a lock whose writer side // is Append on the proxy's request path, with maxEvents unset by default. PromptContext: sess.context.Publish(), - }) + } + s.withPriorLocked(&sum, id, sess) + out = append(out, sum) } // Most recently updated first. sort.Slice(out, func(i, j int) bool { diff --git a/core/sessionapi/archive.go b/core/sessionapi/archive.go index 9b37e0baa..f2687fcc9 100644 --- a/core/sessionapi/archive.go +++ b/core/sessionapi/archive.go @@ -100,7 +100,8 @@ func mergeBySeq(disk, mem []pipeline.SessionEvent, limit int) []pipeline.Session // withArchivedRows adds to the store's list every archived session it does not hold, marked // resident: false, and reports the archive's usage. A session in both is listed once, as its -// resident row: the store knows what only it can — whether it is active, which agent claimed it. +// resident row: the store knows what only it can — whether it is active, which agent claimed it — +// and that row already carries the archive's earlier history (session.PriorKeeper). func (s *Server) withArchivedRows(resident []session.SessionSummary) ([]session.SessionSummary, *pipeline.ArchiveUsage) { held := make(map[string]bool, len(resident)) for _, r := range resident { diff --git a/core/sessionapi/archive_test.go b/core/sessionapi/archive_test.go index 2a727fd5a..2c6a7cc0e 100644 --- a/core/sessionapi/archive_test.go +++ b/core/sessionapi/archive_test.go @@ -257,3 +257,29 @@ func TestHandleGet_FillsTheGapBehindAPinnedIntent(t *testing.T) { } } } + +// A session resumed after a restart lists its whole history — 10 events from before and 1 after — +// in the default list and with ?archived=true alike. The archive marks the entry's start on its +// own goroutine, so poll briefly for it. +func TestHandleList_AResumedSessionListsItsWholeHistory(t *testing.T) { + ts, store, _ := restarted(t) + store.Append("s1", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionRequest}) + for _, url := range []string{ts.URL + "/v1/sessions", ts.URL + "/v1/sessions?archived=true"} { + deadline := time.Now().Add(2 * time.Second) + for { + var n int + for _, s := range getList(t, url).Sessions { + if s.ID == "s1" { + n = s.EventCount + } + } + if n == 11 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%s: s1 lists %d events, want 11", url, n) + } + time.Sleep(5 * time.Millisecond) + } + } +} diff --git a/docs/laptop-service.md b/docs/laptop-service.md index 49b4ee4f2..6fac575d9 100644 --- a/docs/laptop-service.md +++ b/docs/laptop-service.md @@ -301,7 +301,9 @@ Two other knobs, same restart rule: **A local install keeps every session on disk, on by default.** Each event the session store records is also written to `~/.cortex/sessions`, so a session outlives a restart and the store's own eviction (`session.max_sessions`, a `session.ttl`). `agentop` lists every session the archive -holds beside the ones in memory, and Enter opens one like any other. +holds beside the ones in memory, and Enter opens one like any other. Each session keeps its +figures across a restart too — events, tokens, cost, title: the proxy adds what the archive holds +from before the restart to what it has seen since. **Unlike the cost ledger, this is the content.** Prompts, completions, tool arguments and tool results — everything the session API serves — sit in diff --git a/docs/tool-prune-plugin.md b/docs/tool-prune-plugin.md index 9501db6ce..4b0337703 100644 --- a/docs/tool-prune-plugin.md +++ b/docs/tool-prune-plugin.md @@ -191,8 +191,9 @@ aggregate: Independent of `agentop`'s per-run stats pane, which resets when the plugin's counters do. The ledger does not: a window survives a proxy restart, while the per-session total is -scoped to the events the store still holds and resets with it. Both are correct, and they -answer different questions — do not read one as a check on the other. +scoped to the events the store still holds, continues across a restart where a session +archive runs, and resets where none does. Both are correct, and they answer different +questions — do not read one as a check on the other. ### Costing it From 8aa0bfa5bf532e49905fbcbb96dad3a85dd2a23b Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 6 Oct 2026 15:40:13 -0400 Subject: [PATCH 3/5] feat: Replay the archive's last six hours into the usage ring at startup The usage ring behind LAST 1H and the usage pane's duration windows is in memory, so a restart emptied it: LAST 1H read $0.12 minutes after a restart in a $9/hour session. At startup, after the archive opens and before any listener starts, cmd/cortex reads the archive's last six hours (archive.ReplaySince), reduces each event to what the ring reads (usage.ReplayCopy), sorts by time and feeds usage.Aggregator.Replay: Record behind a guard that drops any event whose minute would reset a visible bucket. Never through the store, so the cost ledger and the archive are not written twice. Within a 5s budget, or not at all: a partial ring would show a LAST 1H that is too low as if it were real. A replay holds about 1KB per kept event until it is fed; a six-hour synthetic archive of 2,880 events replays in ~180ms. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- CLAUDE.md | 2 +- cmd/cortex/archive.go | 75 +++++++++++ cmd/cortex/archive_replay_test.go | 147 ++++++++++++++++++++++ cmd/cortex/main.go | 4 + core/cost/usage/replay.go | 67 ++++++++++ core/cost/usage/replay_test.go | 71 +++++++++++ core/session/archive/replay.go | 64 ++++++++++ core/session/archive/replay_bench_test.go | 51 ++++++++ core/session/archive/replay_test.go | 66 ++++++++++ docs/laptop-service.md | 5 +- 10 files changed, 550 insertions(+), 2 deletions(-) create mode 100644 cmd/cortex/archive_replay_test.go create mode 100644 core/cost/usage/replay.go create mode 100644 core/cost/usage/replay_test.go create mode 100644 core/session/archive/replay.go create mode 100644 core/session/archive/replay_bench_test.go create mode 100644 core/session/archive/replay_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 495945eb2..ec54be578 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1095,7 +1095,7 @@ resulting `/shared/client-id.txt` and `/shared/client-secret.txt`. **And the client is the other end of the same problem.** Measured on a laptop, `agentop` sat at 1.64GB against the proxy's 1.03GB — larger than the process it was watching, and the only one of the two still climbing. It made both mistakes the server had just stopped making: it decoded each snapshot as one document (`json.Decoder` only bounds its buffer between *values*, and a whole snapshot is one value), and it kept every string the server's encoder had expanded back out of the store's sharing. Both are fixed in `cmd/agentop/apiclient/snapshot.go`: the response is decoded one event at a time and fed through the same `session.Interner` the store uses, exported for the purpose rather than reimplemented. On a 23.5MB document, `+32.0MB HeapSys` and 26.49MB retained became +0.0MB and 2.31MB (`BenchmarkDecodeSessionViewHeap`). When a memory figure here looks wrong, measure **both** processes — the proxy has not been the larger one for a while. - **The session archive is disk, not memory** (`core/session/archive`, `session.archive`, on by default for a local install bound to loopback only and off everywhere else — `config.ArchiveRunsOnLocalInstall` says why; `DELETE /v1/sessions` erases it with the store). It relieves the store of nothing: every bound above applies unchanged with it on. What it changes is what eviction *costs* — an evicted session is still listed under `?archived=true` and paged from disk, a resumed one lists its whole history, and a `max_events` trim drops nothing a reader can no longer reach — so with it on, a cap or a `ttl` stops being a trade against history. It records through the same `Recorder` hook as the cost pipeline, off the request path: a burst the writer cannot keep up with is dropped from the archive and counted, never from memory. On disk it interns the same five fields the store does, one zstd segment at a time, and it is bounded by `retention_days` (30) and `max_bytes` (2 GiB), not by anything here. `docs/laptop-service.md` has the user-facing version, including what the files contain. + **The session archive is disk, not memory** (`core/session/archive`, `session.archive`, on by default for a local install bound to loopback only and off everywhere else — `config.ArchiveRunsOnLocalInstall` says why; `DELETE /v1/sessions` erases it with the store). It relieves the store of nothing: every bound above applies unchanged with it on. What it changes is what eviction *costs* — an evicted session is still listed under `?archived=true` and paged from disk, a resumed one lists its whole history, and a `max_events` trim drops nothing a reader can no longer reach — so with it on, a cap or a `ttl` stops being a trade against history. It records through the same `Recorder` hook as the cost pipeline, off the request path: a burst the writer cannot keep up with is dropped from the archive and counted, never from memory. On disk it interns the same five fields the store does, one zstd segment at a time, and it is bounded by `retention_days` (30) and `max_bytes` (2 GiB), not by anything here. `docs/laptop-service.md` has the user-facing version, including what the files contain. At startup, before any listener, `cmd/cortex` replays the archive's last six hours into the usage ring — `usage.Aggregator.Replay`, never through the store, so the ledger is not written twice — within `usageReplayBudget` (5s), or not at all. Two layered defenses keep the inbound A2A user intent visible to IBAC even when an agent generates dozens of outbound events per turn: diff --git a/cmd/cortex/archive.go b/cmd/cortex/archive.go index 628f53a0f..e422f1308 100644 --- a/cmd/cortex/archive.go +++ b/cmd/cortex/archive.go @@ -1,10 +1,16 @@ package main import ( + "cmp" + "context" "log/slog" "path/filepath" + "slices" + "time" "github.com/rossoctl/cortex/core/config" + "github.com/rossoctl/cortex/core/cost/usage" + "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/session" "github.com/rossoctl/cortex/core/session/archive" ) @@ -91,3 +97,72 @@ func openSessionArchive(cfg *config.Config, configPath string, sessions *session "default", why) return arch } + +// usageReplayBudget bounds the startup replay of the session archive into the usage ring. The +// replay runs before any listener starts, so this is how long it may delay readiness. Over it the +// replay is abandoned whole; see replayUsage. +const usageReplayBudget = 5 * time.Second + +// replayUsage refills the usage ring — LAST 1H, the usage pane's duration windows — from the +// session archive, so a restart leaves them as they were. It reads the last usage.MaxWindow of +// archived events, reduces each with usage.ReplayCopy, sorts them by time and feeds them to +// agg.Replay. +// +// RUN BEFORE ANY LISTENER STARTS, which is what keeps it simple. Nothing this process records can +// be in the archive yet, so nothing is counted twice. No clear or rekey can race it. And feeding +// in time order makes the ring's eviction and request pairing choose as they did live. +// +// NEVER THROUGH THE STORE: the store would hand every event to the cost ledger and the archive +// again, writing durable rows twice. agg.Replay touches the ring and nothing else, and this takes +// no store or ledger to reach either. +// +// ALL OR NOTHING. A read that does not finish within ctx is abandoned before anything is fed, and +// the ring starts empty as it always did: a partial ring would show a LAST 1H that is too low as if +// it were real. +func replayUsage(ctx context.Context, arch *archive.Archive, agg *usage.Aggregator, now time.Time) (events, sessions, skipped int, err error) { + type replayed struct { + id string + e pipeline.SessionEvent + } + // About 1KB a kept event (see usage.ReplayCopy), held until this returns. Keeping the decoded + // event instead would hold six hours of conversations at once. + var evs []replayed + skipped, err = arch.ReplaySince(ctx, now.Add(-usage.MaxWindow), func(id string, e *pipeline.SessionEvent) { + evs = append(evs, replayed{id, usage.ReplayCopy(e)}) + }) + if err != nil { + return 0, 0, skipped, err + } + // Seq breaks a tie within one session, where it is the order the store appended in; across + // sessions a tie has no order to keep. + slices.SortStableFunc(evs, func(x, y replayed) int { + if c := x.e.At.Compare(y.e.At); c != 0 { + return c + } + return cmp.Compare(x.e.Seq, y.e.Seq) + }) + seen := make(map[string]bool) + for i := range evs { + if agg.Replay(evs[i].id, &evs[i].e) { + events++ + seen[evs[i].id] = true + } + } + return events, len(seen), skipped, nil +} + +// replayUsageAtStartup runs replayUsage within usageReplayBudget and says how it went. +func replayUsageAtStartup(arch *archive.Archive, agg *usage.Aggregator) { + ctx, cancel := context.WithTimeout(context.Background(), usageReplayBudget) + defer cancel() + start := time.Now() + events, sessions, skipped, err := replayUsage(ctx, arch, agg, start) + took := time.Since(start) + if err != nil { + slog.Warn("usage: did not replay the session archive into the usage ring; LAST 1H and the usage pane start empty", + "error", err, "budget", usageReplayBudget, "took", took) + return + } + slog.Info("usage: replayed the session archive into the usage ring", "window", usage.MaxWindow, + "events", events, "sessions", sessions, "unreadableSegments", skipped, "took", took) +} diff --git a/cmd/cortex/archive_replay_test.go b/cmd/cortex/archive_replay_test.go new file mode 100644 index 000000000..102b8b8b5 --- /dev/null +++ b/cmd/cortex/archive_replay_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/event" + "github.com/rossoctl/cortex/core/cost/usage" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" + "github.com/rossoctl/cortex/core/session/archive" +) + +// recordTraffic runs one process's worth of traffic — two sessions interleaved, each turn a +// request carrying a request-phase plugin and a priced response — through a store whose recorders +// are agg and an archive at root, then stops that process. +func recordTraffic(t *testing.T, root string, agg *usage.Aggregator, now time.Time) { + t.Helper() + arch, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + store := session.New(0, 0, 0) + if agg != nil { + store.AddRecorder(agg) + } + store.AddRecorder(arch) + for i, s := range []string{"a", "b", "a", "b"} { + at := now.Add(time.Duration(i-5) * time.Minute) + rid := fmt.Sprintf("r%d", i) + raw, err := json.Marshal(event.Event{CostUSD: 0.01 * float64(i+1), Settled: true, Provenance: "configured"}) + if err != nil { + t.Fatal(err) + } + store.Append(s, pipeline.SessionEvent{At: at, Phase: pipeline.SessionRequest, RequestID: rid, + Invocations: &pipeline.Invocations{Outbound: []pipeline.Invocation{ + {Plugin: "tool-prune", Phase: pipeline.InvocationPhaseRequest}}}}) + store.Append(s, pipeline.SessionEvent{At: at.Add(time.Second), Phase: pipeline.SessionResponse, + RequestID: rid, StatusCode: 200, Duration: time.Second, Host: "api.anthropic.com", + Client: &pipeline.EventClient{Name: "claude-code", Version: "2.1.0"}, + Inference: &pipeline.InferenceExtension{Model: "claude-sonnet-5", TotalTokens: 100 * (i + 1), InputTokens: 10}, + Plugins: map[string]json.RawMessage{event.Key: raw}}) + } + store.Close() + if err := arch.Close(); err != nil { + t.Fatal(err) + } +} + +// The round trip the replay exists for: traffic recorded by one process, which then stops, reads +// back into the next process's ring as the first process's ring had it — in every grouping, for +// every session, with request plugins paired to their responses across interleaved sessions. +// +// Run twice: with the default cap, and with a cap of one session ring, where every new session +// evicts the coldest — so the surviving per-session ring is only right if the replay fed events in +// the order they happened. +func TestReplayUsage_TheNextProcessHasTheRingTheLastOneHad(t *testing.T) { + for _, maxSess := range []int{-1, 1} { // -1 keeps the default + t.Run(fmt.Sprint("maxSessions=", maxSess), func(t *testing.T) { + root := t.TempDir() + now := time.Now().Truncate(time.Minute).Add(30 * time.Second) + clock := func() time.Time { return now } + first := usage.New(usage.WithClock(clock), usage.WithMaxSessions(maxSess)) + recordTraffic(t, root, first, now) + + reopened, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + second := usage.New(usage.WithClock(clock), usage.WithMaxSessions(maxSess)) + events, sessions, skipped, err := replayUsage(context.Background(), reopened, second, now) + if err != nil || events != 8 || sessions != 2 || skipped != 0 { + t.Fatalf("replayUsage = %d events, %d sessions, %d skipped, %v; want 8, 2, 0, nil", events, sessions, skipped, err) + } + for _, g := range []usage.Group{usage.GroupNone, usage.GroupModel, usage.GroupPlugin, usage.GroupSession, + usage.GroupAgent, usage.GroupHost, usage.GroupStatus} { + for _, id := range []string{"", "a", "b"} { + w := first.Snapshot(time.Hour, time.Minute, id, g) + got := second.Snapshot(time.Hour, time.Minute, id, g) + if !reflect.DeepEqual(w, got) { + t.Errorf("group %s session %q: replayed ring differs:\nwant %+v\ngot %+v", g, id, w, got) + } + } + } + }) + } +} + +// errAfter is a context whose Err is nil for its first n calls and context.DeadlineExceeded after, +// so a test can run a deadline out part-way through a read instead of before it starts. ReplaySince +// asks Err and nothing else, once per segment and once per event. +type errAfter struct { + context.Context + n int +} + +func (c *errAfter) Err() error { + if c.n > 0 { + c.n-- + return nil + } + return context.DeadlineExceeded +} + +// ALL OR NOTHING: a replay that runs out of time feeds nothing, and the ring starts empty as it +// always did, rather than holding a partial LAST 1H that reads as real. +// +// THE DEADLINE LANDS MID-READ, after ReplaySince has delivered events, not before it starts. Err +// answers nil three times: the first segment's check, then its first turn's request and response. +// A replay that fed the ring from inside the callback has fed that turn by then, so this fails it, +// where a context already done would deliver nothing and let it pass. +func TestReplayUsage_AnExhaustedBudgetFeedsNothing(t *testing.T) { + const allowed = 3 + root := t.TempDir() + now := time.Now() + recordTraffic(t, root, nil, now) + arch, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + defer arch.Close() + + // The premise: this deadline interrupts a read that has started delivering. + delivered := 0 + if _, err := arch.ReplaySince(&errAfter{context.Background(), allowed}, now.Add(-usage.MaxWindow), + func(string, *pipeline.SessionEvent) { delivered++ }); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("ReplaySince err = %v, want context.DeadlineExceeded", err) + } + if delivered == 0 || delivered >= 8 { + t.Fatalf("ReplaySince delivered %d of 8 events before the deadline; want some but not all", delivered) + } + + agg := usage.New(usage.WithClock(func() time.Time { return now })) + ctx := &errAfter{context.Background(), allowed} + if _, _, _, err := replayUsage(ctx, arch, agg, now); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded", err) + } + if n := agg.Snapshot(usage.MaxWindow, usage.MaxWindow, "", usage.GroupNone).Totals.Requests; n != 0 { + t.Fatalf("an abandoned replay fed %d requests", n) + } +} diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index 5473d491e..bcea0e343 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -677,6 +677,10 @@ func main() { // The session archive is the store's third Recorder: it persists every event so // sessions survive a restart and the store's eviction. See core/session/archive. sessArchive = openSessionArchive(cfg, *configPath, sessions) + if sessArchive != nil { + // Before any listener starts: see replayUsage for why that is the whole design. + replayUsageAtStartup(sessArchive, usageAgg) + } // Through lim.LogAttrs, not a hand-rolled attribute list. #999 gave the session // store's limits one home, and the local "ttl=0s would read like a diff --git a/core/cost/usage/replay.go b/core/cost/usage/replay.go new file mode 100644 index 000000000..fc0194b7b --- /dev/null +++ b/core/cost/usage/replay.go @@ -0,0 +1,67 @@ +package usage + +import ( + "encoding/json" + + "github.com/rossoctl/cortex/core/cost/event" + "github.com/rossoctl/cortex/core/pipeline" +) + +// Replay records an event from before this process started — the session archive's, at startup — +// and reports whether it was in range. It is Record with one guard: an event whose minute is in the +// future, or not among the newest NumBuckets-1 minutes, is dropped. +// +// THE GUARD IS WHAT MAKES OLD EVENTS SAFE TO FEED. A slot is a minute modulo NumBuckets, and a +// write to a slot holding a different minute resets it (see Aggregator): an event exactly +// NumBuckets minutes old shares its slot with the current minute and would erase it. Record never +// meets one, because live events are current; a replay is made of nothing else. +// +// ONE MINUTE OF MARGIN, so a minute boundary passing between this check and Record's lock cannot +// carry an event over the edge. A check outside the lock is enough because a replay runs before any +// listener starts: nothing else writes the ring meanwhile. +// +// Feed events in ascending At order. Record pairs a request with the response after it, and +// reclaims the least recently written session ring at the cap, so that order is what makes the +// replayed ring the one the old process had. +func (a *Aggregator) Replay(sessionID string, e *pipeline.SessionEvent) bool { + if e == nil || e.At.IsZero() { + return false + } + now := a.now().Truncate(BucketWidth) + m := e.At.Truncate(BucketWidth) + if m.After(now) || !m.After(now.Add(-(NumBuckets-1)*BucketWidth)) { + return false + } + a.Record(sessionID, e) + return true +} + +// ReplayCopy is e reduced to what Record reads, for a caller that holds many events before +// replaying them: an archived inference event carries its whole conversation, and six hours of +// those would otherwise sit in memory at once to be sorted. A copy costs about 1KB, however long +// the conversation was. +// +// A COPY WITH THE HEAVY FIELDS CLEARED, NOT A COPY OF THE FIELDS RECORD READS, so a field Record +// starts reading later is kept without anyone remembering to add it here. What is cleared is what +// Record has no use for: the protocol payloads, the conversation, the tool manifest, the completion +// and its tool calls, and every plugin entry but the cost record. TestReplayCopy_RecordsWhatTheEventDoes +// holds that. +func ReplayCopy(e *pipeline.SessionEvent) pipeline.SessionEvent { + c := *e + c.A2A, c.MCP = nil, nil + if e.Inference != nil { + inf := *e.Inference + inf.Messages, inf.Tools, inf.ToolChoice, inf.ToolCalls, inf.Completion = nil, nil, nil, nil, "" + c.Inference = &inf + } + c.Plugins = nil + for _, k := range [...]string{event.Key, event.PluginName} { + if raw, ok := e.Plugins[k]; ok { + if c.Plugins == nil { + c.Plugins = make(map[string]json.RawMessage, 1) + } + c.Plugins[k] = raw + } + } + return c +} diff --git a/core/cost/usage/replay_test.go b/core/cost/usage/replay_test.go new file mode 100644 index 000000000..fbe29acba --- /dev/null +++ b/core/cost/usage/replay_test.go @@ -0,0 +1,71 @@ +package usage + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// The guard: an event NumBuckets minutes old shares the current minute's slot, and replaying it +// must not erase that minute; nor may the margin minute or a future event land. +func TestReplay_DropsWhatWouldResetAVisibleBucket(t *testing.T) { + now := time.Date(2026, 10, 5, 12, 0, 30, 0, time.UTC) + a := New(WithClock(func() time.Time { return now })) + a.Record("s1", respEvent(now, 200, time.Second, "m", 100)) + for _, at := range []time.Time{ + now.Add(-NumBuckets * BucketWidth), now.Add(-(NumBuckets - 1) * BucketWidth), now.Add(BucketWidth), + } { + if a.Replay("s1", respEvent(at, 200, time.Second, "m", 7)) { + t.Errorf("replayed an event at %v", at) + } + } + if got := a.Snapshot(MaxWindow, MaxWindow, "", GroupNone).Totals.Tokens; got != 100 { + t.Fatalf("tokens = %d, want only the live event's 100", got) + } + if !a.Replay("s1", respEvent(now.Add(-(NumBuckets-2)*BucketWidth), 200, time.Second, "m", 7)) { + t.Fatal("an event inside the window was dropped") + } + if a.Replay("s1", &pipeline.SessionEvent{Phase: pipeline.SessionResponse}) { + t.Fatal("replayed an event with no time") + } +} + +// ReplayCopy keeps everything Record reads: a ring fed the copies of a fixture that sets every +// bucket field — with a conversation and a stray plugin entry added, which it must drop — is the +// ring fed the events, in every grouping. +func TestReplayCopy_RecordsWhatTheEventDoes(t *testing.T) { + now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + full, slim := New(WithClock(clock)), New(WithClock(clock)) + evs := append(richTurn(t, now.Add(-2*time.Minute), "m1", "claude-code", "api.anthropic.com", 40), + richTurn(t, now.Add(-time.Minute), "m2", "opencode", "api.openai.com", 70)...) + for _, e := range evs { + if e.Inference != nil { + e.Inference.Messages = []pipeline.InferenceMessage{{Role: "user", Content: "a conversation"}} + } + if e.Plugins == nil { + e.Plugins = map[string]json.RawMessage{} + } + e.Plugins["mcp-parser"] = json.RawMessage(`{"x":1}`) + c := ReplayCopy(e) + if c.Inference != nil && c.Inference.Messages != nil { + t.Fatal("ReplayCopy kept the conversation") + } + if _, ok := c.Plugins["mcp-parser"]; ok { + t.Fatal("ReplayCopy kept a plugin entry Record does not read") + } + full.Record("s1", e) + slim.Record("s1", &c) + } + for _, g := range []Group{GroupNone, GroupModel, GroupEndpoint, GroupSession, GroupAgent, + GroupCurrency, GroupStatus, GroupPlugin, GroupHost} { + f := full.Snapshot(10*time.Minute, BucketWidth, "", g) + s := slim.Snapshot(10*time.Minute, BucketWidth, "", g) + if !reflect.DeepEqual(f, s) { + t.Errorf("group %s: the copies record differently:\nfull %+v\nslim %+v", g, f, s) + } + } +} diff --git a/core/session/archive/replay.go b/core/session/archive/replay.go new file mode 100644 index 000000000..84592d17e --- /dev/null +++ b/core/session/archive/replay.go @@ -0,0 +1,64 @@ +package archive + +import ( + "context" + "errors" + "io/fs" + "path/filepath" + "time" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// ReplaySince calls fn for every archived event at or after cutoff that is not a tunnel row, +// stamped with the id its session has now. Order is per segment in seq order; segments and +// sessions come in any order, so a caller that needs time order sorts what it collects. It +// reports how many segments could not be read; the rest still replay. +// +// FOR STARTUP, BEFORE ANY LISTENER. It reads the index as it stands when called. Called before +// the store serves traffic, it sees exactly what earlier processes wrote, since a process never +// appends to a segment it did not open. +// +// A segment last written before cutoff is skipped unread: it can hold nothing newer. Every other +// one is decoded whole, since segments record no event times. Each event fn receives is the +// reader's own, freshly decoded; a caller that keeps many should keep a reduced copy, not the +// event (see usage.ReplayCopy). +// +// It returns ctx.Err() as soon as ctx is done, checked between events, so a deadline bounds it. +func (a *Archive) ReplaySince(ctx context.Context, cutoff time.Time, fn func(sessionID string, e *pipeline.SessionEvent)) (skipped int, err error) { + type source struct{ id, path string } + var srcs []source + a.idxMu.RLock() + for id, e := range a.index { + for _, seg := range e.segments { + if seg.Events > 0 && !seg.LastWrite.Before(cutoff) { + srcs = append(srcs, source{id, filepath.Join(e.dir, seg.File)}) + } + } + } + a.idxMu.RUnlock() + for _, src := range srcs { + if err := ctx.Err(); err != nil { + return skipped, err + } + var stop error + _, rerr := readSegment(src.path, func(ev *pipeline.SessionEvent) bool { + if stop = ctx.Err(); stop != nil { + return false + } + if ev.Tunnel || ev.At.IsZero() || ev.At.Before(cutoff) { + return true + } + ev.SessionID = src.id + fn(src.id, ev) + return true + }) + switch { + case stop != nil: + return skipped, stop + case rerr != nil && !errors.Is(rerr, fs.ErrNotExist): + skipped++ // a segment pruned or renamed since the snapshot is ErrNotExist, and not a loss + } + } + return skipped, nil +} diff --git a/core/session/archive/replay_bench_test.go b/core/session/archive/replay_bench_test.go new file mode 100644 index 000000000..b09c182c1 --- /dev/null +++ b/core/session/archive/replay_bench_test.go @@ -0,0 +1,51 @@ +package archive + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/usage" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// BenchmarkReplaySince_SixHours is the startup replay's read half over a heavy six hours: twelve +// sessions of 120 turns with 64KB prompts, each event reduced with usage.ReplayCopy as +// cmd/cortex does. Its result sizes cmd/cortex's usageReplayBudget. +func BenchmarkReplaySince_SixHours(b *testing.B) { + clk := newClock() + root := b.TempDir() + a, err := Open(root, WithClock(clk.now)) + if err != nil { + b.Fatal(err) + } + st := session.New(0, 0, 0) + st.AddRecorder(a) + for s := range 12 { + for _, e := range synthSession(int64(100+s), 120, 64<<10) { + e.At = clk.now() + st.Append(fmt.Sprintf("s%d", s), e) + } + } + st.Close() + if err := a.Close(); err != nil { + b.Fatal(err) + } + a, err = Open(root, WithClock(clk.now)) + if err != nil { + b.Fatal(err) + } + defer a.Close() + b.ResetTimer() + for b.Loop() { + var kept []pipeline.SessionEvent + if _, err := a.ReplaySince(context.Background(), clk.now().Add(-6*time.Hour), func(_ string, e *pipeline.SessionEvent) { + kept = append(kept, usage.ReplayCopy(e)) + }); err != nil { + b.Fatal(err) + } + b.ReportMetric(float64(len(kept)), "events") + } +} diff --git a/core/session/archive/replay_test.go b/core/session/archive/replay_test.go new file mode 100644 index 000000000..7479969d5 --- /dev/null +++ b/core/session/archive/replay_test.go @@ -0,0 +1,66 @@ +package archive + +import ( + "context" + "errors" + "maps" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// ReplaySince hands over what is at or after the cutoff, from every session, under each session's +// current id, without tunnel rows. A segment last written before the cutoff is not read at all: +// the old segment here holds an event whose own time is new, and it must not arrive. +func TestReplaySince_DeliversWhatIsNewEnough(t *testing.T) { + clk := newClock() + a := openTest(t, t.TempDir(), clk) + defer a.Close() + rec := newRecorder(a) + ev := func(at time.Time, tunnel bool) pipeline.SessionEvent { + return pipeline.SessionEvent{At: at, Phase: pipeline.SessionResponse, StatusCode: 200, Tunnel: tunnel} + } + rec.record("old", ev(clk.now().Add(7*time.Hour), false)) + settle(a) + clk.advance(7 * time.Hour) + tick(a) // closes old's idle writer: a closed segment last written 7h ago + now := clk.now() + cutoff := now.Add(-6 * time.Hour) + rec.record("s1", ev(cutoff.Add(-time.Minute), false), ev(cutoff, false), ev(now, true), ev(now, false)) + rec.record("s2", ev(now.Add(-time.Hour), false)) + settle(a) + a.Rekeyed("s2", "s3") + settle(a) + tick(a) // flush the open segments so a reader sees them + + got := map[string]int{} + skipped, err := a.ReplaySince(context.Background(), cutoff, func(id string, e *pipeline.SessionEvent) { + if e.SessionID != id { + t.Errorf("event stamped %q delivered under %q", e.SessionID, id) + } + got[id]++ + }) + if err != nil || skipped != 0 { + t.Fatalf("ReplaySince: skipped %d, err %v", skipped, err) + } + if want := map[string]int{"s1": 2, "s3": 1}; !maps.Equal(got, want) { + t.Fatalf("delivered %v, want %v", got, want) + } +} + +// A done context stops the replay with its error, before anything more is delivered. +func TestReplaySince_StopsWhenTheContextIsDone(t *testing.T) { + clk := newClock() + a := openTest(t, t.TempDir(), clk) + defer a.Close() + newRecorder(a).record("s1", pipeline.SessionEvent{At: clk.now(), Phase: pipeline.SessionResponse}) + settle(a) + tick(a) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + if _, err := a.ReplaySince(ctx, time.Time{}, func(string, *pipeline.SessionEvent) { calls++ }); !errors.Is(err, context.Canceled) || calls != 0 { + t.Fatalf("err %v after %d calls, want context.Canceled after none", err, calls) + } +} diff --git a/docs/laptop-service.md b/docs/laptop-service.md index 6fac575d9..ce5fd364a 100644 --- a/docs/laptop-service.md +++ b/docs/laptop-service.md @@ -303,7 +303,10 @@ records is also written to `~/.cortex/sessions`, so a session outlives a restart own eviction (`session.max_sessions`, a `session.ttl`). `agentop` lists every session the archive holds beside the ones in memory, and Enter opens one like any other. Each session keeps its figures across a restart too — events, tokens, cost, title: the proxy adds what the archive holds -from before the restart to what it has seen since. +from before the restart to what it has seen since. LAST 1H and the usage pane's hour windows +survive a restart as well: at startup, before it serves anything, the proxy replays the archive's +last six hours into them. If that would take longer than 5 seconds it gives up, and they start +empty as they used to. **Unlike the cost ledger, this is the content.** Prompts, completions, tool arguments and tool results — everything the session API serves — sit in From b69d6e82858e1a15079cf0076116a746ec9a4dd8 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 6 Oct 2026 20:48:57 -0400 Subject: [PATCH 4/5] fix: Give no prior to an entry whose id holds events past its start Fixes review: a rename the archive refused or lost left the renamed entry's events under the old id, and the next entry there listed them a second time Files: - core/session/archive/archive.go - core/session/archive/prior_test.go - core/session/archive/read.go Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/session/archive/archive.go | 8 ++- core/session/archive/prior_test.go | 97 ++++++++++++++++++++++++++++++ core/session/archive/read.go | 5 +- 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 25f9ceccc..5a234d056 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -446,14 +446,18 @@ func (a *Archive) take(o op) { } // begin marks where a new store entry for id began numbering: its events start a segment of -// their own, and the summary of what came before is kept, so a rename can part the two. +// their own. func (a *Archive) begin(id string, after uint64) { s := a.sessions[id] if s == nil { return } a.closeWriter(s) - s.startAfter, s.before = after, s.meta.Summary.Clone() + if s.meta.lastSeq() > after { + s.startAfter, s.before = 0, nil + } else { + s.startAfter, s.before = after, s.meta.Summary.Clone() + } // Published at once rather than with the event's write: a paused or failing write publishes // nothing, and the store's list asks for before from the moment its entry exists. a.publish(s) diff --git a/core/session/archive/prior_test.go b/core/session/archive/prior_test.go index e66c4218d..e8ae6ad8c 100644 --- a/core/session/archive/prior_test.go +++ b/core/session/archive/prior_test.go @@ -1,6 +1,9 @@ package archive import ( + "os" + "path/filepath" + "runtime" "testing" "github.com/rossoctl/cortex/core/pipeline" @@ -132,3 +135,97 @@ func TestPrior_AFoldHandedOutNeverChanges(t *testing.T) { t.Fatalf("a handed-out fold moved from %d to %d events", held, got) } } + +// A rename the archive does not carry out leaves the renamed entry's events under the old id. A +// later entry there must not list them a second time: together, the resident rows hold at most +// what was appended. Each case renames X's entry to Y that way after a restart over 3 events; +// X then gets a new entry, which is renamed in turn. +func TestPrior_ARenameLeftUnderTheOldIdIsNotCountedTwice(t *testing.T) { + earlier, first, second := synthSession(81, 1, 256), synthSession(82, 1, 256), synthSession(83, 1, 256) + ref := session.New(0, 0, 0) + defer ref.Close() + for _, evs := range [][]pipeline.SessionEvent{earlier, first, second} { + for _, e := range evs { + ref.Append("all", e) + } + } + all := listed(t, ref, "all") + + cases := []struct { + name string + // rename moves X's entry to Y in the store and leaves its events under X in the archive. + rename func(t *testing.T, a *Archive, st *session.Store) + }{ + {"the target's directory exists", func(t *testing.T, a *Archive, st *session.Store) { + if err := os.MkdirAll(filepath.Join(a.data, dirName("Y")), dirMode); err != nil { + t.Fatal(err) + } + st.Rekey("X", "Y") + }}, + {"the rename is lost to a full queue", func(t *testing.T, a *Archive, st *session.Store) { + a.gate = make(chan struct{}) + a.Rekeyed("filler", "filler2") // the writer takes it and waits at the gate + for len(a.ops) > 0 { + runtime.Gosched() + } + for len(a.ops) < cap(a.ops) { + a.Rekeyed("filler", "filler2") + } + st.Rekey("X", "Y") + close(a.gate) + if got := a.Stats().DroppedRenames; got != 1 { + t.Fatalf("dropped %d renames, want 1", got) + } + }}, + {"the target's session.json cannot be written", func(t *testing.T, a *Archive, st *session.Store) { + if os.Geteuid() == 0 { + t.Skip("root writes through a read-only directory") + } + if err := os.Chmod(a.data, 0o500); err != nil { + t.Fatal(err) + } + defer os.Chmod(a.data, dirMode) + st.Rekey("X", "Y") + settle(a) + if got := a.Stats().WriteErrors; got != 1 { + t.Fatalf("%d write errors, want the rename's 1", got) + } + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + a, st := restarted(t, t.TempDir(), newClock(), "X", earlier) + defer a.Close() + defer st.Close() + atMost := func(when string) { + t.Helper() + var events, tokens int + for _, s := range st.ListSessions() { + events, tokens = events+s.EventCount, tokens+s.TotalTokens + } + if events > all.EventCount || tokens > all.TotalTokens { + t.Fatalf("%s, the rows hold %d events and %d tokens; %d and %d were appended", + when, events, tokens, all.EventCount, all.TotalTokens) + } + if events < len(first)+len(second) { + t.Fatalf("%s, the rows hold %d events, fewer than the two entries' own %d", + when, events, len(first)+len(second)) + } + } + for _, e := range first { + st.Append("X", e) + } + settle(a) + c.rename(t, a, st) + settle(a) + for _, e := range second { + st.Append("X", e) + } + settle(a) + atMost("with a new entry under X") + st.Rekey("X", "Z") + settle(a) + atMost("with that entry renamed in turn") + }) + } +} diff --git a/core/session/archive/read.go b/core/session/archive/read.go index 77c046f62..ff0d799da 100644 --- a/core/session/archive/read.go +++ b/core/session/archive/read.go @@ -134,9 +134,8 @@ func (a *Archive) Summaries() []session.SessionSummary { // numbering after `after` began. ok only when the writer has marked that entry's start, because // exactly then is before the history up to `after` and nothing of the entry's: begin runs in queue // order, after every earlier event and before any of the entry's. Until the writer takes the -// entry's first event, and wherever it could mark none (begin found no state for the id: after a -// clear, a prune, a whole-directory rename), the answer is false, and the store shows the entry's -// own figures — an undercount, never a double count. +// entry's first event, and wherever it could mark none, the answer is false, and the store shows +// the entry's own figures — an undercount, never a double count. // // A memory lookup under idxMu, safe under the store's read lock because idxMu is a LEAF lock: it // may be taken while the store's read lock is held, as here, but nothing is ever acquired while From 81b9e2b6364cd191c94233c8729f7a15bbaa8906 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 6 Oct 2026 22:39:01 -0400 Subject: [PATCH 5/5] fix: Count segments the startup replay reads only in part Fixes review: ReplaySince dropped readSegment's truncated result, so a torn or unended segment replayed its prefix while the startup log said unreadableSegments=0 Files: - cmd/cortex/archive.go - core/session/archive/replay.go - core/session/archive/replay_test.go Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/cortex/archive.go | 12 +++--- core/session/archive/replay.go | 26 +++++++----- core/session/archive/replay_test.go | 63 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 17 deletions(-) diff --git a/cmd/cortex/archive.go b/cmd/cortex/archive.go index e422f1308..7182b7357 100644 --- a/cmd/cortex/archive.go +++ b/cmd/cortex/archive.go @@ -119,7 +119,7 @@ const usageReplayBudget = 5 * time.Second // ALL OR NOTHING. A read that does not finish within ctx is abandoned before anything is fed, and // the ring starts empty as it always did: a partial ring would show a LAST 1H that is too low as if // it were real. -func replayUsage(ctx context.Context, arch *archive.Archive, agg *usage.Aggregator, now time.Time) (events, sessions, skipped int, err error) { +func replayUsage(ctx context.Context, arch *archive.Archive, agg *usage.Aggregator, now time.Time) (events, sessions, incomplete int, err error) { type replayed struct { id string e pipeline.SessionEvent @@ -127,11 +127,11 @@ func replayUsage(ctx context.Context, arch *archive.Archive, agg *usage.Aggregat // About 1KB a kept event (see usage.ReplayCopy), held until this returns. Keeping the decoded // event instead would hold six hours of conversations at once. var evs []replayed - skipped, err = arch.ReplaySince(ctx, now.Add(-usage.MaxWindow), func(id string, e *pipeline.SessionEvent) { + incomplete, err = arch.ReplaySince(ctx, now.Add(-usage.MaxWindow), func(id string, e *pipeline.SessionEvent) { evs = append(evs, replayed{id, usage.ReplayCopy(e)}) }) if err != nil { - return 0, 0, skipped, err + return 0, 0, incomplete, err } // Seq breaks a tie within one session, where it is the order the store appended in; across // sessions a tie has no order to keep. @@ -148,7 +148,7 @@ func replayUsage(ctx context.Context, arch *archive.Archive, agg *usage.Aggregat seen[evs[i].id] = true } } - return events, len(seen), skipped, nil + return events, len(seen), incomplete, nil } // replayUsageAtStartup runs replayUsage within usageReplayBudget and says how it went. @@ -156,7 +156,7 @@ func replayUsageAtStartup(arch *archive.Archive, agg *usage.Aggregator) { ctx, cancel := context.WithTimeout(context.Background(), usageReplayBudget) defer cancel() start := time.Now() - events, sessions, skipped, err := replayUsage(ctx, arch, agg, start) + events, sessions, incomplete, err := replayUsage(ctx, arch, agg, start) took := time.Since(start) if err != nil { slog.Warn("usage: did not replay the session archive into the usage ring; LAST 1H and the usage pane start empty", @@ -164,5 +164,5 @@ func replayUsageAtStartup(arch *archive.Archive, agg *usage.Aggregator) { return } slog.Info("usage: replayed the session archive into the usage ring", "window", usage.MaxWindow, - "events", events, "sessions", sessions, "unreadableSegments", skipped, "took", took) + "events", events, "sessions", sessions, "incompleteSegments", incomplete, "took", took) } diff --git a/core/session/archive/replay.go b/core/session/archive/replay.go index 84592d17e..02ae5cfa4 100644 --- a/core/session/archive/replay.go +++ b/core/session/archive/replay.go @@ -12,8 +12,7 @@ import ( // ReplaySince calls fn for every archived event at or after cutoff that is not a tunnel row, // stamped with the id its session has now. Order is per segment in seq order; segments and -// sessions come in any order, so a caller that needs time order sorts what it collects. It -// reports how many segments could not be read; the rest still replay. +// sessions come in any order, so a caller that needs time order sorts what it collects. // // FOR STARTUP, BEFORE ANY LISTENER. It reads the index as it stands when called. Called before // the store serves traffic, it sees exactly what earlier processes wrote, since a process never @@ -25,24 +24,27 @@ import ( // event (see usage.ReplayCopy). // // It returns ctx.Err() as soon as ctx is done, checked between events, so a deadline bounds it. -func (a *Archive) ReplaySince(ctx context.Context, cutoff time.Time, fn func(sessionID string, e *pipeline.SessionEvent)) (skipped int, err error) { - type source struct{ id, path string } +func (a *Archive) ReplaySince(ctx context.Context, cutoff time.Time, fn func(sessionID string, e *pipeline.SessionEvent)) (incomplete int, err error) { + type source struct { + id, path string + open bool + } var srcs []source a.idxMu.RLock() for id, e := range a.index { - for _, seg := range e.segments { + for i, seg := range e.segments { if seg.Events > 0 && !seg.LastWrite.Before(cutoff) { - srcs = append(srcs, source{id, filepath.Join(e.dir, seg.File)}) + srcs = append(srcs, source{id, filepath.Join(e.dir, seg.File), e.open && i == len(e.segments)-1}) } } } a.idxMu.RUnlock() for _, src := range srcs { if err := ctx.Err(); err != nil { - return skipped, err + return incomplete, err } var stop error - _, rerr := readSegment(src.path, func(ev *pipeline.SessionEvent) bool { + truncated, rerr := readSegment(src.path, func(ev *pipeline.SessionEvent) bool { if stop = ctx.Err(); stop != nil { return false } @@ -55,10 +57,12 @@ func (a *Archive) ReplaySince(ctx context.Context, cutoff time.Time, fn func(ses }) switch { case stop != nil: - return skipped, stop + return incomplete, stop case rerr != nil && !errors.Is(rerr, fs.ErrNotExist): - skipped++ // a segment pruned or renamed since the snapshot is ErrNotExist, and not a loss + incomplete++ // a segment pruned or renamed since the snapshot is ErrNotExist, and not a loss + case truncated && !src.open: + incomplete++ } } - return skipped, nil + return incomplete, nil } diff --git a/core/session/archive/replay_test.go b/core/session/archive/replay_test.go index 7479969d5..53aaacb5c 100644 --- a/core/session/archive/replay_test.go +++ b/core/session/archive/replay_test.go @@ -4,10 +4,13 @@ import ( "context" "errors" "maps" + "os" + "path/filepath" "testing" "time" "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" ) // ReplaySince hands over what is at or after the cutoff, from every session, under each session's @@ -64,3 +67,63 @@ func TestReplaySince_StopsWhenTheContextIsDone(t *testing.T) { t.Fatalf("err %v after %d calls, want context.Canceled after none", err, calls) } } + +// A segment the replay could read only in part is counted, and its prefix still replays. Each +// case leaves one session's segment as the process that wrote it could have, and the next +// process opens the archive over it. +func TestReplaySince_CountsASegmentReadOnlyInPart(t *testing.T) { + clk := newClock() + var evs []pipeline.SessionEvent + for i := range 3 { + evs = append(evs, pipeline.SessionEvent{Seq: uint64(i + 1), At: clk.now(), Phase: pipeline.SessionResponse, StatusCode: 200}) + } + cases := []struct { + name string + // segment returns the segment's bytes as its writer left them. + segment func(t *testing.T) []byte + incomplete int + delivered int + }{ + {"closed cleanly", func(t *testing.T) []byte { + path, _ := writeSegment(t, evs, true) + return readFile(t, path) + }, 0, 3}, + {"killed before its frame ended", func(t *testing.T) []byte { + path, _ := writeSegment(t, evs, false) + return readFile(t, path) + }, 1, 3}, + {"torn inside its last event", func(t *testing.T) []byte { + path, sizes := writeSegment(t, evs, true) + return readFile(t, path)[:sizes[1]+(sizes[2]-sizes[1])/2] + }, 1, 2}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, dataDirName, dirName("s1")) + if err := writeMeta(dir, &sessionMeta{ID: "s1", CreatedAt: clk.now(), UpdatedAt: clk.now(), Summary: session.NewSummaryFold()}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "0001"+segmentExt), c.segment(t), 0o600); err != nil { + t.Fatal(err) + } + a := openTest(t, root, clk) + defer a.Close() + delivered := 0 + incomplete, err := a.ReplaySince(context.Background(), time.Time{}, func(string, *pipeline.SessionEvent) { delivered++ }) + if err != nil || incomplete != c.incomplete || delivered != c.delivered { + t.Fatalf("ReplaySince counted %d incomplete and delivered %d (err %v); want %d and %d", + incomplete, delivered, err, c.incomplete, c.delivered) + } + }) + } +} + +func readFile(t *testing.T, path string) []byte { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return b +}