From b72fed329aee55b297b1d0d0276e2374de35bbd4 Mon Sep 17 00:00:00 2001 From: Falk Harrison Date: Tue, 29 Sep 2026 10:46:31 +0200 Subject: [PATCH] perf(history): read only the requested recent entries --- internal/history/history.go | 70 +++++++++++++ internal/history/history_test.go | 171 +++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 internal/history/history_test.go diff --git a/internal/history/history.go b/internal/history/history.go index c775ea7..b7df53e 100644 --- a/internal/history/history.go +++ b/internal/history/history.go @@ -86,6 +86,16 @@ func Load(path string, maxEntries int) ([]Entry, error) { } defer f.Close() + if maxEntries > 0 { + info, err := f.Stat() + if err != nil { + return nil, err + } + if info.Mode().IsRegular() { + return loadRecent(f, info.Size(), maxEntries) + } + } + entries := make([]Entry, 0, 256) scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) @@ -114,6 +124,66 @@ func Load(path string, maxEntries int) ([]Entry, error) { return entries, nil } +func loadRecent(r io.ReaderAt, size int64, limit int) ([]Entry, error) { + const maxLineSize = 1024 * 1024 + entries := make([]Entry, 0, min(limit, 256)) + buffer := make([]byte, 64*1024) + var recordSuffix []byte + consume := func(prefix []byte) error { + if len(prefix)+len(recordSuffix) >= maxLineSize { + return bufio.ErrTooLong + } + line := prefix + if len(recordSuffix) > 0 { + line = append(append(make([]byte, 0, len(prefix)+len(recordSuffix)), prefix...), recordSuffix...) + recordSuffix = nil + } + if len(line) == 0 { + return nil + } + var entry Entry + if err := json.Unmarshal(line, &entry); err == nil { + entries = append(entries, entry) + } + return nil + } + for position := size; position > 0; { + n := int(min(position, int64(len(buffer)))) + position -= int64(n) + read, err := r.ReadAt(buffer[:n], position) + if err != nil { + return entries, err + } + if read != n { + return entries, io.ErrUnexpectedEOF + } + end := n + for i := n - 1; i >= 0; i-- { + if buffer[i] != '\n' { + continue + } + if err := consume(buffer[i+1 : end]); err != nil { + return entries, err + } + if len(entries) == limit { + return entries, nil + } + end = i + } + if end+len(recordSuffix) >= maxLineSize { + return entries, bufio.ErrTooLong + } + next := make([]byte, end+len(recordSuffix)) + copy(next, buffer[:end]) + copy(next[end:], recordSuffix) + recordSuffix = next + } + if err := consume(nil); err != nil { + return entries, err + } + return entries, nil +} + // CheatKey returns the stable history identity for a parsed cheat. It matches // recorded executions by source file and section header, so command edits still // keep useful ranking history. diff --git a/internal/history/history_test.go b/internal/history/history_test.go new file mode 100644 index 0000000..6dd882c --- /dev/null +++ b/internal/history/history_test.go @@ -0,0 +1,171 @@ +package history + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +type historyReader struct { + *strings.Reader + bytesRead int +} + +func (r *historyReader) ReadAt(p []byte, offset int64) (int, error) { + n, err := r.Reader.ReadAt(p, offset) + r.bytesRead += n + return n, err +} + +func TestRecentHistoryDoesNotReadOldRecords(t *testing.T) { + line := "{\"cmd\":\"hello\"}\n" + r := &historyReader{Reader: strings.NewReader(strings.Repeat(line, 100000))} + entries, err := loadRecent(r, r.Size(), 1000) + if err != nil || len(entries) != 1000 { + t.Fatalf("load: %d, %v", len(entries), err) + } + if r.bytesRead > 64*1024 { + t.Fatalf("read %d bytes for a tail smaller than one chunk", r.bytesRead) + } +} + +func TestLoadRecentHistory(t *testing.T) { + for _, trailing := range []string{"", "\n"} { + path := filepath.Join(t.TempDir(), "history.jsonl") + spanningRecord, _ := json.Marshal(Entry{Command: strings.Repeat("x", 100000)}) + content := "{\"cmd\":\"first\"}\n\ninvalid json\n" + string(spanningRecord) + "\n{\"cmd\":\"third\"}\n{broken\n{\"cmd\":\"last\"}" + trailing + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + all, err := Load(path, 0) + if err != nil { + t.Fatal(err) + } + if len(all) != 4 || all[0].Command != "last" || all[3].Command != "first" { + t.Fatalf("unexpected history: %d entries", len(all)) + } + for _, limit := range []int{-1, 1, 2, 3, 4, 10} { + got, err := Load(path, limit) + if err != nil { + t.Fatal(err) + } + want := all + if limit > 0 { + want = all[:min(limit, len(all))] + } + if !reflect.DeepEqual(got, want) { + t.Errorf("limit %d: bounded and unlimited loads differ", limit) + } + } + } +} + +func TestLoadEmptyAndMissingHistory(t *testing.T) { + path := filepath.Join(t.TempDir(), "history.jsonl") + got, err := Load(path, 10) + if err != nil || len(got) != 0 { + t.Fatalf("missing: %v, %v", got, err) + } + for _, content := range []string{"", "\n\n", "not json\n"} { + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + got, err := Load(path, 10) + if err != nil || len(got) != 0 { + t.Fatalf("empty: %v, %v", got, err) + } + } +} + +func BenchmarkLoadRecent(b *testing.B) { + for _, count := range []int{1000, 100000} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + path := filepath.Join(b.TempDir(), "history.jsonl") + line := "{\"ts\":\"2026-01-01T00:00:00Z\",\"cmd\":\"echo hello\",\"file\":\"greeting.md\",\"header\":\"Greeting\",\"scope\":{\"value\":\"hello\"}}\n" + if err := os.WriteFile(path, []byte(strings.Repeat(line, count)), 0600); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + entries, err := Load(path, 1000) + if err != nil || len(entries) != 1000 { + b.Fatalf("load: %d, %v", len(entries), err) + } + } + }) + } +} + +func TestLoadRecentLineSizeLimit(t *testing.T) { + for _, ending := range []string{"", "\n", "\r\n"} { + for _, size := range []int{1024*1024 - 1, 1024 * 1024, 1024*1024 + 1} { + t.Run(fmt.Sprintf("size=%d/ending=%q", size, ending), func(t *testing.T) { + command := strings.Repeat("x", size-len(`{"cmd":""}`)) + content := `{"cmd":"` + command + `"}` + ending + path := filepath.Join(t.TempDir(), "history.jsonl") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + for _, limit := range []int{0, 1} { + got, err := Load(path, limit) + oversized := size >= 1024*1024 || size == 1024*1024-1 && ending == "\r\n" + if oversized { + if !errors.Is(err, bufio.ErrTooLong) { + t.Fatalf("limit %d: expected line limit error, got %v", limit, err) + } + } else if err != nil || len(got) != 1 || got[0].Command != command { + t.Fatalf("limit %d: expected one complete record, got %d, %v", limit, len(got), err) + } + } + }) + } + } +} + +func TestLoadRecentStopsBeforeOldOversizedRecord(t *testing.T) { + path := filepath.Join(t.TempDir(), "history.jsonl") + content := strings.Repeat("x", 1024*1024+1) + "\n{\"cmd\":\"recent\"}\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + got, err := Load(path, 1) + if err != nil || len(got) != 1 || got[0].Command != "recent" { + t.Fatalf("expected recent entry without reading old record: %v, %v", got, err) + } + if _, err := Load(path, 0); !errors.Is(err, bufio.ErrTooLong) { + t.Fatalf("unlimited load must still report oversized old record: %v", err) + } +} + +func TestLoadRecentSkipsMalformedRecordsAcrossChunks(t *testing.T) { + path := filepath.Join(t.TempDir(), "history.jsonl") + content := "{\"cmd\":\"first\"}\r\n{\"cmd\":\"second\"}\r\n" + strings.Repeat("invalid\r\n\r\n", 10000) + "{\"cmd\":\"last\"}\r\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + got, err := Load(path, 2) + want := []Entry{{Command: "last"}, {Command: "second"}} + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("expected newest valid entries: %v, %v", got, err) + } +} + +func TestRecentHistorySnapshot(t *testing.T) { + first := "{\"cmd\":\"first\"}\n" + r := strings.NewReader(first + "{\"cmd\":\"later\"}\n") + got, err := loadRecent(r, int64(len(first)), 10) + if err != nil || !reflect.DeepEqual(got, []Entry{{Command: "first"}}) { + t.Fatalf("snapshot included later append: %v, %v", got, err) + } + if _, err := loadRecent(r, r.Size()+1, 10); !errors.Is(err, io.EOF) { + t.Fatalf("expected truncated read error: %v", err) + } +}