From bb891ea0447cf8d17ef83e2e5ab56ecb162a3f4e Mon Sep 17 00:00:00 2001 From: Arunas Skirius Date: Thu, 11 Jun 2026 14:08:35 +0300 Subject: [PATCH 1/3] Rebuild evicted log indexes --- src/Readers/IndexedLogReader.php | 9 +++++++ tests/Unit/LogReaderTest.php | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/Readers/IndexedLogReader.php b/src/Readers/IndexedLogReader.php index 3d93eda0..f91689b6 100644 --- a/src/Readers/IndexedLogReader.php +++ b/src/Readers/IndexedLogReader.php @@ -274,6 +274,15 @@ public function numberOfNewBytes(): int public function requiresScan(): bool { + // File metadata can outlive index cache entries; rebuild when the index was lost. + if ($this->file->size() > 0) { + $index = $this->index(); + + if ($index->getLastScannedFilePosition() === 0 && $index->count() === 0) { + return true; + } + } + if (isset($this->mtimeBeforeScan) && ($this->file->mtime() > $this->mtimeBeforeScan || $this->file->mtime() === time())) { // The file has been modified since the last scan in this request. // Let's only request another scan if it's not the last chunk (smaller than lazyScanChunkSize). diff --git a/tests/Unit/LogReaderTest.php b/tests/Unit/LogReaderTest.php index 82d4e19f..78d0ce4d 100644 --- a/tests/Unit/LogReaderTest.php +++ b/tests/Unit/LogReaderTest.php @@ -2,7 +2,10 @@ use Illuminate\Support\Facades\File; use Opcodes\LogViewer\Exceptions\CannotOpenFileException; +use Opcodes\LogViewer\Facades\Cache as LogViewerCache; +use Opcodes\LogViewer\LogFile; use Opcodes\LogViewer\Readers\IndexedLogReader; +use Opcodes\LogViewer\Utils\GenerateCacheKey; use Spatie\TestTime\TestTime; beforeEach(function () { @@ -43,6 +46,43 @@ ->and($index->getFlatIndex())->toHaveCount(2); }); +it('rebuilds a search index when its cache is evicted but file metadata survives', function () { + $path = $this->file->path; + + $read = function () use ($path) { + IndexedLogReader::clearInstances(); + + $logReader = (new LogFile($path))->logs()->search('Testing'); + $logReader->scan(); + + return [ + 'requires_scan' => $logReader->requiresScan(), + 'new_bytes' => $logReader->numberOfNewBytes(), + 'total' => $logReader->total(), + 'count' => count($logReader->reset()->get()), + ]; + }; + + expect($read())->toMatchArray([ + 'requires_scan' => false, + 'new_bytes' => 0, + 'total' => 1, + 'count' => 1, + ]); + + $index = (new LogFile($path))->index('~Testing~iu'); + + LogViewerCache::forget(GenerateCacheKey::for($index, 'metadata')); + LogViewerCache::forget(GenerateCacheKey::for($index, 'chunk:0')); + + expect($read())->toMatchArray([ + 'requires_scan' => false, + 'new_bytes' => 0, + 'total' => 1, + 'count' => 1, + ]); +}); + it('throws an exception when file cannot be opened for reading', function () { if (PHP_OS_FAMILY === 'Windows') { $this->markTestSkipped('File permissions work differently on Windows. The feature tested might still work.'); From 6f979a05ee3cc54bbc66eb211a54a4cc353b5c15 Mon Sep 17 00:00:00 2001 From: Arunas Skirius Date: Thu, 11 Jun 2026 15:18:08 +0300 Subject: [PATCH 2/3] Detect evicted index chunks and force a full rebuild --- .../LogIndex/CanSplitIndexIntoChunks.php | 6 +++ src/Concerns/LogIndex/HasMetadata.php | 2 + .../LogIndex/PreservesIndexingProgress.php | 18 ++++++++ src/LogIndex.php | 1 + src/Readers/IndexedLogReader.php | 11 +++++ tests/Unit/LogReaderTest.php | 43 +++++++++++++++++++ 6 files changed, 81 insertions(+) diff --git a/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php b/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php index 2132d858..81bf2744 100644 --- a/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php +++ b/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php @@ -68,6 +68,12 @@ public function getChunkData(int $index): ?array $chunkData = $currentChunk->data ?? []; } else { $chunkData = $this->getChunkDataFromCache($index); + + if (is_null($chunkData) && ($this->getChunkDefinition($index)['size'] ?? 0) > 0) { + // The chunk was evicted from cache while the index metadata survived. + // Flag the index so the next scan rebuilds it from scratch. + $this->markForRebuild(); + } } return $chunkData; diff --git a/src/Concerns/LogIndex/HasMetadata.php b/src/Concerns/LogIndex/HasMetadata.php index e6b5bb54..549f7906 100644 --- a/src/Concerns/LogIndex/HasMetadata.php +++ b/src/Concerns/LogIndex/HasMetadata.php @@ -11,6 +11,7 @@ public function getMetadata(): array 'identifier' => $this->identifier, 'last_scanned_file_position' => $this->lastScannedFilePosition, 'last_scanned_index' => $this->lastScannedIndex, + 'rebuild_required' => $this->rebuildRequired ?? false, 'next_log_index_to_create' => $this->nextLogIndexToCreate, 'max_chunk_size' => $this->maxChunkSize, 'current_chunk_index' => $this->getCurrentChunk()->index, @@ -30,6 +31,7 @@ protected function loadMetadata(): void $this->lastScannedFilePosition = $data['last_scanned_file_position'] ?? 0; $this->lastScannedIndex = $data['last_scanned_index'] ?? 0; + $this->rebuildRequired = $data['rebuild_required'] ?? false; $this->nextLogIndexToCreate = $data['next_log_index_to_create'] ?? 0; $this->maxChunkSize = $data['max_chunk_size'] ?? self::DEFAULT_CHUNK_SIZE; $this->chunkDefinitions = $data['chunk_definitions'] ?? []; diff --git a/src/Concerns/LogIndex/PreservesIndexingProgress.php b/src/Concerns/LogIndex/PreservesIndexingProgress.php index 139c809e..cf12b76c 100644 --- a/src/Concerns/LogIndex/PreservesIndexingProgress.php +++ b/src/Concerns/LogIndex/PreservesIndexingProgress.php @@ -40,6 +40,24 @@ public function incomplete(): bool return $this->file->size() !== $this->getLastScannedFilePosition(); } + public function markForRebuild(): void + { + $this->rebuildRequired = true; + + // Persist immediately so the next request sees the flag even though + // the current request is mid-read and won't save the index itself. + $this->saveMetadata(); + } + + public function requiresRebuild(): bool + { + if (! isset($this->rebuildRequired)) { + $this->loadMetadata(); + } + + return $this->rebuildRequired; + } + public function getEarliestTimestamp(): ?int { $earliestTimestamp = null; diff --git a/src/LogIndex.php b/src/LogIndex.php index 3faff11c..cc47c50e 100644 --- a/src/LogIndex.php +++ b/src/LogIndex.php @@ -21,6 +21,7 @@ class LogIndex protected int $nextLogIndexToCreate; protected int $lastScannedFilePosition; protected int $lastScannedIndex; + protected bool $rebuildRequired; public function __construct( public LogFile $file, diff --git a/src/Readers/IndexedLogReader.php b/src/Readers/IndexedLogReader.php index f91689b6..88bde0ca 100644 --- a/src/Readers/IndexedLogReader.php +++ b/src/Readers/IndexedLogReader.php @@ -58,6 +58,12 @@ public function scan(?int $maxBytesToScan = null, bool $force = false): static return $this; } + if ($this->index()->requiresRebuild()) { + // A partially-evicted index cannot be resumed (the scan position is at the end + // of the file) or re-scanned in place (surviving chunks would duplicate entries). + $force = true; + } + if ($this->numberOfNewBytes() < 0) { // the file reduced in size... something must've gone wrong, so let's // force a full re-index. @@ -274,6 +280,11 @@ public function numberOfNewBytes(): int public function requiresScan(): bool { + if ($this->index()->requiresRebuild()) { + // A cached index chunk was lost; rebuild on the next scan. + return true; + } + // File metadata can outlive index cache entries; rebuild when the index was lost. if ($this->file->size() > 0) { $index = $this->index(); diff --git a/tests/Unit/LogReaderTest.php b/tests/Unit/LogReaderTest.php index 78d0ce4d..44a3b251 100644 --- a/tests/Unit/LogReaderTest.php +++ b/tests/Unit/LogReaderTest.php @@ -83,6 +83,49 @@ ]); }); +it('rebuilds the index when a cached chunk is evicted but metadata survives', function () { + File::append($this->file->path, PHP_EOL.makeLaravelLogEntry()); + + $path = $this->file->path; + $this->file->index()->setMaxChunkSize(1); + $this->file->logs()->scan(); + + expect($this->file->logs()->total())->toBe(2); + + LogViewerCache::forget(GenerateCacheKey::for($this->file->index(), 'chunk:0')); + + // The request that discovers the eviction serves the surviving chunks + // and flags the index for a rebuild. + IndexedLogReader::clearInstances(); + $logReader = (new LogFile($path))->logs(); + + expect($logReader->requiresScan())->toBeFalse() + ->and(count($logReader->get()))->toBe(1) + ->and($logReader->requiresScan())->toBeTrue(); + + // The next request (e.g. a page reload) rebuilds the index from scratch. + IndexedLogReader::clearInstances(); + $logReader = (new LogFile($path))->logs(); + + expect($logReader->requiresScan())->toBeTrue(); + + $logReader->scan(); + + expect($logReader->requiresScan())->toBeFalse() + ->and($logReader->total())->toBe(2) + ->and(count($logReader->reset()->get()))->toBe(2); +}); + +it('does not flag a rebuild when reading a chunk the index does not know about', function () { + $this->file->logs()->scan(); + + $index = $this->file->index(); + + expect($index->getChunkData(99))->toBeNull() + ->and($index->requiresRebuild())->toBeFalse() + ->and($this->file->logs()->requiresScan())->toBeFalse(); +}); + it('throws an exception when file cannot be opened for reading', function () { if (PHP_OS_FAMILY === 'Windows') { $this->markTestSkipped('File permissions work differently on Windows. The feature tested might still work.'); From 8605f04f829d9853df377c44c243fa37f309b7e8 Mon Sep 17 00:00:00 2001 From: Arunas Skirius Date: Thu, 11 Jun 2026 15:43:54 +0300 Subject: [PATCH 3/3] Detect evicted current chunk and invalidate memoized chunk on metadata reload --- .../LogIndex/CanSplitIndexIntoChunks.php | 8 ++++- src/Concerns/LogIndex/HasMetadata.php | 4 +++ .../LogIndex/PreservesIndexingProgress.php | 4 +++ tests/Unit/LogReaderTest.php | 31 +++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php b/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php index 81bf2744..1c3dd833 100644 --- a/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php +++ b/src/Concerns/LogIndex/CanSplitIndexIntoChunks.php @@ -35,7 +35,13 @@ public function getCurrentChunk(): LogIndexChunk $this->currentChunk = LogIndexChunk::fromDefinitionArray($this->currentChunkDefinition); if ($this->currentChunk->size > 0) { - $this->currentChunk->data = $this->getChunkDataFromCache($this->currentChunk->index, []); + $data = $this->getChunkDataFromCache($this->currentChunk->index); + $this->currentChunk->data = $data ?? []; + + if (is_null($data)) { + // The current chunk was evicted from cache while the metadata survived. + $this->markForRebuild(); + } } } diff --git a/src/Concerns/LogIndex/HasMetadata.php b/src/Concerns/LogIndex/HasMetadata.php index 549f7906..7f903d60 100644 --- a/src/Concerns/LogIndex/HasMetadata.php +++ b/src/Concerns/LogIndex/HasMetadata.php @@ -36,5 +36,9 @@ protected function loadMetadata(): void $this->maxChunkSize = $data['max_chunk_size'] ?? self::DEFAULT_CHUNK_SIZE; $this->chunkDefinitions = $data['chunk_definitions'] ?? []; $this->currentChunkDefinition = $data['current_chunk_definition'] ?? []; + + // The memoized chunk object is derived from the definition loaded above, + // so it must not survive a metadata (re)load — e.g. after clearCache(). + unset($this->currentChunk); } } diff --git a/src/Concerns/LogIndex/PreservesIndexingProgress.php b/src/Concerns/LogIndex/PreservesIndexingProgress.php index cf12b76c..6a41ed95 100644 --- a/src/Concerns/LogIndex/PreservesIndexingProgress.php +++ b/src/Concerns/LogIndex/PreservesIndexingProgress.php @@ -42,6 +42,10 @@ public function incomplete(): bool public function markForRebuild(): void { + if ($this->rebuildRequired ?? false) { + return; + } + $this->rebuildRequired = true; // Persist immediately so the next request sees the flag even though diff --git a/tests/Unit/LogReaderTest.php b/tests/Unit/LogReaderTest.php index 44a3b251..18c03f29 100644 --- a/tests/Unit/LogReaderTest.php +++ b/tests/Unit/LogReaderTest.php @@ -116,6 +116,37 @@ ->and(count($logReader->reset()->get()))->toBe(2); }); +it('rebuilds the index when the current chunk is evicted but metadata survives', function () { + File::append($this->file->path, PHP_EOL.makeLaravelLogEntry()); + + $path = $this->file->path; + // Default chunk size, so both entries live in the single (current) chunk. + $this->file->logs()->scan(); + + expect($this->file->logs()->total())->toBe(2); + + LogViewerCache::forget(GenerateCacheKey::for($this->file->index(), 'chunk:0')); + + // The request that discovers the eviction flags the index for a rebuild. + IndexedLogReader::clearInstances(); + $logReader = (new LogFile($path))->logs(); + + expect(count($logReader->get()))->toBe(0) + ->and($logReader->requiresScan())->toBeTrue(); + + // The next request (e.g. a page reload) rebuilds the index from scratch. + IndexedLogReader::clearInstances(); + $logReader = (new LogFile($path))->logs(); + + expect($logReader->requiresScan())->toBeTrue(); + + $logReader->scan(); + + expect($logReader->requiresScan())->toBeFalse() + ->and($logReader->total())->toBe(2) + ->and(count($logReader->reset()->get()))->toBe(2); +}); + it('does not flag a rebuild when reading a chunk the index does not know about', function () { $this->file->logs()->scan();