Rebuild evicted log indexes

src/Concerns/LogIndex/CanSplitIndexIntoChunks.php

@@ -35,7 +35,13 @@ public function getCurrentChunk(): LogIndexChunk
             $this->currentChunk = LogIndexChunk::fromDefinitionArray($this->currentChunkDefinition);
 
             if ($this->currentChunk->size > 0) {
-                $this->currentChunk->data = $this->getChunkDataFromCache($this->currentChunk->index, []);
+                $data = $this->getChunkDataFromCache($this->currentChunk->index);
+                $this->currentChunk->data = $data ?? [];
+
+                if (is_null($data)) {
+                    // The current chunk was evicted from cache while the metadata survived.
+                    $this->markForRebuild();
+                }
             }
         }
 
@@ -68,6 +74,12 @@ public function getChunkData(int $index): ?array
             $chunkData = $currentChunk->data ?? [];
         } else {
             $chunkData = $this->getChunkDataFromCache($index);
+
+            if (is_null($chunkData) && ($this->getChunkDefinition($index)['size'] ?? 0) > 0) {
+                // The chunk was evicted from cache while the index metadata survived.
+                // Flag the index so the next scan rebuilds it from scratch.
+                $this->markForRebuild();
+            }
         }
 
         return $chunkData;

src/Concerns/LogIndex/HasMetadata.php

@@ -11,6 +11,7 @@ public function getMetadata(): array
             'identifier' => $this->identifier,
             'last_scanned_file_position' => $this->lastScannedFilePosition,
             'last_scanned_index' => $this->lastScannedIndex,
+            'rebuild_required' => $this->rebuildRequired ?? false,
             'next_log_index_to_create' => $this->nextLogIndexToCreate,
             'max_chunk_size' => $this->maxChunkSize,
             'current_chunk_index' => $this->getCurrentChunk()->index,
@@ -30,9 +31,14 @@ protected function loadMetadata(): void
 
         $this->lastScannedFilePosition = $data['last_scanned_file_position'] ?? 0;
         $this->lastScannedIndex = $data['last_scanned_index'] ?? 0;
+        $this->rebuildRequired = $data['rebuild_required'] ?? false;
         $this->nextLogIndexToCreate = $data['next_log_index_to_create'] ?? 0;
         $this->maxChunkSize = $data['max_chunk_size'] ?? self::DEFAULT_CHUNK_SIZE;
         $this->chunkDefinitions = $data['chunk_definitions'] ?? [];
         $this->currentChunkDefinition = $data['current_chunk_definition'] ?? [];
+
+        // The memoized chunk object is derived from the definition loaded above,
+        // so it must not survive a metadata (re)load — e.g. after clearCache().
+        unset($this->currentChunk);
     }
 }

src/Concerns/LogIndex/PreservesIndexingProgress.php

@@ -40,6 +40,28 @@ public function incomplete(): bool
         return $this->file->size() !== $this->getLastScannedFilePosition();
     }
 
+    public function markForRebuild(): void
+    {
+        if ($this->rebuildRequired ?? false) {
+            return;
+        }
+
+        $this->rebuildRequired = true;
+
+        // Persist immediately so the next request sees the flag even though
+        // the current request is mid-read and won't save the index itself.
+        $this->saveMetadata();
+    }
+
+    public function requiresRebuild(): bool
+    {
+        if (! isset($this->rebuildRequired)) {
+            $this->loadMetadata();
+        }
+
+        return $this->rebuildRequired;
+    }
+
     public function getEarliestTimestamp(): ?int
     {
         $earliestTimestamp = null;

src/LogIndex.php

@@ -21,6 +21,7 @@ class LogIndex
     protected int $nextLogIndexToCreate;
     protected int $lastScannedFilePosition;
     protected int $lastScannedIndex;
+    protected bool $rebuildRequired;
 
     public function __construct(
         public LogFile $file,

src/Readers/IndexedLogReader.php

@@ -58,6 +58,12 @@ public function scan(?int $maxBytesToScan = null, bool $force = false): static
             return $this;
         }
 
+        if ($this->index()->requiresRebuild()) {
+            // A partially-evicted index cannot be resumed (the scan position is at the end
+            // of the file) or re-scanned in place (surviving chunks would duplicate entries).
+            $force = true;
+        }
+
         if ($this->numberOfNewBytes() < 0) {
             // the file reduced in size... something must've gone wrong, so let's
             // force a full re-index.
@@ -274,6 +280,20 @@ public function numberOfNewBytes(): int
 
     public function requiresScan(): bool
     {
+        if ($this->index()->requiresRebuild()) {
+            // A cached index chunk was lost; rebuild on the next scan.
+            return true;
+        }
+
+        // File metadata can outlive index cache entries; rebuild when the index was lost.
+        if ($this->file->size() > 0) {
+            $index = $this->index();
+
+            if ($index->getLastScannedFilePosition() === 0 && $index->count() === 0) {
+                return true;
+            }
+        }
+
         if (isset($this->mtimeBeforeScan) && ($this->file->mtime() > $this->mtimeBeforeScan || $this->file->mtime() === time())) {
             // The file has been modified since the last scan in this request.
             // Let's only request another scan if it's not the last chunk (smaller than lazyScanChunkSize).

tests/Unit/LogReaderTest.php

@@ -2,7 +2,10 @@
 
 use Illuminate\Support\Facades\File;
 use Opcodes\LogViewer\Exceptions\CannotOpenFileException;
+use Opcodes\LogViewer\Facades\Cache as LogViewerCache;
+use Opcodes\LogViewer\LogFile;
 use Opcodes\LogViewer\Readers\IndexedLogReader;
+use Opcodes\LogViewer\Utils\GenerateCacheKey;
 use Spatie\TestTime\TestTime;
 
 beforeEach(function () {
@@ -43,6 +46,117 @@
         ->and($index->getFlatIndex())->toHaveCount(2);
 });
 
+it('rebuilds a search index when its cache is evicted but file metadata survives', function () {
+    $path = $this->file->path;
+
+    $read = function () use ($path) {
+        IndexedLogReader::clearInstances();
+
+        $logReader = (new LogFile($path))->logs()->search('Testing');
+        $logReader->scan();
+
+        return [
+            'requires_scan' => $logReader->requiresScan(),
+            'new_bytes' => $logReader->numberOfNewBytes(),
+            'total' => $logReader->total(),
+            'count' => count($logReader->reset()->get()),
+        ];
+    };
+
+    expect($read())->toMatchArray([
+        'requires_scan' => false,
+        'new_bytes' => 0,
+        'total' => 1,
+        'count' => 1,
+    ]);
+
+    $index = (new LogFile($path))->index('~Testing~iu');
+
+    LogViewerCache::forget(GenerateCacheKey::for($index, 'metadata'));
+    LogViewerCache::forget(GenerateCacheKey::for($index, 'chunk:0'));
+
+    expect($read())->toMatchArray([
+        'requires_scan' => false,
+        'new_bytes' => 0,
+        'total' => 1,
+        'count' => 1,
+    ]);
+});
+
+it('rebuilds the index when a cached chunk is evicted but metadata survives', function () {
+    File::append($this->file->path, PHP_EOL.makeLaravelLogEntry());
+
+    $path = $this->file->path;
+    $this->file->index()->setMaxChunkSize(1);
+    $this->file->logs()->scan();
+
+    expect($this->file->logs()->total())->toBe(2);
+
+    LogViewerCache::forget(GenerateCacheKey::for($this->file->index(), 'chunk:0'));
+
+    // The request that discovers the eviction serves the surviving chunks
+    // and flags the index for a rebuild.
+    IndexedLogReader::clearInstances();
+    $logReader = (new LogFile($path))->logs();
+
+    expect($logReader->requiresScan())->toBeFalse()
+        ->and(count($logReader->get()))->toBe(1)
+        ->and($logReader->requiresScan())->toBeTrue();
+
+    // The next request (e.g. a page reload) rebuilds the index from scratch.
+    IndexedLogReader::clearInstances();
+    $logReader = (new LogFile($path))->logs();
+
+    expect($logReader->requiresScan())->toBeTrue();
+
+    $logReader->scan();
+
+    expect($logReader->requiresScan())->toBeFalse()
+        ->and($logReader->total())->toBe(2)
+        ->and(count($logReader->reset()->get()))->toBe(2);
+});
+
+it('rebuilds the index when the current chunk is evicted but metadata survives', function () {
+    File::append($this->file->path, PHP_EOL.makeLaravelLogEntry());
+
+    $path = $this->file->path;
+    // Default chunk size, so both entries live in the single (current) chunk.
+    $this->file->logs()->scan();
+
+    expect($this->file->logs()->total())->toBe(2);
+
+    LogViewerCache::forget(GenerateCacheKey::for($this->file->index(), 'chunk:0'));
+
+    // The request that discovers the eviction flags the index for a rebuild.
+    IndexedLogReader::clearInstances();
+    $logReader = (new LogFile($path))->logs();
+
+    expect(count($logReader->get()))->toBe(0)
+        ->and($logReader->requiresScan())->toBeTrue();
+
+    // The next request (e.g. a page reload) rebuilds the index from scratch.
+    IndexedLogReader::clearInstances();
+    $logReader = (new LogFile($path))->logs();
+
+    expect($logReader->requiresScan())->toBeTrue();
+
+    $logReader->scan();
+
+    expect($logReader->requiresScan())->toBeFalse()
+        ->and($logReader->total())->toBe(2)
+        ->and(count($logReader->reset()->get()))->toBe(2);
+});
+
+it('does not flag a rebuild when reading a chunk the index does not know about', function () {
+    $this->file->logs()->scan();
+
+    $index = $this->file->index();
+
+    expect($index->getChunkData(99))->toBeNull()
+        ->and($index->requiresRebuild())->toBeFalse()
+        ->and($this->file->logs()->requiresScan())->toBeFalse();
+});
+
 it('throws an exception when file cannot be opened for reading', function () {
     if (PHP_OS_FAMILY === 'Windows') {
         $this->markTestSkipped('File permissions work differently on Windows. The feature tested might still work.');