From 447690063924857833487a202d9331357e5d0404 Mon Sep 17 00:00:00 2001 From: Mario Finelli Date: Sat, 4 Apr 2026 10:55:45 +0200 Subject: [PATCH 1/5] Add nexus cache to export bundle --- cmd/export.go | 2 + internal/exporter/exporter.go | 3 + internal/exporter/full.go | 54 ++++++++++++ internal/exporter/game.go | 154 +++++++++++++++++++++++++++++++++ internal/nexusclient/client.go | 7 ++ 5 files changed, 220 insertions(+) diff --git a/cmd/export.go b/cmd/export.go index 2a08ae8..7fe0304 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -21,6 +21,7 @@ package cmd import ( "fmt" "os" + "path/filepath" "time" "github.com/charmbracelet/lipgloss" @@ -93,6 +94,7 @@ Examples: ModctlVersion: rootCmd.Version, SkipInventory: exportSkipInventory, NoVerify: exportNoVerify, + CacheDBPath: filepath.Join(viper.GetString("cache_dir"), "nexus_cache.db"), } date := time.Now().Format("20060102") diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index 0320848..d351b82 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -49,6 +49,8 @@ type Options struct { OutputPath string // NoVerify skips rehashing blobs before export NoVerify bool + // CacheDBPath is the full path to the nexus_cache.db + CacheDBPath string } const ( @@ -77,6 +79,7 @@ type Manifest struct { ModctlVersion string `json:"modctl_version"` SchemaVersion int64 `json:"schema_version"` DBSha256 string `json:"db_sha256"` + NexusCacheSha256 string `json:"nexus_cache_sha256"` Counts ManifestCounts `json:"counts"` Game *ManifestGame `json:"game,omitempty"` } diff --git a/internal/exporter/full.go b/internal/exporter/full.go index a2e67b9..215e464 100644 --- a/internal/exporter/full.go +++ b/internal/exporter/full.go @@ -28,6 +28,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/mfinelli/modctl/dbq" + "github.com/mfinelli/modctl/internal" "github.com/mfinelli/modctl/internal/blobstore" ) @@ -97,6 +98,15 @@ func Full( } defer os.Remove(dbPath) + // 1b. Snapshot the nexus cache database + cacheDBPath, cacheSha256, err := snapshotCacheDB(ctx, opts.CacheDBPath) + if err != nil { + return fmt.Errorf("snapshot nexus cache: %w", err) + } + if cacheDBPath != "" { + defer os.Remove(cacheDBPath) + } + // 2. Get schema version schemaVersion, err := currentSchemaVersion(ctx, db) if err != nil { @@ -125,6 +135,7 @@ func Full( ModctlVersion: opts.ModctlVersion, SchemaVersion: schemaVersion, DBSha256: dbSha256, + NexusCacheSha256: cacheSha256, Counts: ManifestCounts{ Archives: len(archiveBlobs), Backups: len(backupBlobs), @@ -140,6 +151,13 @@ func Full( return fmt.Errorf("write database: %w", err) } + // 5b. Write nexus cache snapshot if present + if cacheDBPath != "" { + if err := writeFileToTar(tw, cacheDBPath, "nexus_cache.db"); err != nil { + return fmt.Errorf("write nexus cache: %w", err) + } + } + // 6. Write archive blobs var skipped []string for _, b := range archiveBlobs { @@ -217,3 +235,39 @@ func snapshotDB(ctx context.Context, db *sql.DB) (path string, sha256hex string, return tmpPath, sha, nil } + +// snapshotCacheDB creates a consistent snapshot of the nexus cache database. +// Returns the path to the temp file and its sha256. If the cache DB does not +// exist yet, returns empty strings and no error. +func snapshotCacheDB(ctx context.Context, cacheDBPath string) (path string, sha256hex string, err error) { + if _, err := os.Stat(cacheDBPath); os.IsNotExist(err) { + return "", "", nil + } + + tmp, err := os.CreateTemp("", "modctl-export-cache-*.sqlite") + if err != nil { + return "", "", fmt.Errorf("create temp file: %w", err) + } + tmpPath := tmp.Name() + tmp.Close() + + cacheDB, err := sql.Open("sqlite3", cacheDBPath+internal.DB_PRAGMAS) + if err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("open cache db: %w", err) + } + defer cacheDB.Close() + + if _, err := cacheDB.ExecContext(ctx, "VACUUM INTO ?", tmpPath); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("vacuum cache db: %w", err) + } + + sha, err := hashFile(tmpPath) + if err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("hash cache snapshot: %w", err) + } + + return tmpPath, sha, nil +} diff --git a/internal/exporter/game.go b/internal/exporter/game.go index 0afe1ca..b193422 100644 --- a/internal/exporter/game.go +++ b/internal/exporter/game.go @@ -22,6 +22,7 @@ import ( "archive/tar" "context" "database/sql" + "errors" "fmt" "os" "time" @@ -30,6 +31,8 @@ import ( "github.com/mfinelli/modctl/dbq" "github.com/mfinelli/modctl/internal" "github.com/mfinelli/modctl/internal/blobstore" + "github.com/mfinelli/modctl/internal/nexusclient" + "github.com/mfinelli/modctl/internal/nexusclient/dbc" ) // Game performs a game-scoped export containing only data relevant to a @@ -95,6 +98,17 @@ func Game( } defer os.Remove(scopedDBPath) + // 1b. Build scoped nexus cache database + cacheDBPath, cacheSha256, err := buildGameScopedCacheDB( + ctx, q, opts.CacheDBPath, gi.ID, + ) + if err != nil { + return fmt.Errorf("build game-scoped nexus cache: %w", err) + } + if cacheDBPath != "" { + defer os.Remove(cacheDBPath) + } + // 2. Get schema version from source DB schemaVersion, err := currentSchemaVersion(ctx, db) if err != nil { @@ -119,6 +133,7 @@ func Game( ModctlVersion: opts.ModctlVersion, SchemaVersion: schemaVersion, DBSha256: dbSha256, + NexusCacheSha256: cacheSha256, Counts: ManifestCounts{ Archives: archiveCount, Backups: backupCount, @@ -139,6 +154,13 @@ func Game( return fmt.Errorf("write database: %w", err) } + // 5b. Write scoped nexus cache if present + if cacheDBPath != "" { + if err := writeFileToTar(tw, cacheDBPath, "nexus_cache.db"); err != nil { + return fmt.Errorf("write nexus cache: %w", err) + } + } + // 6. Write archive blobs var skipped []string for _, b := range archiveBlobs { @@ -562,3 +584,135 @@ func exportWriteOncePatterns(ctx context.Context, src, dst *dbq.Queries, gameIns } return nil } + +// buildGameScopedCacheDB constructs a fresh nexus cache database containing +// only rows relevant to the given game install's mod pages. Returns the path +// to the temp file and its sha256. If the cache DB does not exist or the game +// has no nexus-linked mod pages, returns empty strings and no error. +func buildGameScopedCacheDB( + ctx context.Context, + q *dbq.Queries, + cacheDBPath string, + gameInstallID int64, +) (path string, sha256hex string, err error) { + if _, err := os.Stat(cacheDBPath); os.IsNotExist(err) { + return "", "", nil + } + + // Get nexus identifiers for this game's mod pages + modPages, err := q.GetNexusLinkedModPages(ctx, gameInstallID) + if err != nil { + return "", "", fmt.Errorf("get nexus linked mod pages: %w", err) + } + if len(modPages) == 0 { + return "", "", nil + } + + // Open source cache DB + srcCacheDB, err := sql.Open("sqlite3", cacheDBPath+internal.DB_PRAGMAS) + if err != nil { + return "", "", fmt.Errorf("open cache db: %w", err) + } + defer srcCacheDB.Close() + + // Create destination temp file + tmp, err := os.CreateTemp("", "modctl-export-cache-*.sqlite") + if err != nil { + return "", "", fmt.Errorf("create temp cache db: %w", err) + } + tmpPath := tmp.Name() + tmp.Close() + + dstCacheDB, err := sql.Open("sqlite3", tmpPath+internal.DB_PRAGMAS) + if err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("open scoped cache db: %w", err) + } + defer dstCacheDB.Close() + + if err := nexusclient.InitCacheDB(ctx, dstCacheDB); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("init scoped cache db: %w", err) + } + + srcQ := dbc.New(srcCacheDB) + dstQ := dbc.New(dstCacheDB) + + // Copy rows for each mod page + for _, mp := range modPages { + domain := mp.NexusGameDomain.String + modID := mp.NexusModID.Int64 + + if err := exportCacheModInfo(ctx, srcQ, dstQ, domain, modID); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("export cache mod info (%s/%d): %w", domain, modID, err) + } + if err := exportCacheFileInfo(ctx, srcQ, dstQ, domain, modID); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("export cache file info (%s/%d): %w", domain, modID, err) + } + if err := exportCacheFileUpdates(ctx, srcQ, dstQ, domain, modID); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("export cache file updates (%s/%d): %w", domain, modID, err) + } + } + + if err := dstCacheDB.Close(); err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("close scoped cache db: %w", err) + } + + sha, err := hashFile(tmpPath) + if err != nil { + os.Remove(tmpPath) + return "", "", fmt.Errorf("hash scoped cache db: %w", err) + } + + return tmpPath, sha, nil +} + +func exportCacheModInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { + row, err := src.GetNexusModInfo(ctx, dbc.GetNexusModInfoParams{ + NexusGameDomain: domain, + NexusModID: modID, + }) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + return dst.UpsertNexusModInfo(ctx, dbc.UpsertNexusModInfoParams(row)) +} + +func exportCacheFileInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { + rows, err := src.GetNexusFileInfoForMod(ctx, dbc.GetNexusFileInfoForModParams{ + NexusGameDomain: domain, + NexusModID: modID, + }) + if err != nil { + return err + } + for _, row := range rows { + if err := dst.UpsertNexusFileInfo(ctx, dbc.UpsertNexusFileInfoParams(row)); err != nil { + return err + } + } + return nil +} + +func exportCacheFileUpdates(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { + rows, err := src.GetNexusFileUpdatesForMod(ctx, dbc.GetNexusFileUpdatesForModParams{ + NexusGameDomain: domain, + NexusModID: modID, + }) + if err != nil { + return err + } + for _, row := range rows { + if err := dst.UpsertNexusFileUpdate(ctx, dbc.UpsertNexusFileUpdateParams(row)); err != nil { + return err + } + } + return nil +} diff --git a/internal/nexusclient/client.go b/internal/nexusclient/client.go index 2a1fbf5..8d2ed9b 100644 --- a/internal/nexusclient/client.go +++ b/internal/nexusclient/client.go @@ -105,6 +105,13 @@ func openCacheDB(ctx context.Context) (*sql.DB, error) { return db, nil } +// InitCacheDB initializes or resets the nexus cache database schema. +// It is exported for use by the exporter when constructing scoped cache +// databases for export bundles. +func InitCacheDB(ctx context.Context, db *sql.DB) error { + return initCacheDB(ctx, db) +} + func initCacheDB(ctx context.Context, db *sql.DB) error { if err := applyCacheSchema(ctx, db); err != nil { return err From 4ead9853740e1eacc081ead80e533073e7a0ffde Mon Sep 17 00:00:00 2001 From: Mario Finelli Date: Sat, 4 Apr 2026 11:20:10 +0200 Subject: [PATCH 2/5] Add cache database check to bundle verify --- cmd/verify.go | 52 +++++++++++++++++++++++++++++++++++++ internal/restore/restore.go | 15 +++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmd/verify.go b/cmd/verify.go index a099071..0ff6c94 100644 --- a/cmd/verify.go +++ b/cmd/verify.go @@ -30,6 +30,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/mfinelli/modctl/dbq" + "github.com/mfinelli/modctl/internal" "github.com/mfinelli/modctl/internal/blobstore" "github.com/mfinelli/modctl/internal/restore" "github.com/spf13/cobra" @@ -139,6 +140,26 @@ Exits non-zero if any integrity issues are found. Version warnings } fmt.Println() + // nexus cache integrity check + if bundle.Manifest.NexusCacheSha256 != "" { + fmt.Println(boldStyle.Render("Nexus Cache Integrity")) + cachePath := filepath.Join(bundle.BundleDir, "nexus_cache.db") + cacheIssues := checkBundleCacheDB(ctx, cachePath) + if len(cacheIssues) == 0 { + fmt.Println(okStyle.Render(" ✓ quick_check OK")) + } else { + for _, iss := range cacheIssues { + fmt.Println(errStyle.Render(" ✗ " + iss)) + issues = append(issues, iss) + } + } + fmt.Println() + } else { + fmt.Println(boldStyle.Render("Nexus Cache Integrity")) + fmt.Println(subtleStyle.Render(" (not present in bundle)")) + fmt.Println() + } + // blob checks fmt.Println(boldStyle.Render("Blob Integrity")) blobIssues := checkBundleBlobs(ctx, bundle, bq) @@ -349,3 +370,34 @@ func checkBundleBlobs(ctx context.Context, bundle *restore.Bundle, bq *dbq.Queri return issues } + +// checkBundleCacheDB runs quick_check on the nexus cache database. +func checkBundleCacheDB(ctx context.Context, cachePath string) []string { + cacheDB, err := sql.Open("sqlite3", cachePath+internal.DB_PRAGMAS+"&mode=ro") + if err != nil { + return []string{"open nexus cache db: " + err.Error()} + } + defer cacheDB.Close() + + rows, err := cacheDB.QueryContext(ctx, "PRAGMA quick_check;") + if err != nil { + return []string{"quick_check failed: " + err.Error()} + } + defer rows.Close() + + var issues []string + for rows.Next() { + var result string + if err := rows.Scan(&result); err != nil { + issues = append(issues, "quick_check scan error: "+err.Error()) + continue + } + if result != "ok" { + issues = append(issues, "quick_check: "+result) + } + } + if err := rows.Err(); err != nil { + issues = append(issues, "quick_check rows error: "+err.Error()) + } + return issues +} diff --git a/internal/restore/restore.go b/internal/restore/restore.go index 2e27808..08f0b39 100644 --- a/internal/restore/restore.go +++ b/internal/restore/restore.go @@ -130,6 +130,21 @@ func OpenAndValidate(ctx context.Context, bundlePath string) (*Bundle, error) { manifest.DBSha256, dbSha) } + // Verify nexus cache integrity if present in bundle + cachePath := filepath.Join(tmpDir, "nexus_cache.db") + if manifest.NexusCacheSha256 != "" { + cacheSha, err := hashFile(cachePath) + if err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("hash bundle nexus cache: %w", err) + } + if cacheSha != manifest.NexusCacheSha256 { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("bundle nexus cache integrity check failed: expected %s got %s", + manifest.NexusCacheSha256, cacheSha) + } + } + // Open bundle DB bundleDB, err := sql.Open("sqlite3", dbPath+internal.DB_PRAGMAS+"&mode=ro") if err != nil { From c5e5e29a3d3688d8f54ee6e460cdd0785f644ae5 Mon Sep 17 00:00:00 2001 From: Mario Finelli Date: Sat, 4 Apr 2026 11:29:50 +0200 Subject: [PATCH 3/5] Update import to use bundled cache --- cmd/import.go | 2 + internal/exporter/game.go | 12 ++-- internal/restore/cache.go | 114 ++++++++++++++++++++++++++++++++++++ internal/restore/full.go | 8 +++ internal/restore/game.go | 9 +++ internal/restore/restore.go | 1 + 6 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 internal/restore/cache.go diff --git a/cmd/import.go b/cmd/import.go index 5240267..bc53095 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "github.com/charmbracelet/lipgloss" "github.com/mfinelli/modctl/dbq" @@ -139,6 +140,7 @@ Use --dry-run to preview what would be imported without making any changes.`, SkipInventory: importSkipInventory, SameMachine: importSameMachine, Game: importGame, + CacheDBPath: filepath.Join(viper.GetString("cache_dir"), "nexus_cache.db"), } // Warn if bundle modctl version is newer diff --git a/internal/exporter/game.go b/internal/exporter/game.go index b193422..694dab7 100644 --- a/internal/exporter/game.go +++ b/internal/exporter/game.go @@ -643,15 +643,15 @@ func buildGameScopedCacheDB( domain := mp.NexusGameDomain.String modID := mp.NexusModID.Int64 - if err := exportCacheModInfo(ctx, srcQ, dstQ, domain, modID); err != nil { + if err := ExportCacheModInfo(ctx, srcQ, dstQ, domain, modID); err != nil { os.Remove(tmpPath) return "", "", fmt.Errorf("export cache mod info (%s/%d): %w", domain, modID, err) } - if err := exportCacheFileInfo(ctx, srcQ, dstQ, domain, modID); err != nil { + if err := ExportCacheFileInfo(ctx, srcQ, dstQ, domain, modID); err != nil { os.Remove(tmpPath) return "", "", fmt.Errorf("export cache file info (%s/%d): %w", domain, modID, err) } - if err := exportCacheFileUpdates(ctx, srcQ, dstQ, domain, modID); err != nil { + if err := ExportCacheFileUpdates(ctx, srcQ, dstQ, domain, modID); err != nil { os.Remove(tmpPath) return "", "", fmt.Errorf("export cache file updates (%s/%d): %w", domain, modID, err) } @@ -671,7 +671,7 @@ func buildGameScopedCacheDB( return tmpPath, sha, nil } -func exportCacheModInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { +func ExportCacheModInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { row, err := src.GetNexusModInfo(ctx, dbc.GetNexusModInfoParams{ NexusGameDomain: domain, NexusModID: modID, @@ -685,7 +685,7 @@ func exportCacheModInfo(ctx context.Context, src, dst *dbc.Queries, domain strin return dst.UpsertNexusModInfo(ctx, dbc.UpsertNexusModInfoParams(row)) } -func exportCacheFileInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { +func ExportCacheFileInfo(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { rows, err := src.GetNexusFileInfoForMod(ctx, dbc.GetNexusFileInfoForModParams{ NexusGameDomain: domain, NexusModID: modID, @@ -701,7 +701,7 @@ func exportCacheFileInfo(ctx context.Context, src, dst *dbc.Queries, domain stri return nil } -func exportCacheFileUpdates(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { +func ExportCacheFileUpdates(ctx context.Context, src, dst *dbc.Queries, domain string, modID int64) error { rows, err := src.GetNexusFileUpdatesForMod(ctx, dbc.GetNexusFileUpdatesForModParams{ NexusGameDomain: domain, NexusModID: modID, diff --git a/internal/restore/cache.go b/internal/restore/cache.go new file mode 100644 index 0000000..139711d --- /dev/null +++ b/internal/restore/cache.go @@ -0,0 +1,114 @@ +/* + * mod control (modctl): command-line mod manager + * Copyright © 2026 Mario Finelli + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package restore + +import ( + "context" + "database/sql" + "fmt" + + "github.com/mfinelli/modctl/dbq" + "github.com/mfinelli/modctl/internal" + "github.com/mfinelli/modctl/internal/exporter" + "github.com/mfinelli/modctl/internal/nexusclient" + "github.com/mfinelli/modctl/internal/nexusclient/dbc" +) + +// importFullCache copies the nexus cache database from the bundle into place, +// initializing it if it does not already exist. +func importFullCache(ctx context.Context, bundleCachePath, destCachePath string) error { + if err := copyFile(bundleCachePath, destCachePath); err != nil { + return fmt.Errorf("copy nexus cache: %w", err) + } + + // Open and initialize to handle schema version check / reset if stale + db, err := sql.Open("sqlite3", destCachePath+internal.DB_PRAGMAS) + if err != nil { + return fmt.Errorf("open nexus cache: %w", err) + } + defer db.Close() + + return nexusclient.InitCacheDB(ctx, db) +} + +// importGameCache merges nexus cache rows for the imported game's mod pages +// into the live cache database. +func importGameCache( + ctx context.Context, + bundleCachePath string, + destCachePath string, + bq *dbq.Queries, + oldGameInstallID int64, +) error { + // Get nexus identifiers for the imported game's mod pages + modPages, err := bq.ExportGetModPagesForGameInstall(ctx, oldGameInstallID) + if err != nil { + return fmt.Errorf("get mod pages: %w", err) + } + + // Filter to nexus mod pages only + type nexusKey struct { + domain string + modID int64 + } + var keys []nexusKey + for _, mp := range modPages { + if mp.NexusGameDomain.Valid && mp.NexusModID.Valid { + keys = append(keys, nexusKey{mp.NexusGameDomain.String, mp.NexusModID.Int64}) + } + } + if len(keys) == 0 { + return nil + } + + // Open bundle cache DB + srcDB, err := sql.Open("sqlite3", bundleCachePath+internal.DB_PRAGMAS+"&mode=ro") + if err != nil { + return fmt.Errorf("open bundle nexus cache: %w", err) + } + defer srcDB.Close() + src := dbc.New(srcDB) + + // Open or create live cache DB + destDB, err := sql.Open("sqlite3", destCachePath+internal.DB_PRAGMAS) + if err != nil { + return fmt.Errorf("open live nexus cache: %w", err) + } + defer destDB.Close() + + if err := nexusclient.InitCacheDB(ctx, destDB); err != nil { + return fmt.Errorf("init live nexus cache: %w", err) + } + dst := dbc.New(destDB) + + // Merge rows for each mod page + for _, key := range keys { + if err := exporter.ExportCacheModInfo(ctx, src, dst, key.domain, key.modID); err != nil { + return fmt.Errorf("merge cache mod info (%s/%d): %w", key.domain, key.modID, err) + } + if err := exporter.ExportCacheFileInfo(ctx, src, dst, key.domain, key.modID); err != nil { + return fmt.Errorf("merge cache file info (%s/%d): %w", key.domain, key.modID, err) + } + if err := exporter.ExportCacheFileUpdates(ctx, src, dst, key.domain, key.modID); err != nil { + return fmt.Errorf("merge cache file updates (%s/%d): %w", key.domain, key.modID, err) + } + } + + return nil +} diff --git a/internal/restore/full.go b/internal/restore/full.go index 4d9fcdd..b2d880d 100644 --- a/internal/restore/full.go +++ b/internal/restore/full.go @@ -111,6 +111,14 @@ func Full( } } + // Import nexus cache if present in bundle + if bundle.Manifest.NexusCacheSha256 != "" { + bundleCachePath := filepath.Join(bundle.BundleDir, "nexus_cache.db") + if err := importFullCache(ctx, bundleCachePath, opts.CacheDBPath); err != nil { + return res, fmt.Errorf("import nexus cache: %w", err) + } + } + // Scan missing inventories against the restored DB bq := dbq.New(bundle.BundleDB) allVersions, err := bq.ListAllModFileVersions(ctx) diff --git a/internal/restore/game.go b/internal/restore/game.go index f7c5a6b..c354c51 100644 --- a/internal/restore/game.go +++ b/internal/restore/game.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "log/slog" + "path/filepath" "strings" "github.com/mfinelli/modctl/dbq" @@ -238,6 +239,14 @@ func Game( return res, fmt.Errorf("import inventory: %w", err) } + // Import nexus cache if present in bundle + if bundle.Manifest.NexusCacheSha256 != "" { + bundleCachePath := filepath.Join(bundle.BundleDir, "nexus_cache.db") + if err := importGameCache(ctx, bundleCachePath, opts.CacheDBPath, bq, oldGameInstallID); err != nil { + return res, fmt.Errorf("import nexus cache: %w", err) + } + } + // Queue missing inventory scans scanned, failed, err := scanMissingInventories(ctx, db, q, bs, oldGameInstallID, bq, opts.SkipInventory, logger) if err != nil { diff --git a/internal/restore/restore.go b/internal/restore/restore.go index 08f0b39..ab7edd4 100644 --- a/internal/restore/restore.go +++ b/internal/restore/restore.go @@ -50,6 +50,7 @@ type Options struct { DryRun bool SameMachine bool Game string // "store_id:store_game_id", only set when extracting a game from a full bundle + CacheDBPath string } type Result struct { From fee306d4c0abf9b277cf7eb6ab893129c21769f3 Mon Sep 17 00:00:00 2001 From: Mario Finelli Date: Sat, 4 Apr 2026 11:42:37 +0200 Subject: [PATCH 4/5] Update documentation for nexus bundle --- DESIGN.md | 42 ++++++++++++++++++++++++++++++--- docs/04_import-export.md | 7 ++++-- docs/07_internals/04_exports.md | 42 ++++++++++++++++++++++++++++----- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index e64d179..2cfdd9d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -239,7 +239,6 @@ Version schema from day 1. A separate SQLite database at `$XDG_CACHE_HOME/modctl/nexus_cache.db` stores cached Nexus API responses. This is intentionally separate from the main DB: - It is safe to delete (will be repopulated on next `mods nexus check-updates`) -- It is excluded from export/import bundles - It uses a simple internal version number; if the schema version does not match the expected version the cache is blown away and recreated @@ -274,6 +273,7 @@ Rationale: A single file (tar + zstd) containing: - `manifest.json` - bundle metadata (see below) - `modctl.db` - database snapshot +- `nexus_cache.db` - nexus cache database snapshot - `archives//` - referenced archive blobs - `backups//` - referenced backup blobs @@ -294,6 +294,8 @@ Import verifies integrity and schema compatibility. to verify integrity on import - `counts`: `{ "archives": N, "backups": N }` - `game` (game-scoped only): `{ "store_id", "store_game_id", "display_name" }` +- `cache_sha256`: sha256 of the nexus cache database snapshot as it appears + in the bundle, used to verify integrity on import #### Full export @@ -1763,6 +1765,10 @@ This is only appropriate when restoring to the same machine where game directories are still intact. `--force` and `--same-machine` are independent and can be combined. +The nexus cache database is copied into place after the main database. +Its schema version is checked independently; if newer than the running +binary the import is refused. + **Game-scoped import** inserts rows into the existing database with fresh IDs assigned by SQLite. A remapping table tracks old→new IDs for each table so foreign key references are correctly updated as rows are inserted in @@ -1773,6 +1779,10 @@ first (cascading via FK). Orphaned blobs from the deleted install are left for `gc` to clean up. `--same-machine` is not valid for game-scoped imports and will produce an error. +Cache rows for the imported game's mod pages are merged into the existing +cache database via `INSERT OR REPLACE`. Existing cache entries for other +games are not affected. + **Importing a game from a full bundle**: passing `--game store_id:store_game_id` with a full bundle routes the import through the game-scoped import path, extracting only the relevant data for that game. modctl prints an @@ -1809,6 +1819,8 @@ Checks performed: - `export_format_version` is supported (hard error if newer) - `PRAGMA quick_check` on the bundle database - `PRAGMA foreign_key_check` on the bundle database +- `cache_sha256` in the manifest matches the actual sha256 of `nexus_cache.db` +- `PRAGMA quick_check` on the cache database - Every blob file in the bundle hashes correctly against its filename - Every blob referenced in the bundle database has a corresponding file - Every blob file in the bundle has a corresponding database row @@ -1870,5 +1882,29 @@ complete integrity check. If the extracted version has a `nexus_file_id` and the mod page has `nexus_mod_id` and `nexus_game_domain` recorded, the Nexus URL and IDs -are printed after extraction. The Nexus cache is not consulted since it -is not included in bundles. +are printed after extraction. + +### Nexus cache: included in export bundles + +The nexus cache database is included in both full and game-scoped export +bundles. Although it is safe to delete locally, it contains useful data +(update chains, file metadata) that would otherwise require API calls to +rebuild on the destination machine. + +For full exports, `VACUUM INTO` is used to produce a clean snapshot, +mirroring the approach used for the main database. + +For game-scoped exports, a fresh cache database is created with migrations +applied, then populated with only the rows for the exported game's mod pages: +all rows in `nexus_mod_info`, `nexus_file_info`, and `nexus_file_updates` +where `(nexus_game_domain, nexus_mod_id)` matches a mod page belonging to +the exported game install. + +On full import, the cache database is copied into place directly. On +game-scoped import, rows are merged into the existing cache database using +`INSERT OR REPLACE` — safe because all cache tables use Nexus-native primary +keys with no local ID remapping required. + +`fetched_at` timestamps are preserved verbatim on import. The TTL logic in +`check-updates` will naturally treat stale entries as cold and refresh them +on the next run. diff --git a/docs/04_import-export.md b/docs/04_import-export.md index 72dd75f..ee694f5 100644 --- a/docs/04_import-export.md +++ b/docs/04_import-export.md @@ -10,11 +10,14 @@ A bundle is a zstd-compressed tar archive containing: - a snapshot of the modctl database - all referenced mod archives, backup, and override blobs +- a snapshot of the Nexus API cache for the exported game(s) - a manifest with metadata and integrity checksums Everything modctl needs to restore your setup is self-contained in the bundle. -The Nexus API cache is not included; it is safe to discard and will be -repopulated automatically when needed. +The Nexus cache snapshot preserves update chain data and file metadata so the +destination machine does not need to make API calls to rebuild it. If the cache +was not present on disk at export time it is simply omitted; `modctl +check-updates` will repopulate it on the next run. ## Exporting diff --git a/docs/07_internals/04_exports.md b/docs/07_internals/04_exports.md index 05b01e9..fab9140 100644 --- a/docs/07_internals/04_exports.md +++ b/docs/07_internals/04_exports.md @@ -14,15 +14,18 @@ A bundle contains the following files: ``` manifest.json modctl.db +nexus_cache.db archives// backups// overrides// ``` `manifest.json` contains metadata about the bundle. `modctl.db` is a snapshot -of the modctl database at the time of export. The `archives/` and `backups/` -directories mirror the layout of the local blob stores, containing only the -blobs referenced by the exported data. +of the modctl database at the time of export. `nexus_cache.db` is a snapshot +of the Nexus API cache, included when it was present on disk at export time. +The `archives/`, `backups/`, and `overrides/` directories mirror the layout of +the local blob stores, containing only the blobs referenced by the exported +data. Note that game-scoped bundles never include a `backups/` directory. Backup blobs describe on-disk state on the source machine and have no meaning on the @@ -41,6 +44,9 @@ destination, so they are excluded from game-scoped exports. - `schema_version`: the database schema version of the snapshot. - `db_sha256`: the SHA-256 hash of `modctl.db` as it appears in the bundle, used to verify integrity on import. +- `nexus_cache_sha256`: the SHA-256 hash of `nexus_cache.db` as it appears in + the bundle. Absent or empty if the Nexus cache was not present on disk at + export time. - `counts`: the number of archive and backup blobs included. - `game`: for game-scoped bundles only: the store ID, store game ID, and display name of the exported game. @@ -59,9 +65,33 @@ The applied profile state is intentionally cleared in game-scoped bundles since the destination machine will have its own game installation. Operation history is also not included in game-scoped bundles. -The applied profile state is intentionally cleared in game-scoped bundles -since the destination machine will have its own game installation. Operation -history is also not included in game-scoped bundles. +## Nexus cache snapshot + +The Nexus API cache is included in bundles when it was present on disk at +export time. It contains cached mod page metadata, file info, and update chain +data, which is what modctl uses to detect available updates without making +additional API calls. + +For full exports the cache snapshot is produced using `VACUUM INTO`, mirroring +the approach used for the main database. + +For game-scoped exports a fresh cache database is created and populated with +only the rows relevant to the exported game's mod pages. No other games' cache +data is included. + +On full import the cache database is copied into place directly. On +game-scoped import the rows are merged into the existing cache database using +`INSERT OR REPLACE`, so cache data for other games already on the destination +machine is not affected. + +The `fetched_at` timestamps in the cache are preserved verbatim. The TTL logic +in `modctl mods nexus check-updates` will naturally treat stale entries as cold +and refresh them on the next run. + +If `nexus_cache_sha256` is absent or empty in the manifest, the cache was not +included in the bundle. This can happen with bundles produced by older versions +of modctl, or when the cache database was not present on disk at export time. +In both cases import silently skips the cache restore step. ## Blob verification From 3d6113d557df70ed388dd9784727d0f1c17d4b30 Mon Sep 17 00:00:00 2001 From: Mario Finelli Date: Sat, 4 Apr 2026 11:43:51 +0200 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ceeec..c9050ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ based on the [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. `modctl` adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## unreleased + +### Changes + +- Start including the nexus cache in export bundles. + ## v0.6.0 - 2026-04-03 This is another pre-release that adds new features.