Refactor tag count

Author: patrikelfstrom

server/catalog/npm-sync-service.ts

@@ -15,7 +15,9 @@ import {
   createReplaceTagStatements,
   createPackageUrl,
   createRebuildPackageSearchStatements,
+  createRebuildTagStatsStatements,
   createRefreshPackageSearchStatementsForPackages,
+  createRefreshTagStatsStatements,
   createUpsertPackageStatement,
   type CatalogPackageRecord,
 } from "./package-store"
@@ -26,6 +28,7 @@ import {
   hasUnpublishedRegistryMarker,
   isSecurityHoldingPackage,
 } from "./package-removal"
+import { normalizeTagValue } from "./tag-normalization"
 
 const DEFAULT_DOWNLOADS_PERIOD = "last-month"
 const DEFAULT_GITHUB_BATCH_SIZE = 50
@@ -210,6 +213,10 @@ export async function syncNpmCatalog(
     const batch = packageMetadata.slice(index, index + DEFAULT_WRITE_BATCH_SIZE)
     writeTasks.push(
       writeQueue.add(async () => {
+        const batchPackageNames = batch.map((item) => item.packageName)
+        const affectedTagIds = new Set(
+          await loadExistingTagIdsForPackages(client, batchPackageNames)
+        )
         const statements = batch.flatMap((item) => {
           const repositoryRef = parseGitHubRepositoryRef(item.repositoryUrl)
           const enrichmentOutcome = item.isRemoved
@@ -255,6 +262,9 @@ export async function syncNpmCatalog(
               item.packageTags
             ),
           ]
+          collectNormalizedTagIds(item.packageTags).forEach((tagId) => {
+            affectedTagIds.add(tagId)
+          })
 
           if (enrichmentOutcome.kind === "replace") {
             statements.push(
@@ -264,17 +274,24 @@ export async function syncNpmCatalog(
                 enrichmentOutcome.repositoryTags
               )
             )
+            collectNormalizedTagIds(enrichmentOutcome.repositoryTags).forEach(
+              (tagId) => {
+                affectedTagIds.add(tagId)
+              }
+            )
           }
 
           return statements
         })
         statements.push(
-          ...createRefreshPackageSearchStatementsForPackages(
-            batch.map((item) => item.packageName)
-          )
+          ...createRefreshPackageSearchStatementsForPackages(batchPackageNames)
         )
 
         await client.batch(statements, "write")
+        await client.batch(
+          createRefreshTagStatsStatements(Array.from(affectedTagIds)),
+          "write"
+        )
         storedCount += batch.length
 
         const now = Date.now()
@@ -298,6 +315,8 @@ export async function syncNpmCatalog(
   await pruneOrphanedTags(client)
   options.onProgress?.("Rebuilding package search index.")
   await client.batch(createRebuildPackageSearchStatements(), "write")
+  options.onProgress?.("Rebuilding tag stats.")
+  await client.batch(createRebuildTagStatsStatements(), "write")
 
   options.onProgress?.(`Stored ${packageMetadata.length} npm packages total.`)
 
@@ -492,10 +511,55 @@ async function loadExistingRepositoryUrls(
   return repositoryUrls
 }
 
+async function loadExistingTagIdsForPackages(
+  client: CatalogDatabaseClient,
+  packageNames: string[]
+) {
+  const uniquePackageNames = Array.from(new Set(packageNames))
+
+  if (uniquePackageNames.length === 0) {
+    return [] as string[]
+  }
+
+  const placeholders = uniquePackageNames.map(() => "?").join(", ")
+  const result = await client.execute({
+    sql: `
+      SELECT DISTINCT tag_id
+      FROM (
+        SELECT tag_id
+        FROM package_tags
+        WHERE package_name IN (${placeholders})
+        UNION ALL
+        SELECT tag_id
+        FROM repository_tags
+        WHERE package_name IN (${placeholders})
+      )
+    `,
+    args: [...uniquePackageNames, ...uniquePackageNames],
+  })
+
+  return result.rows.map((row) => String(row.tag_id))
+}
+
 function normalizeStoredUrl(value: unknown) {
   return typeof value === "string" ? value : null
 }
 
+function collectNormalizedTagIds(rawTags: string[]) {
+  return Array.from(
+    new Set(
+      rawTags.flatMap((rawTag) => {
+        const normalizedRawTag = normalizeOptionalString(rawTag)
+        const normalizedTagId = normalizedRawTag
+          ? normalizeTagValue(normalizedRawTag)
+          : undefined
+
+        return normalizedTagId ? [normalizedTagId] : []
+      })
+    )
+  )
+}
+
 function normalizeDownloadCountSelection(
   selection:
     | number
@@ -751,8 +815,9 @@ async function fetchNpmPackageMetadata(
     })
   ) {
     const npmViewResolves =
-      (await options.npmViewRunner?.(packageName)) ??
-      (await resolvePackageWithNpmView(packageName))
+      typeof options.npmViewRunner === "function"
+        ? await options.npmViewRunner(packageName)
+        : await resolvePackageWithNpmView(packageName)
 
     if (npmViewResolves === null) {
       options.onProgress?.(

server/catalog/package-store.ts

@@ -1,6 +1,7 @@
 import type { InStatement } from "@libsql/client"
 
 import type { CatalogDatabaseClient } from "./database"
+import { createVisiblePackageSql } from "./package-removal"
 import { createTagLabel, normalizeTagValue } from "./tag-normalization"
 
 export type CatalogPackageRecord = {
@@ -43,6 +44,7 @@ export async function upsertPackage(
     [
       createUpsertPackageStatement(packageRecord),
       ...createRefreshPackageSearchStatements(packageRecord.packageName),
+      ...createRebuildTagStatsStatements(),
     ],
     "write"
   )
@@ -58,6 +60,7 @@ export async function replacePackageTags(
     [
       ...createReplaceTagStatements(tableName, packageName, rawTags),
       ...createRefreshPackageSearchStatements(packageName),
+      ...createRebuildTagStatsStatements(),
     ],
     "write"
   )
@@ -229,6 +232,93 @@ export function createRebuildPackageSearchStatements(): InStatement[] {
   ]
 }
 
+export function createRebuildTagStatsStatements(): InStatement[] {
+  return [
+    {
+      sql: `DELETE FROM tag_stats`,
+    },
+    {
+      sql: `
+        INSERT INTO tag_stats (tag_id, package_count)
+        WITH visible_tag_pairs AS (
+          SELECT DISTINCT pt.package_name, pt.tag_id
+          FROM package_tags pt
+          JOIN packages p ON p.package_name = pt.package_name
+          WHERE ${createVisiblePackageSql("p")}
+          UNION
+          SELECT DISTINCT rt.package_name, rt.tag_id
+          FROM repository_tags rt
+          JOIN packages p ON p.package_name = rt.package_name
+          WHERE ${createVisiblePackageSql("p")}
+        )
+        SELECT tag_id, COUNT(*) AS package_count
+        FROM visible_tag_pairs
+        GROUP BY tag_id
+      `,
+    },
+    ...createBumpTagsVersionStatements(),
+  ]
+}
+
+export function createRefreshTagStatsStatements(
+  tagIds: string[]
+): InStatement[] {
+  const uniqueTagIds = Array.from(new Set(tagIds))
+
+  if (uniqueTagIds.length === 0) {
+    return []
+  }
+
+  const placeholders = uniqueTagIds.map(() => "?").join(", ")
+  const valuesClause = uniqueTagIds.map(() => "(?)").join(", ")
+
+  return [
+    {
+      sql: `DELETE FROM tag_stats WHERE tag_id IN (${placeholders})`,
+      args: uniqueTagIds,
+    },
+    {
+      sql: `
+        WITH affected_tags(tag_id) AS (
+          VALUES ${valuesClause}
+        ),
+        visible_tag_pairs AS (
+          SELECT DISTINCT pt.package_name, pt.tag_id
+          FROM package_tags pt
+          JOIN affected_tags at ON at.tag_id = pt.tag_id
+          JOIN packages p ON p.package_name = pt.package_name
+          WHERE ${createVisiblePackageSql("p")}
+          UNION
+          SELECT DISTINCT rt.package_name, rt.tag_id
+          FROM repository_tags rt
+          JOIN affected_tags at ON at.tag_id = rt.tag_id
+          JOIN packages p ON p.package_name = rt.package_name
+          WHERE ${createVisiblePackageSql("p")}
+        )
+        INSERT INTO tag_stats (tag_id, package_count)
+        SELECT tag_id, COUNT(*) AS package_count
+        FROM visible_tag_pairs
+        GROUP BY tag_id
+      `,
+      args: uniqueTagIds,
+    },
+    ...createBumpTagsVersionStatements(),
+  ]
+}
+
+export function createBumpTagsVersionStatements(): InStatement[] {
+  return [
+    {
+      sql: `
+        INSERT INTO catalog_meta (meta_key, meta_value)
+        VALUES ('tags_version', '1')
+        ON CONFLICT(meta_key) DO UPDATE SET
+          meta_value = CAST(COALESCE(CAST(meta_value AS INTEGER), 0) + 1 AS TEXT)
+      `,
+    },
+  ]
+}
+
 const PACKAGE_SEARCH_TEXT_SQL = `
   LOWER(
     TRIM(

server/catalog/read-service.ts

@@ -112,25 +112,13 @@ export async function listCatalogTags(
 
   const result = await client.execute({
     sql: `
-      WITH visible_tag_pairs AS (
-        SELECT DISTINCT pt.package_name, pt.tag_id
-        FROM package_tags pt
-        JOIN packages p ON p.package_name = pt.package_name
-        WHERE ${createVisiblePackageSql("p")}
-        UNION
-        SELECT DISTINCT rt.package_name, rt.tag_id
-        FROM repository_tags rt
-        JOIN packages p ON p.package_name = rt.package_name
-        WHERE ${createVisiblePackageSql("p")}
-      )
       SELECT
-        vtp.tag_id,
+        ts.tag_id,
         t.label,
-        COUNT(*) AS package_count
-      FROM visible_tag_pairs vtp
-      JOIN tags t ON t.tag_id = vtp.tag_id
-      GROUP BY vtp.tag_id, t.label
-      ORDER BY package_count DESC, t.tag_id ASC
+        ts.package_count
+      FROM tag_stats ts
+      JOIN tags t ON t.tag_id = ts.tag_id
+      ORDER BY ts.package_count DESC, t.label COLLATE NOCASE ASC, ts.tag_id ASC
       LIMIT 500
     `,
   })

server/catalog/schema.ts

@@ -1,7 +1,10 @@
 import type { InStatement } from "@libsql/client"
 
 import type { CatalogDatabaseClient } from "./database"
-import { createRebuildPackageSearchStatements } from "./package-store"
+import {
+  createRebuildPackageSearchStatements,
+  createRebuildTagStatsStatements,
+} from "./package-store"
 
 const EXPECTED_TABLE_COLUMNS = {
   packages: [
@@ -20,6 +23,8 @@ const EXPECTED_TABLE_COLUMNS = {
   repository_tags: ["package_name", "tag_id", "raw_value"],
   tags: ["tag_id", "label"],
   tag_aliases: ["alias", "tag_id"],
+  tag_stats: ["tag_id", "package_count"],
+  catalog_meta: ["meta_key", "meta_value"],
 } as const
 
 const schemaStatements: InStatement[] = [
@@ -70,6 +75,24 @@ const schemaStatements: InStatement[] = [
       FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
     )
   `,
+  `
+    CREATE TABLE IF NOT EXISTS tag_stats (
+      tag_id TEXT PRIMARY KEY,
+      package_count INTEGER NOT NULL,
+      FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
+    )
+  `,
+  `
+    CREATE TABLE IF NOT EXISTS catalog_meta (
+      meta_key TEXT PRIMARY KEY,
+      meta_value TEXT NOT NULL
+    )
+  `,
+  `
+    INSERT INTO catalog_meta (meta_key, meta_value)
+    VALUES ('tags_version', '0')
+    ON CONFLICT(meta_key) DO NOTHING
+  `,
   `
     CREATE VIRTUAL TABLE IF NOT EXISTS package_search_fts
     USING fts5(
@@ -139,6 +162,8 @@ const destructiveResetStatements: InStatement[] = [
   "DROP TABLE IF EXISTS package_tags",
   "DROP TABLE IF EXISTS package_search_fts",
   "DROP TABLE IF EXISTS tag_aliases",
+  "DROP TABLE IF EXISTS tag_stats",
+  "DROP TABLE IF EXISTS catalog_meta",
   "DROP TABLE IF EXISTS tags",
   "DROP TABLE IF EXISTS packages",
   "DROP TABLE IF EXISTS raw_ecosystems_packages",
@@ -153,12 +178,14 @@ export async function ensureCatalogSchema(client: CatalogDatabaseClient) {
   await applyStatements(client, obsoleteIndexDropStatements)
   await applyStatements(client, schemaStatements)
   await applyStatements(client, createRebuildPackageSearchStatements())
+  await applyStatements(client, createRebuildTagStatsStatements())
 }
 
 export async function resetCatalogSchema(client: CatalogDatabaseClient) {
   await applyStatements(client, destructiveResetStatements)
   await applyStatements(client, schemaStatements)
   await applyStatements(client, createRebuildPackageSearchStatements())
+  await applyStatements(client, createRebuildTagStatsStatements())
 }
 
 async function applyStatements(