From ce36d6980fdcd5c9c5b26215563e50b50799c8e5 Mon Sep 17 00:00:00 2001 From: Kieron Lanning Date: Mon, 16 Nov 2026 13:18:46 +0000 Subject: [PATCH] perf(sync): mirror skills concurrently and keep the offline modes offline Follow-up to the concurrent documentation fetch: syncSkills fetched one file at a time, per skill, for 108 skills (~160 requests). It now plans the work first - which ref, cached or not, which files - fetches the flattened list eight at a time, and only then writes, marks and reports, in plan order, so the mirror and the log are identical to the serial result. cache and fixture promised a sync that touches no network but listed every repository tree anyway before requiring the skills cache. They now take the skill set from the cache (cachedSkills): the per-skill .ref markers say what was fetched and at which ref, and a per-project marker records the ref the project was last synced for. That marker is what tells an empty skill set apart from one that was never fetched, so an offline sync neither makes a request nor goes quiet when the cache is cold. auto no longer lists a tree for a project whose declared skills are all cached and that asks for no discovery - there is nothing to look up. listRepositoryFiles is memoized per repository and ref for the process, so a sync stops listing the same tree twice (once for the documentation channels, once for the skills mirror). Measured on one machine: a live sync goes from 140.0s to 74.6s (210.6s before the first concurrent-fetch change), an auto sync with a warm cache to 7.1s, and a cache-mode sync to 1s. Unit tests cover the cached skill set (ref filtering, stray files, an absent cache directory) and the project ref marker. --- skills/site-validation-loop/SKILL.md | 16 +- src/scripts/sync-skills.ts | 267 ++++++++++++++++++++++----- src/src/lib/docs/aggregate.ts | 32 +++- src/tests/unit/skills.test.ts | 55 +++++- 4 files changed, 318 insertions(+), 52 deletions(-) diff --git a/skills/site-validation-loop/SKILL.md b/skills/site-validation-loop/SKILL.md index 7731a13..49ab909 100644 --- a/skills/site-validation-loop/SKILL.md +++ b/skills/site-validation-loop/SKILL.md @@ -18,6 +18,13 @@ tags: (`purview-build.json` → `bun run ci:build`; deploy.yml runs `just validate` first). Treat a green `just validate` as the definition of done. +The same chain deliberately runs twice on one change: once in the PR gate (the merge condition, via +the shared Purview.Build pipeline) and once on `main` in `deploy.yml` (the publish gate). They are +not shared because `main` is a different commit on a different runner — the change is validated +before it merges, but what is published is validated again, from its own fresh data sync, before it +is published. Anything that removes that second run has to be a decision about what the deployment +no longer has to prove; it is not a free saving. + ## The chain, in order | Step | Command | Catches | @@ -83,6 +90,13 @@ each declared repository file with the docs aggregator's `fetchRawFile`, writes caches it under `.cache/assets/`. A declared asset that cannot be resolved fails the sync, so a missing schema never silently ships a broken URL. +The same mode governs the **skills mirror** (`skills:` in a record): the skill set and its files are +read from the project's documentation ref and cached under `.cache/skills///`, each +with a `.ref` marker naming the ref it was fetched from. `live` re-reads everything; `auto` +reuses the cache; and `cache`/`fixture` make no request at all — they mirror the skills the cache +holds for the resolved ref and fail loudly when a declared skill was never fetched, because those +modes promise a sync that touches no network. + `DATA_FALLBACK_TO_FIXTURES=false` makes `auto` fail loudly instead of silently using fixtures. Set `GITHUB_TOKEN` (or `gh auth token`) to avoid unauthenticated rate limits. @@ -115,6 +129,6 @@ just build && just check-links # Catalogue drift in isolation just check-projects -# Release-data problems without network +# Release-data problems without network (the skills mirror reads the cache too) DATA_MODE=fixture just build && just test-dist ``` diff --git a/src/scripts/sync-skills.ts b/src/scripts/sync-skills.ts index a70a3be..d4c0b98 100644 --- a/src/scripts/sync-skills.ts +++ b/src/scripts/sync-skills.ts @@ -4,6 +4,7 @@ import type { DataMode } from './sync-assets'; import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; +import { mapWithConcurrency } from '../src/lib/concurrency'; import { fetchRawFileContent, listRepositoryFiles } from '../src/lib/docs/aggregate'; import { resolveDocChannels } from '../src/lib/docs/channels'; import { loadProjects } from '../src/lib/manifest/load'; @@ -18,7 +19,8 @@ import { declaresSkills, skillSourcePath, SKILLS_PUBLIC_DIR } from '../src/lib/s * prerelease ref — so the skills on purview.dev always match the release the * documentation describes. The catalogue only says "this project publishes * skills" (`skills: true`); the list is discovered on every build, which is why - * a new skill needs no site change. + * a new skill needs no site change. Modes that promise no network (`cache`, + * `fixture`) read the cached skill set instead — see `planSkills`. */ export const SKILLS_CACHE_DIR = resolve('.cache/skills'); @@ -89,29 +91,202 @@ function publishCached(cache: string, target: string): string[] { return files; } -async function fetchAndPublish(skill: SkillTarget, tree: string[]): Promise { - const written: string[] = []; - const cache = resolve(SKILLS_CACHE_DIR, skill.projectId, skill.name); - const target = resolve(SKILLS_PUBLIC_DIR, skill.projectId, skill.name); - for (const path of tree.filter((entry) => entry.startsWith(`${skill.sourcePath}/`))) { - const content = await fetchRawFileContent(skill.repository, skill.ref, path); - if (content === null) { +/** How many skill files the mirror fetches at once; a live sync is otherwise latency-bound. */ +const SKILL_FETCH_CONCURRENCY = 8; + +/** A skill the mirror has to place, and the work placing it needs. */ +interface PlannedSkill { + skill: SkillTarget; + /** Where the skill is cached, served from, and marked as fetched. */ + cache: string; + target: string; + marker: string; + /** Relative paths copied out of an up-to-date cache; null when the skill is re-read. */ + cachedFiles: string[] | null; + /** Repository paths to fetch for a skill the cache cannot serve. */ + sources: string[]; + /** Relative paths written from `sources`, in the order they were fetched. */ + files: string[]; +} + +/** One repository file to fetch for one planned skill. */ +interface SkillSource { + plan: PlannedSkill; + path: string; + relativePath: string; +} + +/** + * Records the ref a project's skills were last synced for. A skill name is a + * lowercase slug, so a leading dot cannot collide with one — and it is a file, + * not a directory, so the cached-skill scan ignores it. + */ +const PROJECT_REF_FILE = '.ref'; + +/** + * The skills a project's cache holds for a ref. + * + * `cache` and `fixture` promise a sync that touches no network, so in those + * modes the cache — not the remote tree — says which skills exist. The `.ref` + * marker written beside each cached skill names the ref it was fetched from, so + * an entry fetched for one release is not offered for another. A stray file or + * directory beside them is not a skill. + * + * `root` is injectable so the resolution is testable against a temp directory. + */ +export function cachedSkills( + project: ResolvedProject, + ref: string, + root = SKILLS_CACHE_DIR, +): SkillTarget[] { + const directory = resolve(root, project.id); + if (!existsSync(directory)) { + return []; + } + const targets: SkillTarget[] = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const marker = resolve(directory, `${entry.name}.ref`); + if (!existsSync(marker) || readFileSync(marker, 'utf8').trim() !== ref) { continue; } - const relativePath = path.slice(skill.sourcePath.length + 1); - write(join(cache, ...relativePath.split('/')), content); - write(join(target, ...relativePath.split('/')), content); - written.push(relativePath); + targets.push({ + projectId: project.id, + repository: project.repository, + name: entry.name, + sourcePath: skillSourcePath({ name: entry.name }), + ref, + }); + } + return targets.toSorted((a, b) => a.name.localeCompare(b.name)); +} + +/** + * The ref a project's skills were last synced for, or null when the cache has + * never held them. A project that publishes no skills has nothing in the cache + * either, so this is what tells "no skills" apart from "never fetched". + */ +export function cachedSkillsRef(project: ResolvedProject, root = SKILLS_CACHE_DIR): string | null { + const marker = resolve(root, project.id, PROJECT_REF_FILE); + return existsSync(marker) ? readFileSync(marker, 'utf8').trim() : null; +} + +/** + * The skills to mirror for one project: the tree's when it is in hand, the + * cache's when the mode forbids a request. + * + * The tree is authoritative whenever it is available — `skills: true` means + * "publish every skill under `skills/`", so a skill added upstream appears with + * no site change. Without a tree the catalogue's declaration has to be satisfied + * by the cache, and a cache that was never synced for this ref, or that is + * missing a declared skill, is an error rather than a silent omission. + */ +function skillsForRef(project: ResolvedProject, ref: string, tree: string[] | null): SkillTarget[] { + if (tree) { + return discoverSkills(project, tree, ref); + } + + const syncedRef = cachedSkillsRef(project); + if (syncedRef !== ref) { + throw new Error( + `Skills for "${project.id}" have not been synced for ref "${ref}"` + + (syncedRef === null ? '' : ` (the cache holds "${syncedRef}")`) + + '. Run `just live-data-sync` once so they can be fetched.', + ); + } + + const cached = new Map(cachedSkills(project, ref).map((skill) => [skill.name, skill])); + for (const skill of project.skills) { + if (!cached.has(skill.name)) { + throw new Error( + `Skill ${project.id}/${skill.name} is not cached for ref "${ref}". ` + + 'Run `just live-data-sync` once so it can be fetched.', + ); + } + cached.set(skill.name, { + projectId: project.id, + repository: project.repository, + name: skill.name, + sourcePath: skillSourcePath(skill), + ref, + }); } - return written; + return [...cached.values()].toSorted((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Plan every skill the catalogue declares: the ref it is read at, whether the + * cache can serve it, and — when it cannot — which files have to be fetched. + * + * Planning is also what decides whether the repository tree is needed at all, + * which is the difference between a sync that costs one request per project and + * one that costs none: a project whose declared skills are all cached and that + * asks for no discovery has nothing to look up. + */ +async function planSkills( + projects: readonly ResolvedProject[], + mode: DataMode, +): Promise { + const releaseData = getReleaseIndex().data; + const plans: PlannedSkill[] = []; + + for (const project of projects) { + const channel = resolveDocChannels(project, releaseData).channels[0]; + const ref = channel?.ref ?? 'main'; + const cachedRef = (name: string): string | null => { + const marker = resolve(SKILLS_CACHE_DIR, project.id, `${name}.ref`); + return existsSync(marker) ? readFileSync(marker, 'utf8').trim() : null; + }; + const missing = project.skills.find((skill) => cachedRef(skill.name) !== ref)?.name ?? null; + + let tree: string[] | null; + if (mode === 'cache' || mode === 'fixture') { + tree = null; // this mode never makes a request + } else if (mode === 'auto' && !project.discoverSkills && missing === null) { + tree = []; // nothing to fetch and nothing to discover, so nothing to list + } else { + tree = await listRepositoryFiles(project.repository, ref); + } + if (tree) { + // The project-level marker is what lets a later offline sync tell "this + // ref publishes no skills" from "these skills were never fetched". + write(resolve(SKILLS_CACHE_DIR, project.id, PROJECT_REF_FILE), ref); + } + + for (const skill of skillsForRef(project, ref, tree)) { + const cached = mode !== 'live' && cachedRef(skill.name) === ref; + const cache = resolve(SKILLS_CACHE_DIR, project.id, skill.name); + const target = resolve(SKILLS_PUBLIC_DIR, project.id, skill.name); + plans.push({ + skill, + cache, + target, + marker: resolve(SKILLS_CACHE_DIR, project.id, `${skill.name}.ref`), + cachedFiles: cached ? publishCached(cache, target) : null, + sources: cached + ? [] + : (tree ?? []).filter((path) => path.startsWith(`${skill.sourcePath}/`)), + files: [], + }); + } + } + + return plans; } /** * Mirror every project's skills. The cache records the ref it was fetched from, * so a new release invalidates it automatically; `live` always re-reads. + * + * The files are fetched eight at a time (`mapWithConcurrency`) because a live + * sync would otherwise be one round trip per file per skill, but everything + * observable is written in plan order — the content of `public/skills`, the + * markers and the report — so the result is identical to the serial version and + * reproducible run to run. */ export async function syncSkills(mode: DataMode): Promise { - const releaseData = getReleaseIndex().data; const projects = loadProjects().filter( (project) => project.status !== 'archived' && declaresSkills(project), ); @@ -121,40 +296,42 @@ export async function syncSkills(mode: DataMode): Promise { } rmSync(SKILLS_PUBLIC_DIR, { recursive: true, force: true }); - let mirrored = 0; - for (const project of projects) { - const channel = resolveDocChannels(project, releaseData).channels[0]; - const ref = channel?.ref ?? 'main'; - const tree = await listRepositoryFiles(project.repository, ref); + const plans = await planSkills(projects, mode); - for (const skill of discoverSkills(project, tree, ref)) { - const cache = resolve(SKILLS_CACHE_DIR, project.id, skill.name); - const marker = resolve(SKILLS_CACHE_DIR, project.id, `${skill.name}.ref`); - const cached = existsSync(marker) && readFileSync(marker, 'utf8').trim() === ref; - const target = resolve(SKILLS_PUBLIC_DIR, project.id, skill.name); - - let files: string[]; - if (mode !== 'live' && cached) { - files = publishCached(cache, target); - } else if (mode === 'cache' || mode === 'fixture') { - throw new Error( - `Skill ${project.id}/${skill.name} is not cached for ref "${ref}". ` + - 'Run `just live-data-sync` once so it can be fetched.', - ); - } else { - files = await fetchAndPublish(skill, tree); - } + const sources: SkillSource[] = plans.flatMap((plan) => + plan.sources.map((path) => ({ + plan, + path, + relativePath: path.slice(plan.skill.sourcePath.length + 1), + })), + ); + const contents = await mapWithConcurrency(sources, SKILL_FETCH_CONCURRENCY, (source) => + fetchRawFileContent(source.plan.skill.repository, source.plan.skill.ref, source.path), + ); + sources.forEach((source, index) => { + const content = contents[index]; + if (typeof content !== 'string') { + return; + } + write(join(source.plan.cache, ...source.relativePath.split('/')), content); + write(join(source.plan.target, ...source.relativePath.split('/')), content); + source.plan.files.push(source.relativePath); + }); - if (files.length === 0) { - throw new Error( - `Skill not found: ${skill.repository}/${skill.sourcePath}@${ref} ` + - `(project "${project.id}"). Publish the skill at that path or correct the manifest.`, - ); - } - write(marker, ref); - mirrored += 1; - console.log(` skill ${project.id}/${skill.name}@${ref}: ${files.length} file(s)`); + let mirrored = 0; + for (const plan of plans) { + const files = plan.cachedFiles ?? plan.files; + if (files.length === 0) { + throw new Error( + `Skill not found: ${plan.skill.repository}/${plan.skill.sourcePath}@${plan.skill.ref} ` + + `(project "${plan.skill.projectId}"). Publish the skill at that path or correct the manifest.`, + ); } + write(plan.marker, plan.skill.ref); + mirrored += 1; + console.log( + ` skill ${plan.skill.projectId}/${plan.skill.name}@${plan.skill.ref}: ${files.length} file(s)`, + ); } console.log(`Skills mirrored: ${mirrored} skill(s).`); return mirrored; diff --git a/src/src/lib/docs/aggregate.ts b/src/src/lib/docs/aggregate.ts index 1ceccc7..65062d2 100644 --- a/src/src/lib/docs/aggregate.ts +++ b/src/src/lib/docs/aggregate.ts @@ -271,11 +271,33 @@ type SourceFileReader = (path: string) => Promise; */ const DOC_FETCH_CONCURRENCY = 8; -export async function listRepositoryFiles(repository: string, branch: string): Promise { - const tree = (await githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`)) as { - tree?: { path?: string }[]; - }; - return (tree.tree ?? []).map((entry) => entry.path ?? ''); +/** + * The repository tree, memoized for the process. + * + * One `just data-sync` lists the same tree twice for most projects — once for + * the documentation channels and once for the skills mirror — and every listing + * is a REST call. The cache is keyed by repository and ref, and a failed listing + * is dropped again so a transient error is retried rather than remembered. + */ +const repositoryTrees = new Map>(); + +export function listRepositoryFiles(repository: string, branch: string): Promise { + const key = `${repository}@${branch}`; + const cached = repositoryTrees.get(key); + if (cached) { + return cached; + } + const request = githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`) + .then((body) => { + const tree = body as { tree?: { path?: string }[] }; + return (tree.tree ?? []).map((entry) => entry.path ?? ''); + }) + .catch((error: unknown) => { + repositoryTrees.delete(key); + throw error; + }); + repositoryTrees.set(key, request); + return request; } async function cloneWiki( diff --git a/src/tests/unit/skills.test.ts b/src/tests/unit/skills.test.ts index e5b6985..8163e7c 100644 --- a/src/tests/unit/skills.test.ts +++ b/src/tests/unit/skills.test.ts @@ -3,7 +3,8 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { discoverSkills } from '../../scripts/sync-skills'; +import { cachedSkills, cachedSkillsRef, discoverSkills } from '../../scripts/sync-skills'; +import { loadProjects } from '../../src/lib/manifest/load'; import { manifestSchema } from '../../src/lib/manifest/schema'; import { listMirroredSkillFiles, @@ -190,4 +191,56 @@ describe('skills in the manifest', () => { }); expect(invalid.success).toBe(false); }); + describe('cached skill set', () => { + test('offers the skills cached for the requested ref, and only those', () => { + const project = loadProjects().find((candidate) => candidate.id === PROJECT); + if (!project) { + throw new Error(`the ${PROJECT} catalogue record is missing`); + } + const root = mkdtempSync(join(tmpdir(), 'purview-skill-cache-')); + try { + mkdirSync(join(root, PROJECT, 'pm-park'), { recursive: true }); + writeFileSync(join(root, PROJECT, 'pm-park.ref'), 'main\n'); + mkdirSync(join(root, PROJECT, 'pm-review'), { recursive: true }); + writeFileSync(join(root, PROJECT, 'pm-review.ref'), 'v3.0.0'); + // A file beside the skills is not a skill, and an entry cached for another + // ref is not offered for this one. + writeFileSync(join(root, PROJECT, 'notes.txt'), 'x'); + + expect(cachedSkills(project, 'main', root).map((skill) => skill.name)).toEqual(['pm-park']); + expect(cachedSkills(project, 'v3.0.0', root).map((skill) => skill.name)).toEqual([ + 'pm-review', + ]); + expect(cachedSkills(project, 'main', root)[0]).toEqual({ + projectId: PROJECT, + repository: REPOSITORY, + name: 'pm-park', + sourcePath: 'skills/pm-park', + ref: 'main', + }); + expect(cachedSkills(project, 'v9.9.9', root)).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('a project with no cache directory has no cached skills', () => { + const project = loadProjects().find((candidate) => candidate.id === PROJECT); + if (!project) { + throw new Error(`the ${PROJECT} catalogue record is missing`); + } + const root = mkdtempSync(join(tmpdir(), 'purview-skill-cache-')); + try { + expect(cachedSkills(project, 'main', root)).toEqual([]); + // Nothing has ever been synced for it, which is not the same as "it has + // no skills" — `skillsForRef` tells those apart by this marker. + expect(cachedSkillsRef(project, root)).toBeNull(); + mkdirSync(join(root, PROJECT), { recursive: true }); + writeFileSync(join(root, PROJECT, '.ref'), 'main\n'); + expect(cachedSkillsRef(project, root)).toBe('main'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + }); });