diff --git a/skills/site-validation-loop/SKILL.md b/skills/site-validation-loop/SKILL.md index 7731a13..49ab909 100644 --- a/skills/site-validation-loop/SKILL.md +++ b/skills/site-validation-loop/SKILL.md @@ -18,6 +18,13 @@ tags: (`purview-build.json` → `bun run ci:build`; deploy.yml runs `just validate` first). Treat a green `just validate` as the definition of done. +The same chain deliberately runs twice on one change: once in the PR gate (the merge condition, via +the shared Purview.Build pipeline) and once on `main` in `deploy.yml` (the publish gate). They are +not shared because `main` is a different commit on a different runner — the change is validated +before it merges, but what is published is validated again, from its own fresh data sync, before it +is published. Anything that removes that second run has to be a decision about what the deployment +no longer has to prove; it is not a free saving. + ## The chain, in order | Step | Command | Catches | @@ -83,6 +90,13 @@ each declared repository file with the docs aggregator's `fetchRawFile`, writes caches it under `.cache/assets/`. A declared asset that cannot be resolved fails the sync, so a missing schema never silently ships a broken URL. +The same mode governs the **skills mirror** (`skills:` in a record): the skill set and its files are +read from the project's documentation ref and cached under `.cache/skills///`, each +with a `.ref` marker naming the ref it was fetched from. `live` re-reads everything; `auto` +reuses the cache; and `cache`/`fixture` make no request at all — they mirror the skills the cache +holds for the resolved ref and fail loudly when a declared skill was never fetched, because those +modes promise a sync that touches no network. + `DATA_FALLBACK_TO_FIXTURES=false` makes `auto` fail loudly instead of silently using fixtures. Set `GITHUB_TOKEN` (or `gh auth token`) to avoid unauthenticated rate limits. @@ -115,6 +129,6 @@ just build && just check-links # Catalogue drift in isolation just check-projects -# Release-data problems without network +# Release-data problems without network (the skills mirror reads the cache too) DATA_MODE=fixture just build && just test-dist ``` diff --git a/src/scripts/sync-skills.ts b/src/scripts/sync-skills.ts index a70a3be..d4c0b98 100644 --- a/src/scripts/sync-skills.ts +++ b/src/scripts/sync-skills.ts @@ -4,6 +4,7 @@ import type { DataMode } from './sync-assets'; import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; +import { mapWithConcurrency } from '../src/lib/concurrency'; import { fetchRawFileContent, listRepositoryFiles } from '../src/lib/docs/aggregate'; import { resolveDocChannels } from '../src/lib/docs/channels'; import { loadProjects } from '../src/lib/manifest/load'; @@ -18,7 +19,8 @@ import { declaresSkills, skillSourcePath, SKILLS_PUBLIC_DIR } from '../src/lib/s * prerelease ref — so the skills on purview.dev always match the release the * documentation describes. The catalogue only says "this project publishes * skills" (`skills: true`); the list is discovered on every build, which is why - * a new skill needs no site change. + * a new skill needs no site change. Modes that promise no network (`cache`, + * `fixture`) read the cached skill set instead — see `planSkills`. */ export const SKILLS_CACHE_DIR = resolve('.cache/skills'); @@ -89,29 +91,202 @@ function publishCached(cache: string, target: string): string[] { return files; } -async function fetchAndPublish(skill: SkillTarget, tree: string[]): Promise { - const written: string[] = []; - const cache = resolve(SKILLS_CACHE_DIR, skill.projectId, skill.name); - const target = resolve(SKILLS_PUBLIC_DIR, skill.projectId, skill.name); - for (const path of tree.filter((entry) => entry.startsWith(`${skill.sourcePath}/`))) { - const content = await fetchRawFileContent(skill.repository, skill.ref, path); - if (content === null) { +/** How many skill files the mirror fetches at once; a live sync is otherwise latency-bound. */ +const SKILL_FETCH_CONCURRENCY = 8; + +/** A skill the mirror has to place, and the work placing it needs. */ +interface PlannedSkill { + skill: SkillTarget; + /** Where the skill is cached, served from, and marked as fetched. */ + cache: string; + target: string; + marker: string; + /** Relative paths copied out of an up-to-date cache; null when the skill is re-read. */ + cachedFiles: string[] | null; + /** Repository paths to fetch for a skill the cache cannot serve. */ + sources: string[]; + /** Relative paths written from `sources`, in the order they were fetched. */ + files: string[]; +} + +/** One repository file to fetch for one planned skill. */ +interface SkillSource { + plan: PlannedSkill; + path: string; + relativePath: string; +} + +/** + * Records the ref a project's skills were last synced for. A skill name is a + * lowercase slug, so a leading dot cannot collide with one — and it is a file, + * not a directory, so the cached-skill scan ignores it. + */ +const PROJECT_REF_FILE = '.ref'; + +/** + * The skills a project's cache holds for a ref. + * + * `cache` and `fixture` promise a sync that touches no network, so in those + * modes the cache — not the remote tree — says which skills exist. The `.ref` + * marker written beside each cached skill names the ref it was fetched from, so + * an entry fetched for one release is not offered for another. A stray file or + * directory beside them is not a skill. + * + * `root` is injectable so the resolution is testable against a temp directory. + */ +export function cachedSkills( + project: ResolvedProject, + ref: string, + root = SKILLS_CACHE_DIR, +): SkillTarget[] { + const directory = resolve(root, project.id); + if (!existsSync(directory)) { + return []; + } + const targets: SkillTarget[] = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const marker = resolve(directory, `${entry.name}.ref`); + if (!existsSync(marker) || readFileSync(marker, 'utf8').trim() !== ref) { continue; } - const relativePath = path.slice(skill.sourcePath.length + 1); - write(join(cache, ...relativePath.split('/')), content); - write(join(target, ...relativePath.split('/')), content); - written.push(relativePath); + targets.push({ + projectId: project.id, + repository: project.repository, + name: entry.name, + sourcePath: skillSourcePath({ name: entry.name }), + ref, + }); + } + return targets.toSorted((a, b) => a.name.localeCompare(b.name)); +} + +/** + * The ref a project's skills were last synced for, or null when the cache has + * never held them. A project that publishes no skills has nothing in the cache + * either, so this is what tells "no skills" apart from "never fetched". + */ +export function cachedSkillsRef(project: ResolvedProject, root = SKILLS_CACHE_DIR): string | null { + const marker = resolve(root, project.id, PROJECT_REF_FILE); + return existsSync(marker) ? readFileSync(marker, 'utf8').trim() : null; +} + +/** + * The skills to mirror for one project: the tree's when it is in hand, the + * cache's when the mode forbids a request. + * + * The tree is authoritative whenever it is available — `skills: true` means + * "publish every skill under `skills/`", so a skill added upstream appears with + * no site change. Without a tree the catalogue's declaration has to be satisfied + * by the cache, and a cache that was never synced for this ref, or that is + * missing a declared skill, is an error rather than a silent omission. + */ +function skillsForRef(project: ResolvedProject, ref: string, tree: string[] | null): SkillTarget[] { + if (tree) { + return discoverSkills(project, tree, ref); + } + + const syncedRef = cachedSkillsRef(project); + if (syncedRef !== ref) { + throw new Error( + `Skills for "${project.id}" have not been synced for ref "${ref}"` + + (syncedRef === null ? '' : ` (the cache holds "${syncedRef}")`) + + '. Run `just live-data-sync` once so they can be fetched.', + ); + } + + const cached = new Map(cachedSkills(project, ref).map((skill) => [skill.name, skill])); + for (const skill of project.skills) { + if (!cached.has(skill.name)) { + throw new Error( + `Skill ${project.id}/${skill.name} is not cached for ref "${ref}". ` + + 'Run `just live-data-sync` once so it can be fetched.', + ); + } + cached.set(skill.name, { + projectId: project.id, + repository: project.repository, + name: skill.name, + sourcePath: skillSourcePath(skill), + ref, + }); } - return written; + return [...cached.values()].toSorted((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Plan every skill the catalogue declares: the ref it is read at, whether the + * cache can serve it, and — when it cannot — which files have to be fetched. + * + * Planning is also what decides whether the repository tree is needed at all, + * which is the difference between a sync that costs one request per project and + * one that costs none: a project whose declared skills are all cached and that + * asks for no discovery has nothing to look up. + */ +async function planSkills( + projects: readonly ResolvedProject[], + mode: DataMode, +): Promise { + const releaseData = getReleaseIndex().data; + const plans: PlannedSkill[] = []; + + for (const project of projects) { + const channel = resolveDocChannels(project, releaseData).channels[0]; + const ref = channel?.ref ?? 'main'; + const cachedRef = (name: string): string | null => { + const marker = resolve(SKILLS_CACHE_DIR, project.id, `${name}.ref`); + return existsSync(marker) ? readFileSync(marker, 'utf8').trim() : null; + }; + const missing = project.skills.find((skill) => cachedRef(skill.name) !== ref)?.name ?? null; + + let tree: string[] | null; + if (mode === 'cache' || mode === 'fixture') { + tree = null; // this mode never makes a request + } else if (mode === 'auto' && !project.discoverSkills && missing === null) { + tree = []; // nothing to fetch and nothing to discover, so nothing to list + } else { + tree = await listRepositoryFiles(project.repository, ref); + } + if (tree) { + // The project-level marker is what lets a later offline sync tell "this + // ref publishes no skills" from "these skills were never fetched". + write(resolve(SKILLS_CACHE_DIR, project.id, PROJECT_REF_FILE), ref); + } + + for (const skill of skillsForRef(project, ref, tree)) { + const cached = mode !== 'live' && cachedRef(skill.name) === ref; + const cache = resolve(SKILLS_CACHE_DIR, project.id, skill.name); + const target = resolve(SKILLS_PUBLIC_DIR, project.id, skill.name); + plans.push({ + skill, + cache, + target, + marker: resolve(SKILLS_CACHE_DIR, project.id, `${skill.name}.ref`), + cachedFiles: cached ? publishCached(cache, target) : null, + sources: cached + ? [] + : (tree ?? []).filter((path) => path.startsWith(`${skill.sourcePath}/`)), + files: [], + }); + } + } + + return plans; } /** * Mirror every project's skills. The cache records the ref it was fetched from, * so a new release invalidates it automatically; `live` always re-reads. + * + * The files are fetched eight at a time (`mapWithConcurrency`) because a live + * sync would otherwise be one round trip per file per skill, but everything + * observable is written in plan order — the content of `public/skills`, the + * markers and the report — so the result is identical to the serial version and + * reproducible run to run. */ export async function syncSkills(mode: DataMode): Promise { - const releaseData = getReleaseIndex().data; const projects = loadProjects().filter( (project) => project.status !== 'archived' && declaresSkills(project), ); @@ -121,40 +296,42 @@ export async function syncSkills(mode: DataMode): Promise { } rmSync(SKILLS_PUBLIC_DIR, { recursive: true, force: true }); - let mirrored = 0; - for (const project of projects) { - const channel = resolveDocChannels(project, releaseData).channels[0]; - const ref = channel?.ref ?? 'main'; - const tree = await listRepositoryFiles(project.repository, ref); + const plans = await planSkills(projects, mode); - for (const skill of discoverSkills(project, tree, ref)) { - const cache = resolve(SKILLS_CACHE_DIR, project.id, skill.name); - const marker = resolve(SKILLS_CACHE_DIR, project.id, `${skill.name}.ref`); - const cached = existsSync(marker) && readFileSync(marker, 'utf8').trim() === ref; - const target = resolve(SKILLS_PUBLIC_DIR, project.id, skill.name); - - let files: string[]; - if (mode !== 'live' && cached) { - files = publishCached(cache, target); - } else if (mode === 'cache' || mode === 'fixture') { - throw new Error( - `Skill ${project.id}/${skill.name} is not cached for ref "${ref}". ` + - 'Run `just live-data-sync` once so it can be fetched.', - ); - } else { - files = await fetchAndPublish(skill, tree); - } + const sources: SkillSource[] = plans.flatMap((plan) => + plan.sources.map((path) => ({ + plan, + path, + relativePath: path.slice(plan.skill.sourcePath.length + 1), + })), + ); + const contents = await mapWithConcurrency(sources, SKILL_FETCH_CONCURRENCY, (source) => + fetchRawFileContent(source.plan.skill.repository, source.plan.skill.ref, source.path), + ); + sources.forEach((source, index) => { + const content = contents[index]; + if (typeof content !== 'string') { + return; + } + write(join(source.plan.cache, ...source.relativePath.split('/')), content); + write(join(source.plan.target, ...source.relativePath.split('/')), content); + source.plan.files.push(source.relativePath); + }); - if (files.length === 0) { - throw new Error( - `Skill not found: ${skill.repository}/${skill.sourcePath}@${ref} ` + - `(project "${project.id}"). Publish the skill at that path or correct the manifest.`, - ); - } - write(marker, ref); - mirrored += 1; - console.log(` skill ${project.id}/${skill.name}@${ref}: ${files.length} file(s)`); + let mirrored = 0; + for (const plan of plans) { + const files = plan.cachedFiles ?? plan.files; + if (files.length === 0) { + throw new Error( + `Skill not found: ${plan.skill.repository}/${plan.skill.sourcePath}@${plan.skill.ref} ` + + `(project "${plan.skill.projectId}"). Publish the skill at that path or correct the manifest.`, + ); } + write(plan.marker, plan.skill.ref); + mirrored += 1; + console.log( + ` skill ${plan.skill.projectId}/${plan.skill.name}@${plan.skill.ref}: ${files.length} file(s)`, + ); } console.log(`Skills mirrored: ${mirrored} skill(s).`); return mirrored; diff --git a/src/src/lib/docs/aggregate.ts b/src/src/lib/docs/aggregate.ts index 1ceccc7..65062d2 100644 --- a/src/src/lib/docs/aggregate.ts +++ b/src/src/lib/docs/aggregate.ts @@ -271,11 +271,33 @@ type SourceFileReader = (path: string) => Promise; */ const DOC_FETCH_CONCURRENCY = 8; -export async function listRepositoryFiles(repository: string, branch: string): Promise { - const tree = (await githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`)) as { - tree?: { path?: string }[]; - }; - return (tree.tree ?? []).map((entry) => entry.path ?? ''); +/** + * The repository tree, memoized for the process. + * + * One `just data-sync` lists the same tree twice for most projects — once for + * the documentation channels and once for the skills mirror — and every listing + * is a REST call. The cache is keyed by repository and ref, and a failed listing + * is dropped again so a transient error is retried rather than remembered. + */ +const repositoryTrees = new Map>(); + +export function listRepositoryFiles(repository: string, branch: string): Promise { + const key = `${repository}@${branch}`; + const cached = repositoryTrees.get(key); + if (cached) { + return cached; + } + const request = githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`) + .then((body) => { + const tree = body as { tree?: { path?: string }[] }; + return (tree.tree ?? []).map((entry) => entry.path ?? ''); + }) + .catch((error: unknown) => { + repositoryTrees.delete(key); + throw error; + }); + repositoryTrees.set(key, request); + return request; } async function cloneWiki( diff --git a/src/tests/unit/skills.test.ts b/src/tests/unit/skills.test.ts index e5b6985..8163e7c 100644 --- a/src/tests/unit/skills.test.ts +++ b/src/tests/unit/skills.test.ts @@ -3,7 +3,8 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { discoverSkills } from '../../scripts/sync-skills'; +import { cachedSkills, cachedSkillsRef, discoverSkills } from '../../scripts/sync-skills'; +import { loadProjects } from '../../src/lib/manifest/load'; import { manifestSchema } from '../../src/lib/manifest/schema'; import { listMirroredSkillFiles, @@ -190,4 +191,56 @@ describe('skills in the manifest', () => { }); expect(invalid.success).toBe(false); }); + describe('cached skill set', () => { + test('offers the skills cached for the requested ref, and only those', () => { + const project = loadProjects().find((candidate) => candidate.id === PROJECT); + if (!project) { + throw new Error(`the ${PROJECT} catalogue record is missing`); + } + const root = mkdtempSync(join(tmpdir(), 'purview-skill-cache-')); + try { + mkdirSync(join(root, PROJECT, 'pm-park'), { recursive: true }); + writeFileSync(join(root, PROJECT, 'pm-park.ref'), 'main\n'); + mkdirSync(join(root, PROJECT, 'pm-review'), { recursive: true }); + writeFileSync(join(root, PROJECT, 'pm-review.ref'), 'v3.0.0'); + // A file beside the skills is not a skill, and an entry cached for another + // ref is not offered for this one. + writeFileSync(join(root, PROJECT, 'notes.txt'), 'x'); + + expect(cachedSkills(project, 'main', root).map((skill) => skill.name)).toEqual(['pm-park']); + expect(cachedSkills(project, 'v3.0.0', root).map((skill) => skill.name)).toEqual([ + 'pm-review', + ]); + expect(cachedSkills(project, 'main', root)[0]).toEqual({ + projectId: PROJECT, + repository: REPOSITORY, + name: 'pm-park', + sourcePath: 'skills/pm-park', + ref: 'main', + }); + expect(cachedSkills(project, 'v9.9.9', root)).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test('a project with no cache directory has no cached skills', () => { + const project = loadProjects().find((candidate) => candidate.id === PROJECT); + if (!project) { + throw new Error(`the ${PROJECT} catalogue record is missing`); + } + const root = mkdtempSync(join(tmpdir(), 'purview-skill-cache-')); + try { + expect(cachedSkills(project, 'main', root)).toEqual([]); + // Nothing has ever been synced for it, which is not the same as "it has + // no skills" — `skillsForRef` tells those apart by this marker. + expect(cachedSkillsRef(project, root)).toBeNull(); + mkdirSync(join(root, PROJECT), { recursive: true }); + writeFileSync(join(root, PROJECT, '.ref'), 'main\n'); + expect(cachedSkillsRef(project, root)).toBe('main'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + }); });