diff --git a/src/scripts/sync-assets.ts b/src/scripts/sync-assets.ts index 926a6b8..6a9fd08 100644 --- a/src/scripts/sync-assets.ts +++ b/src/scripts/sync-assets.ts @@ -3,14 +3,14 @@ import type { ResolvedProject } from '../src/lib/manifest/load'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; -import { fetchRawFile } from '../src/lib/docs/aggregate'; +import { fetchRawFileContent } from '../src/lib/docs/aggregate'; import { loadProjects } from '../src/lib/manifest/load'; /** * Mirror the repository files declared in a project's `assets` into `public/`, * so a stable purview.dev URL can serve an artefact that lives in a product * repository (for example a JSON Schema). Assets are read from the repository's - * `main` branch with the docs aggregator's `fetchRawFile`, cached under + * `main` branch with the docs aggregator's `fetchRawFileContent`, cached under * `.cache/assets/`, and backed by committed fixtures under `fixtures/assets/` * for the offline `fixture` mode. */ @@ -60,14 +60,14 @@ function writeFile(target: string, content: string): void { } async function fetchAsset(asset: DeclaredAsset): Promise { - const raw = await fetchRawFile(asset.repository, 'main', asset.path); - if (!raw) { + const content = await fetchRawFileContent(asset.repository, 'main', asset.path); + if (content === null) { throw new Error( `Declared asset not found: ${asset.repository}/${asset.path} (project "${asset.projectId}"). ` + "Publish the file on the repository's main branch or correct the manifest.", ); } - return raw.content; + return content; } /** Resolve one asset's content for the requested data mode. */ diff --git a/src/scripts/sync-skills.ts b/src/scripts/sync-skills.ts index 3452010..a70a3be 100644 --- a/src/scripts/sync-skills.ts +++ b/src/scripts/sync-skills.ts @@ -4,7 +4,7 @@ import type { DataMode } from './sync-assets'; import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; -import { fetchRawFile, listRepositoryFiles } from '../src/lib/docs/aggregate'; +import { fetchRawFileContent, listRepositoryFiles } from '../src/lib/docs/aggregate'; import { resolveDocChannels } from '../src/lib/docs/channels'; import { loadProjects } from '../src/lib/manifest/load'; import { getReleaseIndex } from '../src/lib/releases/runtime'; @@ -94,13 +94,13 @@ async function fetchAndPublish(skill: SkillTarget, tree: string[]): Promise entry.startsWith(`${skill.sourcePath}/`))) { - const raw = await fetchRawFile(skill.repository, skill.ref, path); - if (!raw) { + const content = await fetchRawFileContent(skill.repository, skill.ref, path); + if (content === null) { continue; } const relativePath = path.slice(skill.sourcePath.length + 1); - write(join(cache, ...relativePath.split('/')), raw.content); - write(join(target, ...relativePath.split('/')), raw.content); + write(join(cache, ...relativePath.split('/')), content); + write(join(target, ...relativePath.split('/')), content); written.push(relativePath); } return written; diff --git a/src/src/lib/concurrency.ts b/src/src/lib/concurrency.ts new file mode 100644 index 0000000..e57d895 --- /dev/null +++ b/src/src/lib/concurrency.ts @@ -0,0 +1,39 @@ +/** + * Map over items with a bounded number of promises in flight, preserving the + * input order in the result. + * + * The data sync is dominated by HTTPS round trips — one raw file (and, for + * documentation pages, one commit feed) per page, skill file and asset — and + * issuing them strictly one at a time made a live sync take minutes. The fetches + * have no ordering relationship of their own, but every caller derives slugs, + * page order and reporting order from the array it builds, so the result order + * must match the input exactly. `limit` keeps the burst small enough not to trip + * GitHub's or the registries' rate limits. + */ +export async function mapWithConcurrency( + items: readonly T[], + limit: number, + map: (item: T, index: number) => Promise, +): Promise { + const results: R[] = []; + let next = 0; + const worker = async (): Promise => { + for (;;) { + const index = next; + next += 1; + if (index >= items.length) { + return; + } + results[index] = await map(items[index] as T, index); + } + }; + // Started explicitly rather than with `Array.from({ length }, worker)`, so the + // worker count cannot be mistaken for the array's only element. + const workerCount = Math.max(1, Math.min(limit, items.length)); + const workers: Promise[] = []; + for (let started = 0; started < workerCount; started += 1) { + workers.push(worker()); + } + await Promise.all(workers); + return results; +} diff --git a/src/src/lib/docs/aggregate.ts b/src/src/lib/docs/aggregate.ts index c3ecbfc..1ceccc7 100644 --- a/src/src/lib/docs/aggregate.ts +++ b/src/src/lib/docs/aggregate.ts @@ -8,6 +8,7 @@ import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'nod import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; +import { mapWithConcurrency } from '../concurrency'; import { resolveGitHubToken } from '../github/token'; import { getReleaseIndex } from '../releases/runtime'; import { docsDescription } from '../seo'; @@ -242,8 +243,17 @@ export async function fetchRawFile( return { path, content, lastModified }; } -/** A repository file's text at a ref, or null when it cannot be read. No commit date is looked up. */ -async function fetchRawText(repository: string, ref: string, path: string): Promise { +/** + * A repository file's text at a ref, or null when it cannot be read, with no + * commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second + * request to the commit feed); the skill and asset mirrors only need the bytes, + * so they use this and skip it. + */ +export async function fetchRawFileContent( + repository: string, + ref: string, + path: string, +): Promise { const response = await fetch(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`); return response.ok ? response.text() : null; } @@ -251,6 +261,16 @@ async function fetchRawText(repository: string, ref: string, path: string): Prom /** Reads a repository file for the page being built (a diagram's `.c4` model, ADR 0013). */ type SourceFileReader = (path: string) => Promise; +/** + * How many repository files one channel fetches at a time. A live sync reads + * every page, its commit feed, the wiki sidebar, the README and any redirects + * file, and doing that one request at a time dominated the sync's wall time. + * Eight keeps the burst modest for GitHub's raw and commit-feed endpoints + * (neither consumes the REST API rate limit, but neither appreciates a + * stampede). + */ +const DOC_FETCH_CONCURRENCY = 8; + export async function listRepositoryFiles(repository: string, branch: string): Promise { const tree = (await githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`)) as { tree?: { path?: string }[]; @@ -645,7 +665,7 @@ async function collectGithubPathChannel( const files = await listRepositoryFiles(project.repository, ref); const tree = new Set(files); const readSourceFile: SourceFileReader = (path) => - tree.has(path) ? fetchRawText(project.repository, ref, path) : Promise.resolve(null); + tree.has(path) ? fetchRawFileContent(project.repository, ref, path) : Promise.resolve(null); // The tree is already in hand, so noticing a repository site configuration // costs nothing — and it is reported, never enforced. warnLegacySiteConfig(project.id, files); @@ -655,13 +675,11 @@ async function collectGithubPathChannel( .filter((path) => !isExcluded(path.split('/').pop() ?? '', layout.exclude)) .toSorted(); - const rawFiles: RawSourceFile[] = []; - for (const path of docFiles) { - const raw = await fetchRawFile(project.repository, ref, path); - if (raw) { - rawFiles.push(raw); - } - } + const rawFiles = ( + await mapWithConcurrency(docFiles, DOC_FETCH_CONCURRENCY, (path) => + fetchRawFile(project.repository, ref, path), + ) + ).filter((raw): raw is RawSourceFile => raw !== null); // A tag never moves, so a pinned channel dates its pages with the release // instead of the branch's commit feed (which would drift on every commit). @@ -875,7 +893,7 @@ async function collectReadmeChannel( // A README is one page, so it has nothing to redirect, but it can still pair a // diagram with a model elsewhere in the repository. const readSourceFile: SourceFileReader = (path) => - fetchRawText(project.repository, channel.ref, path); + fetchRawFileContent(project.repository, channel.ref, path); return { pages: [ await buildPage( diff --git a/src/tests/unit/concurrency.test.ts b/src/tests/unit/concurrency.test.ts new file mode 100644 index 0000000..407f87d --- /dev/null +++ b/src/tests/unit/concurrency.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from 'bun:test'; + +import { mapWithConcurrency } from '../../src/lib/concurrency'; + +const settle = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +describe('mapWithConcurrency', () => { + test('preserves input order however the work completes', async () => { + const delays = [25, 0, 10, 5]; + const results = await mapWithConcurrency(delays, 4, async (delay, index) => { + await settle(delay); + return index; + }); + expect(results).toEqual([0, 1, 2, 3]); + }); + + test('never runs more than the limit at once', async () => { + let inFlight = 0; + let peak = 0; + const items = Array.from({ length: 20 }, (_, index) => index); + await mapWithConcurrency(items, 3, async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await settle(1); + inFlight -= 1; + return null; + }); + expect(peak).toBeLessThanOrEqual(3); + expect(peak).toBeGreaterThan(1); + }); + + test('maps an empty list without starting any work', async () => { + let calls = 0; + const results = await mapWithConcurrency([], 4, async () => { + calls += 1; + return 'never'; + }); + expect(results).toEqual([]); + expect(calls).toBe(0); + }); + + test('propagates a rejection', async () => { + let caught: unknown; + try { + await mapWithConcurrency([1, 2, 3], 2, async (value) => { + if (value === 2) { + throw new Error('boom'); + } + return value; + }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toBe('boom'); + }); +});