Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/scripts/sync-assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ import type { ResolvedProject } from '../src/lib/manifest/load';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';

import { fetchRawFile } from '../src/lib/docs/aggregate';
import { fetchRawFileContent } from '../src/lib/docs/aggregate';
import { loadProjects } from '../src/lib/manifest/load';

/**
* Mirror the repository files declared in a project's `assets` into `public/`,
* so a stable purview.dev URL can serve an artefact that lives in a product
* repository (for example a JSON Schema). Assets are read from the repository's
* `main` branch with the docs aggregator's `fetchRawFile`, cached under
* `main` branch with the docs aggregator's `fetchRawFileContent`, cached under
* `.cache/assets/`, and backed by committed fixtures under `fixtures/assets/`
* for the offline `fixture` mode.
*/
Expand Down Expand Up @@ -60,14 +60,14 @@ function writeFile(target: string, content: string): void {
}

async function fetchAsset(asset: DeclaredAsset): Promise<string> {
const raw = await fetchRawFile(asset.repository, 'main', asset.path);
if (!raw) {
const content = await fetchRawFileContent(asset.repository, 'main', asset.path);
if (content === null) {
throw new Error(
`Declared asset not found: ${asset.repository}/${asset.path} (project "${asset.projectId}"). ` +
"Publish the file on the repository's main branch or correct the manifest.",
);
}
return raw.content;
return content;
}

/** Resolve one asset's content for the requested data mode. */
Expand Down
10 changes: 5 additions & 5 deletions src/scripts/sync-skills.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { DataMode } from './sync-assets';
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';

import { fetchRawFile, listRepositoryFiles } from '../src/lib/docs/aggregate';
import { fetchRawFileContent, listRepositoryFiles } from '../src/lib/docs/aggregate';
import { resolveDocChannels } from '../src/lib/docs/channels';
import { loadProjects } from '../src/lib/manifest/load';
import { getReleaseIndex } from '../src/lib/releases/runtime';
Expand Down Expand Up @@ -94,13 +94,13 @@ async function fetchAndPublish(skill: SkillTarget, tree: string[]): Promise<stri
const cache = resolve(SKILLS_CACHE_DIR, skill.projectId, skill.name);
const target = resolve(SKILLS_PUBLIC_DIR, skill.projectId, skill.name);
for (const path of tree.filter((entry) => entry.startsWith(`${skill.sourcePath}/`))) {
const raw = await fetchRawFile(skill.repository, skill.ref, path);
if (!raw) {
const content = await fetchRawFileContent(skill.repository, skill.ref, path);
if (content === null) {
continue;
}
const relativePath = path.slice(skill.sourcePath.length + 1);
write(join(cache, ...relativePath.split('/')), raw.content);
write(join(target, ...relativePath.split('/')), raw.content);
write(join(cache, ...relativePath.split('/')), content);
write(join(target, ...relativePath.split('/')), content);
written.push(relativePath);
}
return written;
Expand Down
39 changes: 39 additions & 0 deletions src/src/lib/concurrency.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/**
* Map over items with a bounded number of promises in flight, preserving the
* input order in the result.
*
* The data sync is dominated by HTTPS round trips — one raw file (and, for
* documentation pages, one commit feed) per page, skill file and asset — and
* issuing them strictly one at a time made a live sync take minutes. The fetches
* have no ordering relationship of their own, but every caller derives slugs,
* page order and reporting order from the array it builds, so the result order
* must match the input exactly. `limit` keeps the burst small enough not to trip
* GitHub's or the registries' rate limits.
*/
export async function mapWithConcurrency<T, R>(
items: readonly T[],
limit: number,
map: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const results: R[] = [];
let next = 0;
const worker = async (): Promise<void> => {
for (;;) {
const index = next;
next += 1;
if (index >= items.length) {
return;
}
results[index] = await map(items[index] as T, index);
}
};
// Started explicitly rather than with `Array.from({ length }, worker)`, so the
// worker count cannot be mistaken for the array's only element.
const workerCount = Math.max(1, Math.min(limit, items.length));
const workers: Promise<void>[] = [];
for (let started = 0; started < workerCount; started += 1) {
workers.push(worker());
}
await Promise.all(workers);
return results;
}
40 changes: 29 additions & 11 deletions src/src/lib/docs/aggregate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'nod
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';

import { mapWithConcurrency } from '../concurrency';
import { resolveGitHubToken } from '../github/token';
import { getReleaseIndex } from '../releases/runtime';
import { docsDescription } from '../seo';
Expand Down Expand Up @@ -242,15 +243,34 @@ export async function fetchRawFile(
return { path, content, lastModified };
}

/** A repository file's text at a ref, or null when it cannot be read. No commit date is looked up. */
async function fetchRawText(repository: string, ref: string, path: string): Promise<string | null> {
/**
* A repository file's text at a ref, or null when it cannot be read, with no
* commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second
* request to the commit feed); the skill and asset mirrors only need the bytes,
* so they use this and skip it.
*/
export async function fetchRawFileContent(
repository: string,
ref: string,
path: string,
): Promise<string | null> {
const response = await fetch(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`);
return response.ok ? response.text() : null;
}

/** Reads a repository file for the page being built (a diagram's `.c4` model, ADR 0013). */
type SourceFileReader = (path: string) => Promise<string | null>;

/**
* How many repository files one channel fetches at a time. A live sync reads
* every page, its commit feed, the wiki sidebar, the README and any redirects
* file, and doing that one request at a time dominated the sync's wall time.
* Eight keeps the burst modest for GitHub's raw and commit-feed endpoints
* (neither consumes the REST API rate limit, but neither appreciates a
* stampede).
*/
const DOC_FETCH_CONCURRENCY = 8;

export async function listRepositoryFiles(repository: string, branch: string): Promise<string[]> {
const tree = (await githubJson(`/repos/${repository}/git/trees/${branch}?recursive=1`)) as {
tree?: { path?: string }[];
Expand Down Expand Up @@ -645,7 +665,7 @@ async function collectGithubPathChannel(
const files = await listRepositoryFiles(project.repository, ref);
const tree = new Set(files);
const readSourceFile: SourceFileReader = (path) =>
tree.has(path) ? fetchRawText(project.repository, ref, path) : Promise.resolve(null);
tree.has(path) ? fetchRawFileContent(project.repository, ref, path) : Promise.resolve(null);
// The tree is already in hand, so noticing a repository site configuration
// costs nothing — and it is reported, never enforced.
warnLegacySiteConfig(project.id, files);
Expand All @@ -655,13 +675,11 @@ async function collectGithubPathChannel(
.filter((path) => !isExcluded(path.split('/').pop() ?? '', layout.exclude))
.toSorted();

const rawFiles: RawSourceFile[] = [];
for (const path of docFiles) {
const raw = await fetchRawFile(project.repository, ref, path);
if (raw) {
rawFiles.push(raw);
}
}
const rawFiles = (
await mapWithConcurrency(docFiles, DOC_FETCH_CONCURRENCY, (path) =>
fetchRawFile(project.repository, ref, path),
)
).filter((raw): raw is RawSourceFile => raw !== null);

// A tag never moves, so a pinned channel dates its pages with the release
// instead of the branch's commit feed (which would drift on every commit).
Expand Down Expand Up @@ -875,7 +893,7 @@ async function collectReadmeChannel(
// A README is one page, so it has nothing to redirect, but it can still pair a
// diagram with a model elsewhere in the repository.
const readSourceFile: SourceFileReader = (path) =>
fetchRawText(project.repository, channel.ref, path);
fetchRawFileContent(project.repository, channel.ref, path);
return {
pages: [
await buildPage(
Expand Down
57 changes: 57 additions & 0 deletions src/tests/unit/concurrency.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import { describe, expect, test } from 'bun:test';

import { mapWithConcurrency } from '../../src/lib/concurrency';

const settle = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));

describe('mapWithConcurrency', () => {
test('preserves input order however the work completes', async () => {
const delays = [25, 0, 10, 5];
const results = await mapWithConcurrency(delays, 4, async (delay, index) => {
await settle(delay);
return index;
});
expect(results).toEqual([0, 1, 2, 3]);
});

test('never runs more than the limit at once', async () => {
let inFlight = 0;
let peak = 0;
const items = Array.from({ length: 20 }, (_, index) => index);
await mapWithConcurrency(items, 3, async () => {
inFlight += 1;
peak = Math.max(peak, inFlight);
await settle(1);
inFlight -= 1;
return null;
});
expect(peak).toBeLessThanOrEqual(3);
expect(peak).toBeGreaterThan(1);
});

test('maps an empty list without starting any work', async () => {
let calls = 0;
const results = await mapWithConcurrency([], 4, async () => {
calls += 1;
return 'never';
});
expect(results).toEqual([]);
expect(calls).toBe(0);
});

test('propagates a rejection', async () => {
let caught: unknown;
try {
await mapWithConcurrency([1, 2, 3], 2, async (value) => {
if (value === 2) {
throw new Error('boom');
}
return value;
});
} catch (error) {
caught = error;
}
expect(caught).toBeInstanceOf(Error);
expect((caught as Error).message).toBe('boom');
});
});
Loading