|
| 1 | +/** |
| 2 | + * Opt-in resolver for external (`$ref`) JSON Schema references (SEP-2106, R-2106-10). |
| 3 | + * |
| 4 | + * By default the SDK **refuses** to dereference any `$ref`/`$dynamicRef` that is not a same-document |
| 5 | + * reference (see {@link ./schemaBounds.js | assertSchemaSafeToCompile}). That safe default protects |
| 6 | + * against the SSRF / fetch-amplification primitive a naive resolver would expose. SEP-2106 permits an |
| 7 | + * **opt-in** mode that fetches non-local references, but requires it to be: |
| 8 | + * |
| 9 | + * - **disabled by default** — this is a separate function an operator must call explicitly |
| 10 | + * ("explicit operator action", per the SEP); it is never invoked during normal validation; |
| 11 | + * - **host-restricted** — it SHOULD enforce an allowlist of hosts, and at minimum reject loopback, |
| 12 | + * link-local, and private network addresses; |
| 13 | + * - **bounded** — it MUST apply timeouts and response size limits; |
| 14 | + * - **observable** — it SHOULD log the URIs it dereferences; |
| 15 | + * - **fail-closed** — a reference that cannot be resolved MUST cause rejection, never a silent pass. |
| 16 | + * |
| 17 | + * Rather than teaching the (synchronous) validators to fetch, this resolver runs **ahead of time** |
| 18 | + * and returns a self-contained schema: each external document is fetched once and **flattened** into |
| 19 | + * the root document's `$defs`, and every reference (external, and the internal references inside a |
| 20 | + * fetched document) is rewritten to a root-relative same-document JSON Pointer. The result therefore |
| 21 | + * contains **only** local pointer references — no nested `$id` scopes — so it passes the default |
| 22 | + * safety guard and compiles with the standard validators (AJV / cfworker) without any network access |
| 23 | + * at validation time. |
| 24 | + * |
| 25 | + * @example |
| 26 | + * ```ts source="./externalRefResolver.examples.ts#resolveExternalSchemaRefs_basic" |
| 27 | + * const resolved = await resolveExternalSchemaRefs(toolOutputSchema, { |
| 28 | + * allowlist: ['schemas.example.com'] |
| 29 | + * }); |
| 30 | + * // `resolved` has no external $refs; hand it to registerTool / fromJsonSchema as usual. |
| 31 | + * ``` |
| 32 | + * |
| 33 | + * @module |
| 34 | + */ |
| 35 | + |
| 36 | +import type { JsonSchemaType } from './types.js'; |
| 37 | + |
| 38 | +/** Default per-request fetch timeout, in milliseconds. */ |
| 39 | +export const DEFAULT_REF_FETCH_TIMEOUT_MS = 5000; |
| 40 | + |
| 41 | +/** Default maximum size of a fetched schema document, in bytes. */ |
| 42 | +export const DEFAULT_REF_MAX_BYTES = 1_000_000; |
| 43 | + |
| 44 | +/** Default maximum number of distinct external documents fetched while resolving one schema. */ |
| 45 | +export const DEFAULT_REF_MAX_DOCUMENTS = 50; |
| 46 | + |
| 47 | +/** Options controlling {@link resolveExternalSchemaRefs}. */ |
| 48 | +export interface ResolveExternalRefsOptions { |
| 49 | + /** |
| 50 | + * Allowlist of permitted hosts (e.g. `'schemas.example.com'`). When provided, only references |
| 51 | + * whose host exactly matches an entry are fetched; everything else is rejected. **Strongly |
| 52 | + * recommended** — without it, the resolver still rejects loopback/link-local/private targets, |
| 53 | + * but cannot defend against a public URL that an attacker controls. |
| 54 | + */ |
| 55 | + allowlist?: readonly string[]; |
| 56 | + /** |
| 57 | + * Permitted URL protocols. Defaults to `['https:']`. Add `'http:'` only for trusted internal |
| 58 | + * use; plaintext fetches are easier to tamper with in transit. |
| 59 | + */ |
| 60 | + allowedProtocols?: readonly string[]; |
| 61 | + /** Per-request timeout in milliseconds (default {@link DEFAULT_REF_FETCH_TIMEOUT_MS}). */ |
| 62 | + timeoutMs?: number; |
| 63 | + /** Maximum size of a single fetched document in bytes (default {@link DEFAULT_REF_MAX_BYTES}). */ |
| 64 | + maxBytes?: number; |
| 65 | + /** Maximum number of distinct documents fetched (default {@link DEFAULT_REF_MAX_DOCUMENTS}). */ |
| 66 | + maxDocuments?: number; |
| 67 | + /** |
| 68 | + * Fetch implementation. Defaults to the global `fetch`. Inject a custom one for tests or to add |
| 69 | + * proxying/auth. Must honour the `AbortSignal` passed in `init.signal`. |
| 70 | + */ |
| 71 | + fetch?: typeof globalThis.fetch; |
| 72 | + /** |
| 73 | + * Called with each external URI **before** it is dereferenced, so operators can audit/log |
| 74 | + * network access (the SEP asks implementations to log dereferenced URIs). Defaults to a no-op. |
| 75 | + */ |
| 76 | + onDereference?: (uri: string) => void; |
| 77 | +} |
| 78 | + |
| 79 | +interface ResolvedOptions { |
| 80 | + allowlist?: readonly string[]; |
| 81 | + allowedProtocols: readonly string[]; |
| 82 | + timeoutMs: number; |
| 83 | + maxBytes: number; |
| 84 | + maxDocuments: number; |
| 85 | + fetchImpl: typeof globalThis.fetch; |
| 86 | + onDereference: (uri: string) => void; |
| 87 | +} |
| 88 | + |
| 89 | +/** Split a reference into its base (document) URI and fragment (without the leading `#`). */ |
| 90 | +function splitRef(ref: string): { base: string; fragment: string } { |
| 91 | + const hashIndex = ref.indexOf('#'); |
| 92 | + if (hashIndex === -1) { |
| 93 | + return { base: ref, fragment: '' }; |
| 94 | + } |
| 95 | + return { base: ref.slice(0, hashIndex), fragment: ref.slice(hashIndex + 1) }; |
| 96 | +} |
| 97 | + |
| 98 | +/** A reference is "external" when it has a non-empty base (i.e. it does not start with `#`). */ |
| 99 | +function isExternalRef(ref: string): boolean { |
| 100 | + return ref.length > 0 && !ref.startsWith('#'); |
| 101 | +} |
| 102 | + |
| 103 | +/** |
| 104 | + * Reject hosts that are obvious SSRF targets: loopback, link-local, and private ranges. This is a |
| 105 | + * best-effort literal-address check (it does not resolve DNS); the allowlist is the real defence. |
| 106 | + */ |
| 107 | +function assertHostAllowed(url: URL, options: ResolvedOptions): void { |
| 108 | + if (!options.allowedProtocols.includes(url.protocol)) { |
| 109 | + throw new Error( |
| 110 | + `Refusing to dereference "${url.href}": protocol "${url.protocol}" is not allowed (allowed: ${options.allowedProtocols.join(', ')}).` |
| 111 | + ); |
| 112 | + } |
| 113 | + |
| 114 | + const host = url.hostname.toLowerCase(); |
| 115 | + |
| 116 | + if (options.allowlist) { |
| 117 | + if (!options.allowlist.includes(host)) { |
| 118 | + throw new Error(`Refusing to dereference "${url.href}": host "${host}" is not in the allowlist.`); |
| 119 | + } |
| 120 | + return; |
| 121 | + } |
| 122 | + |
| 123 | + // No allowlist: reject the most dangerous literal targets so an unguarded call still cannot |
| 124 | + // trivially hit internal services / cloud metadata endpoints. |
| 125 | + const blocked = |
| 126 | + host === 'localhost' || |
| 127 | + host === '::1' || |
| 128 | + host === '0.0.0.0' || |
| 129 | + host.endsWith('.localhost') || |
| 130 | + host.endsWith('.internal') || |
| 131 | + /^127\./.test(host) || |
| 132 | + /^10\./.test(host) || |
| 133 | + /^192\.168\./.test(host) || |
| 134 | + /^169\.254\./.test(host) || |
| 135 | + /^172\.(1[6-9]|2\d|3[01])\./.test(host) || |
| 136 | + host.startsWith('fc') || |
| 137 | + host.startsWith('fd') || |
| 138 | + host.startsWith('fe80'); |
| 139 | + if (blocked) { |
| 140 | + throw new Error( |
| 141 | + `Refusing to dereference "${url.href}": host "${host}" resolves to a loopback/link-local/private address. ` + |
| 142 | + `Provide an explicit allowlist to dereference internal hosts intentionally.` |
| 143 | + ); |
| 144 | + } |
| 145 | +} |
| 146 | + |
| 147 | +/** Read a response body, enforcing the byte cap as it streams. */ |
| 148 | +async function readBounded(response: Response, maxBytes: number, uri: string): Promise<string> { |
| 149 | + const declared = response.headers.get('content-length'); |
| 150 | + if (declared && Number(declared) > maxBytes) { |
| 151 | + throw new Error(`Refusing to dereference "${uri}": declared content-length ${declared} exceeds max ${maxBytes} bytes.`); |
| 152 | + } |
| 153 | + |
| 154 | + const body = response.body; |
| 155 | + if (!body) { |
| 156 | + const text = await response.text(); |
| 157 | + if (text.length > maxBytes) { |
| 158 | + throw new Error(`Refusing to dereference "${uri}": response exceeds max ${maxBytes} bytes.`); |
| 159 | + } |
| 160 | + return text; |
| 161 | + } |
| 162 | + |
| 163 | + const reader = body.getReader(); |
| 164 | + const decoder = new TextDecoder(); |
| 165 | + let received = 0; |
| 166 | + let out = ''; |
| 167 | + for (;;) { |
| 168 | + const { done, value } = await reader.read(); |
| 169 | + if (done) { |
| 170 | + break; |
| 171 | + } |
| 172 | + received += value.byteLength; |
| 173 | + if (received > maxBytes) { |
| 174 | + await reader.cancel(); |
| 175 | + throw new Error(`Refusing to dereference "${uri}": response exceeds max ${maxBytes} bytes.`); |
| 176 | + } |
| 177 | + out += decoder.decode(value, { stream: true }); |
| 178 | + } |
| 179 | + out += decoder.decode(); |
| 180 | + return out; |
| 181 | +} |
| 182 | + |
| 183 | +/** Fetch and parse one external schema document, applying timeout + size bounds. */ |
| 184 | +async function fetchDocument(uri: string, options: ResolvedOptions): Promise<Record<string, unknown>> { |
| 185 | + let url: URL; |
| 186 | + try { |
| 187 | + url = new URL(uri); |
| 188 | + } catch { |
| 189 | + throw new Error(`Refusing to dereference "${uri}": not an absolute URI.`); |
| 190 | + } |
| 191 | + assertHostAllowed(url, options); |
| 192 | + |
| 193 | + options.onDereference(url.href); |
| 194 | + |
| 195 | + const controller = new AbortController(); |
| 196 | + const timer = setTimeout(() => controller.abort(), options.timeoutMs); |
| 197 | + let text: string; |
| 198 | + try { |
| 199 | + const response = await options.fetchImpl(url.href, { signal: controller.signal, redirect: 'error' }); |
| 200 | + if (!response.ok) { |
| 201 | + throw new Error(`Refusing to use "${url.href}": fetch returned HTTP ${response.status}.`); |
| 202 | + } |
| 203 | + text = await readBounded(response, options.maxBytes, url.href); |
| 204 | + } catch (error) { |
| 205 | + if (error instanceof Error && error.name === 'AbortError') { |
| 206 | + throw new Error(`Refusing to dereference "${url.href}": fetch timed out after ${options.timeoutMs}ms.`); |
| 207 | + } |
| 208 | + throw error instanceof Error ? error : new Error(String(error)); |
| 209 | + } finally { |
| 210 | + clearTimeout(timer); |
| 211 | + } |
| 212 | + |
| 213 | + let parsed: unknown; |
| 214 | + try { |
| 215 | + parsed = JSON.parse(text); |
| 216 | + } catch { |
| 217 | + throw new Error(`Refusing to use "${url.href}": response is not valid JSON.`); |
| 218 | + } |
| 219 | + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { |
| 220 | + throw new Error(`Refusing to use "${url.href}": resolved document is not a JSON Schema object.`); |
| 221 | + } |
| 222 | + return parsed as Record<string, unknown>; |
| 223 | +} |
| 224 | + |
| 225 | +/** |
| 226 | + * Resolve and inline all external `$ref`/`$dynamicRef` references in a JSON Schema, returning a |
| 227 | + * self-contained schema with only same-document references. |
| 228 | + * |
| 229 | + * This is the **opt-in** external-reference mode described by SEP-2106 (R-2106-10): it is never |
| 230 | + * invoked during normal validation and must be called explicitly. Each distinct external document is |
| 231 | + * fetched once (subject to the allowlist, protocol, timeout, size, and document-count limits) and |
| 232 | + * bundled under a generated `$defs` slot that preserves the document's canonical `$id`; every |
| 233 | + * external reference to that document is rewritten to a local JSON Pointer into the slot. References |
| 234 | + * the resolver cannot satisfy cause rejection (fail-closed) rather than a silent pass. |
| 235 | + * |
| 236 | + * @param schema - the schema to resolve. Not mutated; a new object is returned. |
| 237 | + * @param options - allowlist and bounds. Supplying an `allowlist` is strongly recommended. |
| 238 | + * @returns a schema whose references are all same-document (safe to compile with the default guard). |
| 239 | + * @throws Error if a reference targets a disallowed host, cannot be fetched within bounds, uses an |
| 240 | + * external `$anchor` fragment (unsupported), or the document budget is exceeded. |
| 241 | + */ |
| 242 | +export async function resolveExternalSchemaRefs(schema: JsonSchemaType, options: ResolveExternalRefsOptions = {}): Promise<JsonSchemaType> { |
| 243 | + const resolved: ResolvedOptions = { |
| 244 | + allowlist: options.allowlist, |
| 245 | + allowedProtocols: options.allowedProtocols ?? ['https:'], |
| 246 | + timeoutMs: options.timeoutMs ?? DEFAULT_REF_FETCH_TIMEOUT_MS, |
| 247 | + maxBytes: options.maxBytes ?? DEFAULT_REF_MAX_BYTES, |
| 248 | + maxDocuments: options.maxDocuments ?? DEFAULT_REF_MAX_DOCUMENTS, |
| 249 | + fetchImpl: options.fetch ?? globalThis.fetch, |
| 250 | + onDereference: options.onDereference ?? (() => {}) |
| 251 | + }; |
| 252 | + if (typeof resolved.fetchImpl !== 'function') { |
| 253 | + throw new TypeError('resolveExternalSchemaRefs: no fetch implementation available; pass options.fetch.'); |
| 254 | + } |
| 255 | + |
| 256 | + // Map of base URI -> generated $defs slot key, and the collected bundle of fetched documents. |
| 257 | + const slotByBase = new Map<string, string>(); |
| 258 | + const bundle: Record<string, Record<string, unknown>> = {}; |
| 259 | + |
| 260 | + const ensureDocument = async (base: string): Promise<string> => { |
| 261 | + const existing = slotByBase.get(base); |
| 262 | + if (existing) { |
| 263 | + return existing; |
| 264 | + } |
| 265 | + if (slotByBase.size >= resolved.maxDocuments) { |
| 266 | + throw new Error(`Refusing to resolve more than ${resolved.maxDocuments} external schema documents.`); |
| 267 | + } |
| 268 | + const slot = `__externalRef_${slotByBase.size}`; |
| 269 | + slotByBase.set(base, slot); |
| 270 | + |
| 271 | + const doc = await fetchDocument(base, resolved); |
| 272 | + // Flatten the document into the root's $defs under `slot`. Its own identity/dialect keywords |
| 273 | + // are dropped (it no longer is a standalone document), and its internal references are |
| 274 | + // rewritten to root-relative pointers that target this slot. |
| 275 | + const { $id: _id, $schema: _schema, ...rest } = doc; |
| 276 | + void _id; |
| 277 | + void _schema; |
| 278 | + const flattened = await rewrite(rest, `/$defs/${slot}`); |
| 279 | + bundle[slot] = flattened as Record<string, unknown>; |
| 280 | + return slot; |
| 281 | + }; |
| 282 | + |
| 283 | + /** |
| 284 | + * Rewrite references in `node` to root-relative same-document pointers. |
| 285 | + * |
| 286 | + * @param node - the schema (sub)tree. |
| 287 | + * @param slotPrefix - `''` for the root document; `/$defs/<slot>` when rewriting a fetched |
| 288 | + * document that is being flattened into that slot (so its internal `#/x` refs become |
| 289 | + * `#/$defs/<slot>/x`). |
| 290 | + */ |
| 291 | + async function rewrite(node: unknown, slotPrefix: string): Promise<unknown> { |
| 292 | + if (Array.isArray(node)) { |
| 293 | + return Promise.all(node.map(item => rewrite(item, slotPrefix))); |
| 294 | + } |
| 295 | + if (node === null || typeof node !== 'object') { |
| 296 | + return node; |
| 297 | + } |
| 298 | + const out: Record<string, unknown> = {}; |
| 299 | + for (const [key, value] of Object.entries(node as Record<string, unknown>)) { |
| 300 | + if ((key === '$ref' || key === '$dynamicRef') && typeof value === 'string') { |
| 301 | + if (isExternalRef(value)) { |
| 302 | + const { base, fragment } = splitRef(value); |
| 303 | + if (fragment && !fragment.startsWith('/')) { |
| 304 | + throw new Error( |
| 305 | + `Cannot resolve external ${key} "${value}": external "$anchor" fragments are not supported. ` + |
| 306 | + `Use a JSON Pointer fragment (e.g. "#/$defs/Foo") or restructure the schema.` |
| 307 | + ); |
| 308 | + } |
| 309 | + const slot = await ensureDocument(base); |
| 310 | + out[key] = `#/$defs/${slot}${fragment}`; |
| 311 | + } else if (slotPrefix === '') { |
| 312 | + out[key] = value; |
| 313 | + } else { |
| 314 | + // Internal reference inside a flattened document: re-base it onto the slot. |
| 315 | + const fragment = value.slice(1); |
| 316 | + if (fragment !== '' && !fragment.startsWith('/')) { |
| 317 | + throw new Error( |
| 318 | + `Cannot flatten ${key} "${value}" from an external document: "$anchor" references inside ` + |
| 319 | + `fetched schemas are not supported. Use JSON Pointer references (e.g. "#/$defs/Foo").` |
| 320 | + ); |
| 321 | + } |
| 322 | + out[key] = `#${slotPrefix}${fragment}`; |
| 323 | + } |
| 324 | + } else if (slotPrefix !== '' && (key === '$id' || key === '$anchor' || key === '$dynamicAnchor')) { |
| 325 | + // Scope-defining keywords inside a flattened document cannot be preserved once the |
| 326 | + // document loses its own identity; reject rather than silently change semantics. |
| 327 | + throw new Error( |
| 328 | + `Cannot flatten external schema: nested "${key}" is not supported. ` + |
| 329 | + `Restructure the referenced document to use plain JSON Pointer references.` |
| 330 | + ); |
| 331 | + } else { |
| 332 | + out[key] = await rewrite(value, slotPrefix); |
| 333 | + } |
| 334 | + } |
| 335 | + return out; |
| 336 | + } |
| 337 | + |
| 338 | + const rewrittenRoot = (await rewrite(schema, '')) as Record<string, unknown>; |
| 339 | + |
| 340 | + if (slotByBase.size === 0) { |
| 341 | + // No external references; return the (structurally identical) schema unchanged. |
| 342 | + return rewrittenRoot as JsonSchemaType; |
| 343 | + } |
| 344 | + |
| 345 | + const existingDefs = (rewrittenRoot.$defs as Record<string, unknown> | undefined) ?? {}; |
| 346 | + return { ...rewrittenRoot, $defs: { ...existingDefs, ...bundle } } as JsonSchemaType; |
| 347 | +} |
0 commit comments