From 862f37f12e22d305d85f702e0a962a39a65edb78 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Sun, 31 May 2026 18:34:45 -0400 Subject: [PATCH 1/3] chore: harden all HTTP calls across SDK Since reporters should be durable by design, we will now retry by default all HTTP calls we do here. --- src/_internalUtils.ts | 16 +++++- src/fetchTestDurations.ts | 51 ++++++------------ src/githubOIDC.ts | 22 ++++---- src/uploadReport.ts | 105 ++++++++++++++++---------------------- 4 files changed, 89 insertions(+), 105 deletions(-) diff --git a/src/_internalUtils.ts b/src/_internalUtils.ts index 0ff34c8..9a604a0 100644 --- a/src/_internalUtils.ts +++ b/src/_internalUtils.ts @@ -43,6 +43,20 @@ export async function retryWithBackoff(job: () => Promise, backoff: number return await job(); } +export const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; + +export async function fetchWithRetries(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { + return await retryWithBackoff(async () => { + const response = await fetch(input, init); + if (!response.ok) { + const url = response.url || (input instanceof URL ? input.href : typeof input === 'string' ? input : input.url); + const body = await response.text().catch(() => ''); + throw new Error(response.status + ' ' + url + ' ' + body); + } + return response; + }, backoff); +} + export function shell(command: string, args?: string[], options?: SpawnSyncOptionsWithStringEncoding) { try { const result = spawnSync(command, args, { encoding: 'utf-8', ...options }); @@ -92,4 +106,4 @@ export function randomUUIDBase62(): string { } return chars.reverse().join(''); -} \ No newline at end of file +} diff --git a/src/fetchTestDurations.ts b/src/fetchTestDurations.ts index 801d74c..acd37b3 100644 --- a/src/fetchTestDurations.ts +++ b/src/fetchTestDurations.ts @@ -1,6 +1,6 @@ import { FlakinessReport } from '@flakiness/flakiness-report'; import { URL } from 'url'; -import { compressTextAsync, retryWithBackoff, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, fetchWithRetries, sha1Text } from './_internalUtils.js'; import { GithubOIDC } from './githubOIDC.js'; type TestDurationsFetcherOptions = { @@ -35,9 +35,6 @@ export type FetchTestDurationsOptions = { flakinessAccessToken?: string; } -// Retry schedule for uploading the report to the presigned URL. -const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; - // The computed historical durations are not ready immediately after submit, so // we poll the download URL for up to ~90 seconds before giving up. const DOWNLOAD_BACKOFF = [Array(10).fill(1000), Array(10).fill(2000), Array(20).fill(3000)].flat(); @@ -107,19 +104,14 @@ class TestDurationsFetcher { private async _api(pathname: string, token: string, body?: any): Promise { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; - return await retryWithBackoff(async () => { - const response = await fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - }); - if (!response.ok) - throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`) - return response; - }, HTTP_BACKOFF).then(async response => await response.json()); + return await fetchWithRetries(url, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }).then(async response => await response.json()); } async fetch(): Promise { @@ -147,12 +139,7 @@ class TestDurationsFetcher { createResponse.testDurationsToken, ); - return await retryWithBackoff(async () => { - const response = await fetch(submitResponse.downloadUrl); - if (!response.ok) - throw new Error(`Request to ${submitResponse.downloadUrl} failed with ${response.status}`); - return response; - }, DOWNLOAD_BACKOFF).then(async response => await response.json() as FlakinessReport.Report); + return await fetchWithRetries(submitResponse.downloadUrl, undefined, DOWNLOAD_BACKOFF).then(async response => await response.json() as FlakinessReport.Report); } private async _uploadReport(data: string, uploadUrl: string) { @@ -162,16 +149,12 @@ class TestDurationsFetcher { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); - if (!response.ok) - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + const response = await fetchWithRetries(uploadUrl, { + method: 'PUT', + headers, + body: Buffer.from(compressed), + }); + // Read response to ensure it completes + await response.arrayBuffer(); } } diff --git a/src/githubOIDC.ts b/src/githubOIDC.ts index 2db8174..44ba5a5 100644 --- a/src/githubOIDC.ts +++ b/src/githubOIDC.ts @@ -1,3 +1,5 @@ +import { fetchWithRetries } from './_internalUtils.js'; + /** * Provides GitHub Actions OIDC (OpenID Connect) token exchange. * @@ -56,16 +58,16 @@ export class GithubOIDC { const url = new URL(this._requestUrl); url.searchParams.set('audience', flakinessProject); - const response = await fetch(url, { - headers: { - 'Authorization': `bearer ${this._requestToken}`, - 'Accept': 'application/json; api-version=2.0', - }, - }); - - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`Failed to request GitHub OIDC token: ${response.status} ${body}`); + let response: Response; + try { + response = await fetchWithRetries(url, { + headers: { + 'Authorization': `bearer ${this._requestToken}`, + 'Accept': 'application/json; api-version=2.0', + }, + }); + } catch (error: any) { + throw new Error(`Failed to request GitHub OIDC token: ${error.message || String(error)}`); } const json = await response.json() as { value?: string }; diff --git a/src/uploadReport.ts b/src/uploadReport.ts index 6f29a8c..542d563 100644 --- a/src/uploadReport.ts +++ b/src/uploadReport.ts @@ -3,7 +3,7 @@ import assert from 'assert'; import fs from 'fs'; import { URL } from 'url'; import { GithubOIDC } from './githubOIDC.js'; -import { compressTextAsync, retryWithBackoff, sha1File, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, fetchWithRetries, sha1File, sha1Text } from './_internalUtils.js'; type ReportUploaderOptions = { flakinessEndpoint: string; @@ -293,8 +293,6 @@ export async function uploadReport( } } -const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; - class ReportUpload { private _report: FlakinessReport.Report; private _attachments: Attachment[]; @@ -309,23 +307,25 @@ class ReportUpload { private async _api(pathname: string, token: string, body?: any): Promise<{ result?: OUTPUT, error?: string }> { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; - return await fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - }).then(async response => !response.ok ? { - result: undefined, - error: response.status + ' ' + url.href + ' ' + await response.text(), - } : { - result: await response.json() as OUTPUT, - error: undefined, - }).catch(error => ({ - result: undefined, - error, - })); + try { + const response = await fetchWithRetries(url, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + return { + result: await response.json() as OUTPUT, + error: undefined, + }; + } catch (error: any) { + return { + result: undefined, + error: error instanceof Error ? error.message : String(error), + }; + } } async upload(): Promise<{ success: false, message?: string } | { success: true, reportUrl: string }> { @@ -370,18 +370,13 @@ class ReportUpload { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + const response = await fetchWithRetries(uploadUrl, { + method: 'PUT', + headers, + body: Buffer.from(compressed), + }); + // Read response to ensure it completes + await response.arrayBuffer(); } private async _uploadAttachment(attachment: Attachment, uploadUrl: string) { @@ -393,22 +388,17 @@ class ReportUpload { ; // Stream file only if there's attachment path and we should NOT compress it. if (!compressable && attachment.type === 'file') { - await retryWithBackoff(async () => { - const fileBuffer = await fs.promises.readFile(attachment.path); - const response = await fetch(uploadUrl, { - method: 'PUT', - headers: { - 'Content-Type': attachment.contentType, - 'Content-Length': fileBuffer.length + '', - }, - body: new Uint8Array(fileBuffer), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + const fileBuffer = await fs.promises.readFile(attachment.path); + const response = await fetchWithRetries(uploadUrl, { + method: 'PUT', + headers: { + 'Content-Type': attachment.contentType, + 'Content-Length': fileBuffer.length + '', + }, + body: new Uint8Array(fileBuffer), + }); + // Read response to ensure it completes + await response.arrayBuffer(); return; } let buffer = attachment.type === 'buffer' ? attachment.body : await fs.promises.readFile(attachment.path); @@ -427,17 +417,12 @@ class ReportUpload { headers['Content-Encoding'] = encoding; } - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: new Uint8Array(buffer), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + const response = await fetchWithRetries(uploadUrl, { + method: 'PUT', + headers, + body: new Uint8Array(buffer), + }); + // Read response to ensure it completes + await response.arrayBuffer(); } } From 0ec2f709ce2367fcb1d513ddc76826b2e0114c5d Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Sun, 31 May 2026 18:44:24 -0400 Subject: [PATCH 2/3] fix --- src/_internalUtils.ts | 25 +++++++++++++++++-------- src/fetchTestDurations.ts | 6 ++---- src/uploadReport.ts | 14 ++++---------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/_internalUtils.ts b/src/_internalUtils.ts index 9a604a0..c2b7a0b 100644 --- a/src/_internalUtils.ts +++ b/src/_internalUtils.ts @@ -45,15 +45,24 @@ export async function retryWithBackoff(job: () => Promise, backoff: number export const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; +async function fetchOk(input: RequestInfo | URL, init?: RequestInit): Promise { + const response = await fetch(input, init); + if (!response.ok) { + const url = response.url || (input instanceof URL ? input.href : typeof input === 'string' ? input : input.url); + const body = await response.text().catch(() => ''); + throw new Error(response.status + ' ' + url + ' ' + body); + } + return response; +} + export async function fetchWithRetries(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { - return await retryWithBackoff(async () => { - const response = await fetch(input, init); - if (!response.ok) { - const url = response.url || (input instanceof URL ? input.href : typeof input === 'string' ? input : input.url); - const body = await response.text().catch(() => ''); - throw new Error(response.status + ' ' + url + ' ' + body); - } - return response; + return await retryWithBackoff(async () => await fetchOk(input, init), backoff); +} + +export async function fetchAndDrainWithRetries(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { + await retryWithBackoff(async () => { + const response = await fetchOk(input, init); + await response.arrayBuffer(); }, backoff); } diff --git a/src/fetchTestDurations.ts b/src/fetchTestDurations.ts index acd37b3..cc92364 100644 --- a/src/fetchTestDurations.ts +++ b/src/fetchTestDurations.ts @@ -1,6 +1,6 @@ import { FlakinessReport } from '@flakiness/flakiness-report'; import { URL } from 'url'; -import { compressTextAsync, fetchWithRetries, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, fetchAndDrainWithRetries, fetchWithRetries, sha1Text } from './_internalUtils.js'; import { GithubOIDC } from './githubOIDC.js'; type TestDurationsFetcherOptions = { @@ -149,12 +149,10 @@ class TestDurationsFetcher { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - const response = await fetchWithRetries(uploadUrl, { + await fetchAndDrainWithRetries(uploadUrl, { method: 'PUT', headers, body: Buffer.from(compressed), }); - // Read response to ensure it completes - await response.arrayBuffer(); } } diff --git a/src/uploadReport.ts b/src/uploadReport.ts index 542d563..ac2f19c 100644 --- a/src/uploadReport.ts +++ b/src/uploadReport.ts @@ -3,7 +3,7 @@ import assert from 'assert'; import fs from 'fs'; import { URL } from 'url'; import { GithubOIDC } from './githubOIDC.js'; -import { compressTextAsync, fetchWithRetries, sha1File, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, fetchAndDrainWithRetries, fetchWithRetries, sha1File, sha1Text } from './_internalUtils.js'; type ReportUploaderOptions = { flakinessEndpoint: string; @@ -370,13 +370,11 @@ class ReportUpload { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - const response = await fetchWithRetries(uploadUrl, { + await fetchAndDrainWithRetries(uploadUrl, { method: 'PUT', headers, body: Buffer.from(compressed), }); - // Read response to ensure it completes - await response.arrayBuffer(); } private async _uploadAttachment(attachment: Attachment, uploadUrl: string) { @@ -389,7 +387,7 @@ class ReportUpload { // Stream file only if there's attachment path and we should NOT compress it. if (!compressable && attachment.type === 'file') { const fileBuffer = await fs.promises.readFile(attachment.path); - const response = await fetchWithRetries(uploadUrl, { + await fetchAndDrainWithRetries(uploadUrl, { method: 'PUT', headers: { 'Content-Type': attachment.contentType, @@ -397,8 +395,6 @@ class ReportUpload { }, body: new Uint8Array(fileBuffer), }); - // Read response to ensure it completes - await response.arrayBuffer(); return; } let buffer = attachment.type === 'buffer' ? attachment.body : await fs.promises.readFile(attachment.path); @@ -417,12 +413,10 @@ class ReportUpload { headers['Content-Encoding'] = encoding; } - const response = await fetchWithRetries(uploadUrl, { + await fetchAndDrainWithRetries(uploadUrl, { method: 'PUT', headers, body: new Uint8Array(buffer), }); - // Read response to ensure it completes - await response.arrayBuffer(); } } From 64324926fcb538fc7c92f671684e5b0fbbe6b709 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Sun, 31 May 2026 18:55:51 -0400 Subject: [PATCH 3/3] better helpers --- src/_internalUtils.ts | 18 ++++++++++++------ src/fetchTestDurations.ts | 14 +++++--------- src/githubOIDC.ts | 7 +++---- src/uploadReport.ts | 24 ++++++------------------ 4 files changed, 26 insertions(+), 37 deletions(-) diff --git a/src/_internalUtils.ts b/src/_internalUtils.ts index c2b7a0b..7d99326 100644 --- a/src/_internalUtils.ts +++ b/src/_internalUtils.ts @@ -1,8 +1,6 @@ import { spawnSync, SpawnSyncOptionsWithStringEncoding } from 'child_process'; import crypto from 'crypto'; import fs from 'fs'; -import http from 'http'; -import https from 'https'; import util from 'util'; import zlib from 'zlib'; @@ -55,13 +53,21 @@ async function fetchOk(input: RequestInfo | URL, init?: RequestInit): Promise { - return await retryWithBackoff(async () => await fetchOk(input, init), backoff); +export async function getJSON(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { + return await retryWithBackoff(async () => { + const response = await fetchOk(input, init); + return await response.json() as T; + }, backoff); } -export async function fetchAndDrainWithRetries(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { +export async function putBuffer(input: RequestInfo | URL, body: Buffer, headers?: HeadersInit, backoff: number[] = HTTP_BACKOFF): Promise { await retryWithBackoff(async () => { - const response = await fetchOk(input, init); + const response = await fetchOk(input, { + method: 'PUT', + headers, + body: new Uint8Array(body), + }); + // Read response to ensure it completes. await response.arrayBuffer(); }, backoff); } diff --git a/src/fetchTestDurations.ts b/src/fetchTestDurations.ts index cc92364..551ca16 100644 --- a/src/fetchTestDurations.ts +++ b/src/fetchTestDurations.ts @@ -1,6 +1,6 @@ import { FlakinessReport } from '@flakiness/flakiness-report'; import { URL } from 'url'; -import { compressTextAsync, fetchAndDrainWithRetries, fetchWithRetries, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, getJSON, putBuffer, sha1Text } from './_internalUtils.js'; import { GithubOIDC } from './githubOIDC.js'; type TestDurationsFetcherOptions = { @@ -104,14 +104,14 @@ class TestDurationsFetcher { private async _api(pathname: string, token: string, body?: any): Promise { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; - return await fetchWithRetries(url, { + return await getJSON(url, { method: 'POST', headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json', }, body: body ? JSON.stringify(body) : undefined, - }).then(async response => await response.json()); + }); } async fetch(): Promise { @@ -139,7 +139,7 @@ class TestDurationsFetcher { createResponse.testDurationsToken, ); - return await fetchWithRetries(submitResponse.downloadUrl, undefined, DOWNLOAD_BACKOFF).then(async response => await response.json() as FlakinessReport.Report); + return await getJSON(submitResponse.downloadUrl, undefined, DOWNLOAD_BACKOFF); } private async _uploadReport(data: string, uploadUrl: string) { @@ -149,10 +149,6 @@ class TestDurationsFetcher { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await fetchAndDrainWithRetries(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); + await putBuffer(uploadUrl, compressed, headers); } } diff --git a/src/githubOIDC.ts b/src/githubOIDC.ts index 44ba5a5..ca6c524 100644 --- a/src/githubOIDC.ts +++ b/src/githubOIDC.ts @@ -1,4 +1,4 @@ -import { fetchWithRetries } from './_internalUtils.js'; +import { getJSON } from './_internalUtils.js'; /** * Provides GitHub Actions OIDC (OpenID Connect) token exchange. @@ -58,9 +58,9 @@ export class GithubOIDC { const url = new URL(this._requestUrl); url.searchParams.set('audience', flakinessProject); - let response: Response; + let json: { value?: string }; try { - response = await fetchWithRetries(url, { + json = await getJSON<{ value?: string }>(url, { headers: { 'Authorization': `bearer ${this._requestToken}`, 'Accept': 'application/json; api-version=2.0', @@ -70,7 +70,6 @@ export class GithubOIDC { throw new Error(`Failed to request GitHub OIDC token: ${error.message || String(error)}`); } - const json = await response.json() as { value?: string }; if (!json.value) throw new Error('GitHub OIDC token response did not contain a token value.'); diff --git a/src/uploadReport.ts b/src/uploadReport.ts index ac2f19c..365ae5f 100644 --- a/src/uploadReport.ts +++ b/src/uploadReport.ts @@ -3,7 +3,7 @@ import assert from 'assert'; import fs from 'fs'; import { URL } from 'url'; import { GithubOIDC } from './githubOIDC.js'; -import { compressTextAsync, fetchAndDrainWithRetries, fetchWithRetries, sha1File, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, getJSON, putBuffer, sha1File, sha1Text } from './_internalUtils.js'; type ReportUploaderOptions = { flakinessEndpoint: string; @@ -308,7 +308,7 @@ class ReportUpload { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; try { - const response = await fetchWithRetries(url, { + const result = await getJSON(url, { method: 'POST', headers: { 'Authorization': `Bearer ${token}`, @@ -317,7 +317,7 @@ class ReportUpload { body: body ? JSON.stringify(body) : undefined, }); return { - result: await response.json() as OUTPUT, + result, error: undefined, }; } catch (error: any) { @@ -370,11 +370,7 @@ class ReportUpload { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await fetchAndDrainWithRetries(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); + await putBuffer(uploadUrl, compressed, headers); } private async _uploadAttachment(attachment: Attachment, uploadUrl: string) { @@ -387,13 +383,9 @@ class ReportUpload { // Stream file only if there's attachment path and we should NOT compress it. if (!compressable && attachment.type === 'file') { const fileBuffer = await fs.promises.readFile(attachment.path); - await fetchAndDrainWithRetries(uploadUrl, { - method: 'PUT', - headers: { + await putBuffer(uploadUrl, fileBuffer, { 'Content-Type': attachment.contentType, 'Content-Length': fileBuffer.length + '', - }, - body: new Uint8Array(fileBuffer), }); return; } @@ -413,10 +405,6 @@ class ReportUpload { headers['Content-Encoding'] = encoding; } - await fetchAndDrainWithRetries(uploadUrl, { - method: 'PUT', - headers, - body: new Uint8Array(buffer), - }); + await putBuffer(uploadUrl, buffer, headers); } }