diff --git a/src/_internalUtils.ts b/src/_internalUtils.ts index 0ff34c8..7d99326 100644 --- a/src/_internalUtils.ts +++ b/src/_internalUtils.ts @@ -1,8 +1,6 @@ import { spawnSync, SpawnSyncOptionsWithStringEncoding } from 'child_process'; import crypto from 'crypto'; import fs from 'fs'; -import http from 'http'; -import https from 'https'; import util from 'util'; import zlib from 'zlib'; @@ -43,6 +41,37 @@ export async function retryWithBackoff(job: () => Promise, backoff: number return await job(); } +export const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; + +async function fetchOk(input: RequestInfo | URL, init?: RequestInit): Promise { + const response = await fetch(input, init); + if (!response.ok) { + const url = response.url || (input instanceof URL ? input.href : typeof input === 'string' ? input : input.url); + const body = await response.text().catch(() => ''); + throw new Error(response.status + ' ' + url + ' ' + body); + } + return response; +} + +export async function getJSON(input: RequestInfo | URL, init?: RequestInit, backoff: number[] = HTTP_BACKOFF): Promise { + return await retryWithBackoff(async () => { + const response = await fetchOk(input, init); + return await response.json() as T; + }, backoff); +} + +export async function putBuffer(input: RequestInfo | URL, body: Buffer, headers?: HeadersInit, backoff: number[] = HTTP_BACKOFF): Promise { + await retryWithBackoff(async () => { + const response = await fetchOk(input, { + method: 'PUT', + headers, + body: new Uint8Array(body), + }); + // Read response to ensure it completes. + await response.arrayBuffer(); + }, backoff); +} + export function shell(command: string, args?: string[], options?: SpawnSyncOptionsWithStringEncoding) { try { const result = spawnSync(command, args, { encoding: 'utf-8', ...options }); @@ -92,4 +121,4 @@ export function randomUUIDBase62(): string { } return chars.reverse().join(''); -} \ No newline at end of file +} diff --git a/src/fetchTestDurations.ts b/src/fetchTestDurations.ts index 801d74c..551ca16 100644 --- a/src/fetchTestDurations.ts +++ b/src/fetchTestDurations.ts @@ -1,6 +1,6 @@ import { FlakinessReport } from '@flakiness/flakiness-report'; import { URL } from 'url'; -import { compressTextAsync, retryWithBackoff, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, getJSON, putBuffer, sha1Text } from './_internalUtils.js'; import { GithubOIDC } from './githubOIDC.js'; type TestDurationsFetcherOptions = { @@ -35,9 +35,6 @@ export type FetchTestDurationsOptions = { flakinessAccessToken?: string; } -// Retry schedule for uploading the report to the presigned URL. -const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; - // The computed historical durations are not ready immediately after submit, so // we poll the download URL for up to ~90 seconds before giving up. const DOWNLOAD_BACKOFF = [Array(10).fill(1000), Array(10).fill(2000), Array(20).fill(3000)].flat(); @@ -107,19 +104,14 @@ class TestDurationsFetcher { private async _api(pathname: string, token: string, body?: any): Promise { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; - return await retryWithBackoff(async () => { - const response = await fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - }); - if (!response.ok) - throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`) - return response; - }, HTTP_BACKOFF).then(async response => await response.json()); + return await getJSON(url, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); } async fetch(): Promise { @@ -147,12 +139,7 @@ class TestDurationsFetcher { createResponse.testDurationsToken, ); - return await retryWithBackoff(async () => { - const response = await fetch(submitResponse.downloadUrl); - if (!response.ok) - throw new Error(`Request to ${submitResponse.downloadUrl} failed with ${response.status}`); - return response; - }, DOWNLOAD_BACKOFF).then(async response => await response.json() as FlakinessReport.Report); + return await getJSON(submitResponse.downloadUrl, undefined, DOWNLOAD_BACKOFF); } private async _uploadReport(data: string, uploadUrl: string) { @@ -162,16 +149,6 @@ class TestDurationsFetcher { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); - if (!response.ok) - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + await putBuffer(uploadUrl, compressed, headers); } } diff --git a/src/githubOIDC.ts b/src/githubOIDC.ts index 2db8174..ca6c524 100644 --- a/src/githubOIDC.ts +++ b/src/githubOIDC.ts @@ -1,3 +1,5 @@ +import { getJSON } from './_internalUtils.js'; + /** * Provides GitHub Actions OIDC (OpenID Connect) token exchange. * @@ -56,19 +58,18 @@ export class GithubOIDC { const url = new URL(this._requestUrl); url.searchParams.set('audience', flakinessProject); - const response = await fetch(url, { - headers: { - 'Authorization': `bearer ${this._requestToken}`, - 'Accept': 'application/json; api-version=2.0', - }, - }); - - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`Failed to request GitHub OIDC token: ${response.status} ${body}`); + let json: { value?: string }; + try { + json = await getJSON<{ value?: string }>(url, { + headers: { + 'Authorization': `bearer ${this._requestToken}`, + 'Accept': 'application/json; api-version=2.0', + }, + }); + } catch (error: any) { + throw new Error(`Failed to request GitHub OIDC token: ${error.message || String(error)}`); } - const json = await response.json() as { value?: string }; if (!json.value) throw new Error('GitHub OIDC token response did not contain a token value.'); diff --git a/src/uploadReport.ts b/src/uploadReport.ts index 6f29a8c..365ae5f 100644 --- a/src/uploadReport.ts +++ b/src/uploadReport.ts @@ -3,7 +3,7 @@ import assert from 'assert'; import fs from 'fs'; import { URL } from 'url'; import { GithubOIDC } from './githubOIDC.js'; -import { compressTextAsync, retryWithBackoff, sha1File, sha1Text } from './_internalUtils.js'; +import { compressTextAsync, getJSON, putBuffer, sha1File, sha1Text } from './_internalUtils.js'; type ReportUploaderOptions = { flakinessEndpoint: string; @@ -293,8 +293,6 @@ export async function uploadReport( } } -const HTTP_BACKOFF = [100, 500, 1000, 1000, 1000, 1000]; - class ReportUpload { private _report: FlakinessReport.Report; private _attachments: Attachment[]; @@ -309,23 +307,25 @@ class ReportUpload { private async _api(pathname: string, token: string, body?: any): Promise<{ result?: OUTPUT, error?: string }> { const url = new URL(this._options.flakinessEndpoint); url.pathname = pathname; - return await fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - }).then(async response => !response.ok ? { - result: undefined, - error: response.status + ' ' + url.href + ' ' + await response.text(), - } : { - result: await response.json() as OUTPUT, - error: undefined, - }).catch(error => ({ - result: undefined, - error, - })); + try { + const result = await getJSON(url, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + return { + result, + error: undefined, + }; + } catch (error: any) { + return { + result: undefined, + error: error instanceof Error ? error.message : String(error), + }; + } } async upload(): Promise<{ success: false, message?: string } | { success: true, reportUrl: string }> { @@ -370,18 +370,7 @@ class ReportUpload { 'Content-Length': Buffer.byteLength(compressed) + '', 'Content-Encoding': 'br', }; - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: Buffer.from(compressed), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + await putBuffer(uploadUrl, compressed, headers); } private async _uploadAttachment(attachment: Attachment, uploadUrl: string) { @@ -393,22 +382,11 @@ class ReportUpload { ; // Stream file only if there's attachment path and we should NOT compress it. if (!compressable && attachment.type === 'file') { - await retryWithBackoff(async () => { - const fileBuffer = await fs.promises.readFile(attachment.path); - const response = await fetch(uploadUrl, { - method: 'PUT', - headers: { - 'Content-Type': attachment.contentType, - 'Content-Length': fileBuffer.length + '', - }, - body: new Uint8Array(fileBuffer), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + const fileBuffer = await fs.promises.readFile(attachment.path); + await putBuffer(uploadUrl, fileBuffer, { + 'Content-Type': attachment.contentType, + 'Content-Length': fileBuffer.length + '', + }); return; } let buffer = attachment.type === 'buffer' ? attachment.body : await fs.promises.readFile(attachment.path); @@ -427,17 +405,6 @@ class ReportUpload { headers['Content-Encoding'] = encoding; } - await retryWithBackoff(async () => { - const response = await fetch(uploadUrl, { - method: 'PUT', - headers, - body: new Uint8Array(buffer), - }); - if (!response.ok) { - throw new Error(`Request to ${uploadUrl} failed with ${response.status}`); - } - // Read response to ensure it completes - await response.arrayBuffer(); - }, HTTP_BACKOFF); + await putBuffer(uploadUrl, buffer, headers); } }