From 5e3da6134dd8cd3907cb03ab02af594766e870b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 10:29:48 +0200 Subject: [PATCH 01/17] Add chunked media upload to bypass Vercel 4.5 MB request limit Vercel serverless functions cap request bodies at 4.5 MB, which after base64 overhead limited media uploads to ~3.3 MB. The browser now slices files into ~3 MB chunks, POSTs each to /api/upload/chunk (multipart), and then calls /api/upload/finalize which reassembles, pushes to GitHub, and deletes the chunks. Chunks are staged in a new upload_chunk table (Postgres text, base64). Stale chunks are reaped opportunistically on each insert. Max file size is 50 MB / 50 chunks, configurable in both endpoints and the client. githubSaveFile is extracted to lib/utils/github-save-file.ts so both the existing files endpoint and the new finalize endpoint share rename-on- conflict logic. --- .../[repo]/[branch]/files/[path]/route.ts | 158 +- app/api/upload/chunk/route.ts | 59 + app/api/upload/finalize/route.ts | 171 ++ components/media/media-upload.tsx | 47 +- db/migrations/0013_upload_chunks.sql | 12 + db/migrations/meta/0013_snapshot.json | 1611 +++++++++++++++++ db/migrations/meta/_journal.json | 7 + db/schema.ts | 15 +- lib/utils/github-save-file.ts | 156 ++ scripts/check-chunk-assembly.mjs | 40 + 10 files changed, 2101 insertions(+), 175 deletions(-) create mode 100644 app/api/upload/chunk/route.ts create mode 100644 app/api/upload/finalize/route.ts create mode 100644 db/migrations/0013_upload_chunks.sql create mode 100644 db/migrations/meta/0013_snapshot.json create mode 100644 lib/utils/github-save-file.ts create mode 100644 scripts/check-chunk-assembly.mjs diff --git a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts index 372453981..df6e2704c 100644 --- a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts @@ -14,6 +14,7 @@ import { createHttpError, toErrorResponse } from "@/lib/api-error"; import mergeWith from "lodash.mergewith"; import { buildCommitTokens, resolveCommitIdentity, resolveCommitMessage } from "@/lib/commit-message"; import { requireApiUserSession } from "@/lib/session-server"; +import { githubSaveFile } from "@/lib/utils/github-save-file"; /** * Create, update and delete individual files in a GitHub repository. @@ -289,163 +290,6 @@ export async function POST( } }; -// Helper function to save a file to GitHub (with retry logic for new files) -const githubSaveFile = async ( - token: string, - owner: string, - repo: string, - branch: string, - path: string, - contentBase64: string, - sha?: string, - options?: { - configObject?: Record; - templatesOverride?: Record; - contentName?: string; - user?: string; - onConflict?: "rename" | "error"; - committer?: { name: string; email: string }; - }, -) => { - // We disable retries for 409 errors as it means the file has changed (conflict on SHA) - const octokit = createOctokitInstance(token, { retry: { doNotRetry: [409] } }); - - const message = resolveCommitMessage({ - configObject: options?.configObject, - templatesOverride: options?.templatesOverride, - action: sha ? "update" : "create", - tokens: buildCommitTokens({ - action: sha ? "update" : "create", - owner, - repo, - branch, - path, - contentName: options?.contentName, - user: options?.user, - userName: options?.committer?.name, - userEmail: options?.committer?.email, - }), - }); - - try { - // First attempt: try with original path - const response = await octokit.rest.repos.createOrUpdateFileContents({ - owner, - repo, - path, - message, - content: contentBase64, - branch, - sha: sha || undefined, - committer: options?.committer, - }); - - if (response.data.content && response.data.commit) { - return response; - } - throw new Error("Invalid response structure"); - } catch (error: any) { - const githubMessage = typeof error?.response?.data?.message === "string" - ? error.response.data.message - : undefined; - - if (error.status === 409) { - if (githubMessage?.includes("Repository rule violations found")) { - throw createHttpError( - "This repository requires changes through a pull request. Save to a different branch or fork, or ask a maintainer to relax the repository rule for direct edits.", - 409, - ); - } - - if (sha) { - throw createHttpError( - "File has changed since you last loaded it. Please refresh the page and try again.", - 409, - ); - } - } - - // Only handle 422 errors for new files (no sha) - if (error.status === 422 && !sha) { - if (options?.onConflict === "error") { - throw createHttpError(`File \"${path}\" already exists.`, 409); - } - - // Get directory contents to find next available name - const parentDir = getParentPath(path); - const { data } = await octokit.rest.repos.getContent({ - owner, - repo, - path: parentDir || '.', - ref: branch, - }); - - if (!Array.isArray(data)) { - throw new Error('Expected directory listing'); - } - - const basename = path.split('/').pop() || ""; - const lastDotIndex = basename.lastIndexOf("."); - const filename = lastDotIndex > 0 ? basename.slice(0, lastDotIndex) : basename; - const extension = lastDotIndex > 0 ? basename.slice(lastDotIndex + 1) : ""; - const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const escapedFilename = escapeRegExp(filename); - const escapedExtension = escapeRegExp(extension); - const pattern = extension - ? new RegExp(`^${escapedFilename}-(\\d+)\\.${escapedExtension}$`) - : new RegExp(`^${escapedFilename}-(\\d+)$`); - const maxNumber = Math.max(0, ...data - .map(file => { - const match = file.name.match(pattern); - return match ? parseInt(match[1], 10) : 0; - })); - - // Try up to 3 times with incrementing numbers - for (let i = 1; i <= 3; i++) { - const candidateFilename = extension - ? `${filename}-${maxNumber + i}.${extension}` - : `${filename}-${maxNumber + i}`; - const newPath = `${parentDir ? parentDir + '/' : ''}${candidateFilename}`; - const fallbackMessage = resolveCommitMessage({ - configObject: options?.configObject, - templatesOverride: options?.templatesOverride, - action: "create", - tokens: buildCommitTokens({ - action: "create", - owner, - repo, - branch, - path: newPath, - contentName: options?.contentName, - user: options?.user, - userName: options?.committer?.name, - userEmail: options?.committer?.email, - }), - }); - try { - const response = await octokit.rest.repos.createOrUpdateFileContents({ - owner, - repo, - path: newPath, - message: fallbackMessage, - content: contentBase64, - branch, - committer: options?.committer, - }); - - if (response.data.content && response.data.commit) { - return response; - } - } catch (error: any) { - if (i === 3 || error.status !== 422) throw error; - // Continue to next attempt if 422 (file already exists) - } - } - } - throw error; - } -}; - export async function DELETE( request: NextRequest, context: { params: Promise<{ owner: string, repo: string, branch: string, path: string }> } diff --git a/app/api/upload/chunk/route.ts b/app/api/upload/chunk/route.ts new file mode 100644 index 000000000..ac93b562b --- /dev/null +++ b/app/api/upload/chunk/route.ts @@ -0,0 +1,59 @@ +import { db } from "@/db"; +import { uploadChunkTable } from "@/db/schema"; +import { eq, lt } from "drizzle-orm"; +import { requireApiUserSession } from "@/lib/session-server"; +import { createHttpError, toErrorResponse } from "@/lib/api-error"; + +const MAX_CHUNK_BYTES = 4 * 1024 * 1024; +const STALE_CHUNK_AGE_MS = 60 * 60 * 1000; + +export async function POST(request: Request) { + try { + const sessionResult = await requireApiUserSession(); + if ("response" in sessionResult) return sessionResult.response; + const user = sessionResult.user; + + const formData = await request.formData(); + const uploadId = formData.get("uploadId"); + const idxRaw = formData.get("idx"); + const chunk = formData.get("chunk"); + + if (typeof uploadId !== "string" || uploadId.length === 0 || uploadId.length > 64) { + throw createHttpError(`Invalid "uploadId".`, 400); + } + const idx = typeof idxRaw === "string" ? parseInt(idxRaw, 10) : NaN; + if (!Number.isInteger(idx) || idx < 0 || idx > 9999) { + throw createHttpError(`Invalid "idx".`, 400); + } + if (!(chunk instanceof Blob)) { + throw createHttpError(`Invalid "chunk".`, 400); + } + if (chunk.size === 0 || chunk.size > MAX_CHUNK_BYTES) { + throw createHttpError(`Chunk size must be between 1 and ${MAX_CHUNK_BYTES} bytes.`, 413); + } + + const buffer = Buffer.from(await chunk.arrayBuffer()); + const base64 = buffer.toString("base64"); + + await db.insert(uploadChunkTable).values({ + uploadId, + userId: user.id, + chunkIdx: idx, + data: base64, + }).onConflictDoUpdate({ + target: [uploadChunkTable.uploadId, uploadChunkTable.chunkIdx], + set: { data: base64, createdAt: new Date() }, + setWhere: eq(uploadChunkTable.userId, user.id), + }); + + // ponytail: oportunistic stale-chunk cleanup; cron-free housekeeping for Hobby plan + await db.delete(uploadChunkTable).where( + lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), + ); + + return Response.json({ status: "success" }); + } catch (error: any) { + if (!error?.status || error.status >= 500) console.error(error); + return toErrorResponse(error); + } +} diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts new file mode 100644 index 000000000..b8a645aa1 --- /dev/null +++ b/app/api/upload/finalize/route.ts @@ -0,0 +1,171 @@ +import { db } from "@/db"; +import { uploadChunkTable } from "@/db/schema"; +import { and, asc, eq } from "drizzle-orm"; +import { requireApiUserSession } from "@/lib/session-server"; +import { createHttpError, toErrorResponse } from "@/lib/api-error"; +import { getToken } from "@/lib/token"; +import { getConfig } from "@/lib/config-store"; +import { getSchemaByName } from "@/lib/schema"; +import { getFileExtension, getFileName, normalizePath } from "@/lib/utils/file"; +import { resolveCommitIdentity } from "@/lib/commit-message"; +import { githubSaveFile } from "@/lib/utils/github-save-file"; +import { updateFileCache } from "@/lib/github-cache-file"; + +const MAX_TOTAL_BYTES = 50 * 1024 * 1024; +const MAX_CHUNKS = 50; + +export async function POST(request: Request) { + try { + const sessionResult = await requireApiUserSession(); + if ("response" in sessionResult) return sessionResult.response; + const user = sessionResult.user; + + const data: any = await request.json(); + const uploadId = typeof data.uploadId === "string" ? data.uploadId : ""; + const totalChunks = Number.isInteger(data.totalChunks) ? data.totalChunks : -1; + const owner = typeof data.owner === "string" ? data.owner : ""; + const repo = typeof data.repo === "string" ? data.repo : ""; + const branch = typeof data.branch === "string" ? data.branch : ""; + const path = typeof data.path === "string" ? data.path : ""; + const name = typeof data.name === "string" ? data.name : ""; + const sha = typeof data.sha === "string" ? data.sha : undefined; + const onConflict = data.onConflict === "error" ? "error" : "rename"; + + if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); + if (totalChunks < 1 || totalChunks > MAX_CHUNKS) { + throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); + } + if (!owner || !repo || !branch || !path || !name) { + throw createHttpError(`Missing required fields.`, 400); + } + + const { token } = await getToken(user, owner, repo, true); + if (!token) throw new Error("Token not found"); + + const normalizedPath = normalizePath(path); + + const config = await getConfig(owner, repo, branch, { getToken: async () => token }); + if (!config) throw new Error(`Configuration not found for ${owner}/${repo}/${branch}.`); + + const schema = getSchemaByName(config.object, name, "media"); + if (!schema) throw new Error(`Media schema not found for ${name}.`); + if (!normalizedPath.startsWith(schema.input)) { + throw new Error(`Invalid path "${path}" for media "${name}".`); + } + if ( + schema.extensions?.length > 0 && + !schema.extensions.includes(getFileExtension(normalizedPath)) + ) { + throw new Error(`Invalid extension "${getFileExtension(normalizedPath)}" for media.`); + } + if (getFileName(normalizedPath) === ".gitkeep") { + throw createHttpError(`Use the files endpoint to create empty folders.`, 400); + } + + const chunks = await db + .select({ chunkIdx: uploadChunkTable.chunkIdx, data: uploadChunkTable.data }) + .from(uploadChunkTable) + .where(and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + )) + .orderBy(asc(uploadChunkTable.chunkIdx)); + + if (chunks.length !== totalChunks) { + throw createHttpError( + `Expected ${totalChunks} chunks but found ${chunks.length}.`, + 400, + ); + } + for (let i = 0; i < chunks.length; i++) { + if (chunks[i].chunkIdx !== i) { + throw createHttpError(`Missing chunk at index ${i}.`, 400); + } + } + + const buffers = chunks.map(c => Buffer.from(c.data, "base64")); + const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); + if (totalSize > MAX_TOTAL_BYTES) { + throw createHttpError( + `File too large (${totalSize} bytes). Max ${MAX_TOTAL_BYTES} bytes.`, + 413, + ); + } + const contentBase64 = Buffer.concat(buffers).toString("base64"); + + const schemaCommitTemplates = schema?.commit?.templates; + const schemaCommitIdentity = schema?.commit?.identity; + const commitIdentity = resolveCommitIdentity({ + configObject: config.object, + identityOverride: schemaCommitIdentity, + }); + const committer = (commitIdentity === "user" && user.email) + ? { name: user.name?.trim() || user.email, email: user.email } + : undefined; + + const response = await githubSaveFile( + token, + owner, + repo, + branch, + normalizedPath, + contentBase64, + sha, + { + configObject: config.object, + templatesOverride: schemaCommitTemplates, + contentName: name, + user: user.email || user.name || String(user.id || ""), + onConflict, + committer, + } + ); + + const savedPath = response?.data.content?.path; + + if (response?.data.content && response?.data.commit) { + await updateFileCache( + 'media', + owner, + repo, + branch, + { + type: sha ? 'modify' : 'add', + path: response.data.content.path!, + sha: response.data.content.sha!, + content: Buffer.from(contentBase64, 'base64').toString('utf-8'), + size: response.data.content.size, + downloadUrl: response.data.content.download_url, + commit: { + sha: response.data.commit.sha!, + timestamp: new Date(response.data.commit.committer?.date ?? new Date().toISOString()).getTime() + } + } + ); + } + + await db.delete(uploadChunkTable).where(and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + )); + + return Response.json({ + status: "success", + message: savedPath !== normalizedPath + ? `File "${normalizedPath}" saved successfully but renamed to "${savedPath}" to avoid naming conflict.` + : `File "${normalizedPath}" saved successfully.`, + data: { + type: response?.data.content?.type, + sha: response?.data.content?.sha, + name: response?.data.content?.name, + path: savedPath, + extension: getFileExtension(response?.data.content?.name || ""), + size: response?.data.content?.size, + url: response?.data.content?.download_url, + } + }); + } catch (error: any) { + if (!error?.status || error.status >= 500) console.error(error); + return toErrorResponse(error); + } +} diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index a9d4ce289..40be00353 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -64,6 +64,9 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }, [extensions, configMedia?.extensions]); const handleFiles = useCallback(async (files: File[]) => { + const CHUNK_BYTES = 3 * 1024 * 1024; + const MAX_TOTAL_BYTES = 50 * 1024 * 1024; + try { for (const file of files) { const uploadFilename = getUploadFileName( @@ -72,32 +75,42 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple ); const uploadPromise = (async () => { - const content = await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const base64Content = (reader.result as string).replace(/^(.+,)/, ""); - resolve(base64Content); - }; - reader.onerror = () => reject(new Error("Failed to read file")); - reader.readAsDataURL(file); - }); + if (file.size === 0) throw new Error("File is empty"); + if (file.size > MAX_TOTAL_BYTES) { + throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); + } + + const uploadId = crypto.randomUUID(); + const totalChunks = Math.ceil(file.size / CHUNK_BYTES); + + for (let idx = 0; idx < totalChunks; idx++) { + const start = idx * CHUNK_BYTES; + const end = Math.min(start + CHUNK_BYTES, file.size); + const blob = file.slice(start, end); + const form = new FormData(); + form.set("uploadId", uploadId); + form.set("idx", String(idx)); + form.set("chunk", blob); + const chunkResponse = await fetch("/api/upload/chunk", { method: "POST", body: form }); + await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); + } const fullPath = joinPathSegments([path ?? "", uploadFilename]); - const response = await fetch(`/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullPath)}`, { + const finalizeResponse = await fetch("/api/upload/finalize", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - type: "media", + uploadId, + totalChunks, + owner: config.owner, + repo: config.repo, + branch: config.branch, + path: fullPath, name: configMedia.name, - content, }), }); - const data = await requireApiSuccess( - response, - "Failed to upload file", - ); - + const data = await requireApiSuccess(finalizeResponse, "Failed to upload file"); return data.data as FileSaveData; })(); diff --git a/db/migrations/0013_upload_chunks.sql b/db/migrations/0013_upload_chunks.sql new file mode 100644 index 000000000..682a5ff42 --- /dev/null +++ b/db/migrations/0013_upload_chunks.sql @@ -0,0 +1,12 @@ +CREATE TABLE "upload_chunk" ( + "id" serial PRIMARY KEY NOT NULL, + "upload_id" text NOT NULL, + "user_id" text NOT NULL, + "chunk_idx" integer NOT NULL, + "data" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "upload_chunk" ADD CONSTRAINT "upload_chunk_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "uq_upload_chunk_uploadId_chunkIdx" ON "upload_chunk" USING btree ("upload_id","chunk_idx");--> statement-breakpoint +CREATE INDEX "idx_upload_chunk_createdAt" ON "upload_chunk" USING btree ("created_at"); \ No newline at end of file diff --git a/db/migrations/meta/0013_snapshot.json b/db/migrations/meta/0013_snapshot.json new file mode 100644 index 000000000..72da85a09 --- /dev/null +++ b/db/migrations/meta/0013_snapshot.json @@ -0,0 +1,1611 @@ +{ + "id": "5147365d-5a58-4806-8db4-ec7e4d10989b", + "prevId": "4655a2e1-f322-41b1-b587-9fa5d038d98a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_account_userId": { + "name": "idx_account_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_providerId": { + "name": "idx_account_providerId", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_run": { + "name": "action_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_ref": { + "name": "workflow_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_type": { + "name": "context_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_name": { + "name": "context_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context_path": { + "name": "context_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow": { + "name": "workflow", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_run_id": { + "name": "workflow_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conclusion": { + "name": "conclusion", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triggered_by": { + "name": "triggered_by", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "failure": { + "name": "failure", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_action_run_owner_repo_createdAt": { + "name": "idx_action_run_owner_repo_createdAt", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_owner_repo_actionName": { + "name": "idx_action_run_owner_repo_actionName", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_owner_repo_status": { + "name": "idx_action_run_owner_repo_status", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_context": { + "name": "idx_action_run_context", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_workflowRunId": { + "name": "idx_action_run_workflowRunId", + "columns": [ + { + "expression": "workflow_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_file_meta": { + "name": "cache_file_meta", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'branch'" + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_timestamp": { + "name": "commit_timestamp", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ok'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_cache_file_meta_owner_repo_branch_path_context": { + "name": "idx_cache_file_meta_owner_repo_branch_path_context", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_file": { + "name": "cache_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'collection'" + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_path": { + "name": "parent_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_timestamp": { + "name": "commit_timestamp", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_cache_file_owner_repo_branch_parentPath": { + "name": "idx_cache_file_owner_repo_branch_parentPath", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_cache_file_owner_repo_branch_path": { + "name": "idx_cache_file_owner_repo_branch_path", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_permission": { + "name": "cache_permission", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "github_id": { + "name": "github_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_updated": { + "name": "last_updated", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_cache_permission_githubId_owner_repo": { + "name": "idx_cache_permission_githubId_owner_repo", + "columns": [ + { + "expression": "github_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collaborator_invite": { + "name": "collaborator_invite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uq_collaborator_invite_token": { + "name": "uq_collaborator_invite_token", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_collaborator_invite_owner_repo_email": { + "name": "idx_collaborator_invite_owner_repo_email", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_collaborator_invite_owner_repo_email_ci": { + "name": "uq_collaborator_invite_owner_repo_email_ci", + "columns": [ + { + "expression": "lower(\"owner\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"repo\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collaborator": { + "name": "collaborator", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "repo_id": { + "name": "repo_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_collaborator_owner_repo_email": { + "name": "idx_collaborator_owner_repo_email", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_collaborator_userId": { + "name": "idx_collaborator_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_collaborator_owner_repo_email_ci": { + "name": "uq_collaborator_owner_repo_email_ci", + "columns": [ + { + "expression": "lower(\"owner\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"repo\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collaborator_user_id_user_id_fk": { + "name": "collaborator_user_id_user_id_fk", + "tableFrom": "collaborator", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "collaborator_invited_by_user_id_fk": { + "name": "collaborator_invited_by_user_id_fk", + "tableFrom": "collaborator", + "tableTo": "user", + "columnsFrom": [ + "invited_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.config": { + "name": "config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_config_owner_repo_branch": { + "name": "idx_config_owner_repo_branch", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.github_installation_token": { + "name": "github_installation_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "uq_github_installation_token_installationId": { + "name": "uq_github_installation_token_installationId", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_session_userId": { + "name": "idx_session_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upload_chunk": { + "name": "upload_chunk", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "upload_id": { + "name": "upload_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_idx": { + "name": "chunk_idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uq_upload_chunk_uploadId_chunkIdx": { + "name": "uq_upload_chunk_uploadId_chunkIdx", + "columns": [ + { + "expression": "upload_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_idx", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_upload_chunk_createdAt": { + "name": "idx_upload_chunk_createdAt", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "upload_chunk_user_id_user_id_fk": { + "name": "upload_chunk_user_id_user_id_fk", + "tableFrom": "upload_chunk", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_username": { + "name": "github_username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_verification_identifier": { + "name": "idx_verification_identifier", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/db/migrations/meta/_journal.json b/db/migrations/meta/_journal.json index 8f67a9031..e17433125 100644 --- a/db/migrations/meta/_journal.json +++ b/db/migrations/meta/_journal.json @@ -92,6 +92,13 @@ "when": 1778731770759, "tag": "0012_collaborator_invites", "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1782289030662, + "tag": "0013_upload_chunks", + "breakpoints": true } ] } \ No newline at end of file diff --git a/db/schema.ts b/db/schema.ts index b301fa740..2c80ff8f0 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -209,6 +209,18 @@ const actionRunTable = pgTable("action_run", { idx_action_run_workflowRunId: uniqueIndex("idx_action_run_workflowRunId").on(table.workflowRunId), })); +const uploadChunkTable = pgTable("upload_chunk", { + id: serial("id").primaryKey(), + uploadId: text("upload_id").notNull(), + userId: text("user_id").notNull().references(() => userTable.id, { onDelete: "cascade" }), + chunkIdx: integer("chunk_idx").notNull(), + data: text("data").notNull(), + createdAt: timestamp("created_at").notNull().defaultNow(), +}, (table) => [ + uniqueIndex("uq_upload_chunk_uploadId_chunkIdx").on(table.uploadId, table.chunkIdx), + index("idx_upload_chunk_createdAt").on(table.createdAt), +]); + export { userTable, sessionTable, @@ -221,5 +233,6 @@ export { cacheFileTable, cacheFileMetaTable, cachePermissionTable, - actionRunTable + actionRunTable, + uploadChunkTable }; diff --git a/lib/utils/github-save-file.ts b/lib/utils/github-save-file.ts new file mode 100644 index 000000000..8e53323f7 --- /dev/null +++ b/lib/utils/github-save-file.ts @@ -0,0 +1,156 @@ +import { createOctokitInstance } from "@/lib/utils/octokit"; +import { createHttpError } from "@/lib/api-error"; +import { getParentPath } from "@/lib/utils/file"; +import { buildCommitTokens, resolveCommitMessage } from "@/lib/commit-message"; + +type GithubSaveFileOptions = { + configObject?: Record; + templatesOverride?: Record; + contentName?: string; + user?: string; + onConflict?: "rename" | "error"; + committer?: { name: string; email: string }; +}; + +export const githubSaveFile = async ( + token: string, + owner: string, + repo: string, + branch: string, + path: string, + contentBase64: string, + sha?: string, + options?: GithubSaveFileOptions, +) => { + const octokit = createOctokitInstance(token, { retry: { doNotRetry: [409] } }); + + const message = resolveCommitMessage({ + configObject: options?.configObject, + templatesOverride: options?.templatesOverride, + action: sha ? "update" : "create", + tokens: buildCommitTokens({ + action: sha ? "update" : "create", + owner, + repo, + branch, + path, + contentName: options?.contentName, + user: options?.user, + userName: options?.committer?.name, + userEmail: options?.committer?.email, + }), + }); + + try { + const response = await octokit.rest.repos.createOrUpdateFileContents({ + owner, + repo, + path, + message, + content: contentBase64, + branch, + sha: sha || undefined, + committer: options?.committer, + }); + + if (response.data.content && response.data.commit) { + return response; + } + throw new Error("Invalid response structure"); + } catch (error: any) { + const githubMessage = typeof error?.response?.data?.message === "string" + ? error.response.data.message + : undefined; + + if (error.status === 409) { + if (githubMessage?.includes("Repository rule violations found")) { + throw createHttpError( + "This repository requires changes through a pull request. Save to a different branch or fork, or ask a maintainer to relax the repository rule for direct edits.", + 409, + ); + } + + if (sha) { + throw createHttpError( + "File has changed since you last loaded it. Please refresh the page and try again.", + 409, + ); + } + } + + if (error.status === 422 && !sha) { + if (options?.onConflict === "error") { + throw createHttpError(`File \"${path}\" already exists.`, 409); + } + + const parentDir = getParentPath(path); + const { data } = await octokit.rest.repos.getContent({ + owner, + repo, + path: parentDir || '.', + ref: branch, + }); + + if (!Array.isArray(data)) { + throw new Error('Expected directory listing'); + } + + const basename = path.split('/').pop() || ""; + const lastDotIndex = basename.lastIndexOf("."); + const filename = lastDotIndex > 0 ? basename.slice(0, lastDotIndex) : basename; + const extension = lastDotIndex > 0 ? basename.slice(lastDotIndex + 1) : ""; + const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const escapedFilename = escapeRegExp(filename); + const escapedExtension = escapeRegExp(extension); + const pattern = extension + ? new RegExp(`^${escapedFilename}-(\\d+)\\.${escapedExtension}$`) + : new RegExp(`^${escapedFilename}-(\\d+)$`); + const maxNumber = Math.max(0, ...data + .map(file => { + const match = file.name.match(pattern); + return match ? parseInt(match[1], 10) : 0; + })); + + for (let i = 1; i <= 3; i++) { + const candidateFilename = extension + ? `${filename}-${maxNumber + i}.${extension}` + : `${filename}-${maxNumber + i}`; + const newPath = `${parentDir ? parentDir + '/' : ''}${candidateFilename}`; + const fallbackMessage = resolveCommitMessage({ + configObject: options?.configObject, + templatesOverride: options?.templatesOverride, + action: "create", + tokens: buildCommitTokens({ + action: "create", + owner, + repo, + branch, + path: newPath, + contentName: options?.contentName, + user: options?.user, + userName: options?.committer?.name, + userEmail: options?.committer?.email, + }), + }); + try { + const response = await octokit.rest.repos.createOrUpdateFileContents({ + owner, + repo, + path: newPath, + message: fallbackMessage, + content: contentBase64, + branch, + committer: options?.committer, + }); + + if (response.data.content && response.data.commit) { + return response; + } + } catch (error: any) { + if (i === 3 || error.status !== 422) throw error; + } + } + } + throw error; + } +}; diff --git a/scripts/check-chunk-assembly.mjs b/scripts/check-chunk-assembly.mjs new file mode 100644 index 000000000..87022e35b --- /dev/null +++ b/scripts/check-chunk-assembly.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node +// Self-check for chunk split + base64 round-trip + concat. +// Run: node scripts/check-chunk-assembly.mjs + +import { randomBytes } from "node:crypto"; +import { strict as assert } from "node:assert"; + +const CHUNK_BYTES = 3 * 1024 * 1024; + +function splitToBase64Chunks(buffer, chunkBytes) { + const chunks = []; + for (let start = 0; start < buffer.length; start += chunkBytes) { + const slice = buffer.subarray(start, Math.min(start + chunkBytes, buffer.length)); + chunks.push(slice.toString("base64")); + } + return chunks; +} + +function assembleFromBase64Chunks(chunks) { + return Buffer.concat(chunks.map((c) => Buffer.from(c, "base64"))); +} + +function check(label, sizeBytes) { + const original = randomBytes(sizeBytes); + const chunks = splitToBase64Chunks(original, CHUNK_BYTES); + const expectedChunks = Math.max(1, Math.ceil(sizeBytes / CHUNK_BYTES)); + assert.equal(chunks.length, expectedChunks, `${label}: chunk count`); + const reassembled = assembleFromBase64Chunks(chunks); + assert.equal(reassembled.length, original.length, `${label}: length`); + assert.ok(reassembled.equals(original), `${label}: bytes match`); + console.log(`ok ${label} (${sizeBytes} bytes, ${chunks.length} chunks)`); +} + +check("single chunk", 1024); +check("exactly one chunk", CHUNK_BYTES); +check("two chunks, second partial", CHUNK_BYTES + 1); +check("many chunks", CHUNK_BYTES * 7 + 123); +check("size 1", 1); + +console.log("\nAll chunk-assembly checks passed."); From 21a28388c199250d7786b5927dcd2bb851372a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 10:45:59 +0200 Subject: [PATCH 02/17] Parallelize chunk uploads and move cleanup to after() Client now uploads chunks in batches of 4 instead of sequentially. Server moves opportunistic stale-chunk cleanup (chunk endpoint) and per-upload chunk deletion (finalize endpoint) into next/server `after`, so neither blocks the response. --- app/api/upload/chunk/route.ts | 15 +++++++++++---- app/api/upload/finalize/route.ts | 16 ++++++++++++---- components/media/media-upload.tsx | 12 +++++++++++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/app/api/upload/chunk/route.ts b/app/api/upload/chunk/route.ts index ac93b562b..b7c090568 100644 --- a/app/api/upload/chunk/route.ts +++ b/app/api/upload/chunk/route.ts @@ -1,3 +1,4 @@ +import { after } from "next/server"; import { db } from "@/db"; import { uploadChunkTable } from "@/db/schema"; import { eq, lt } from "drizzle-orm"; @@ -46,10 +47,16 @@ export async function POST(request: Request) { setWhere: eq(uploadChunkTable.userId, user.id), }); - // ponytail: oportunistic stale-chunk cleanup; cron-free housekeeping for Hobby plan - await db.delete(uploadChunkTable).where( - lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), - ); + // ponytail: oportunistic stale-chunk cleanup runs after the response; cron-free housekeeping for Hobby + after(async () => { + try { + await db.delete(uploadChunkTable).where( + lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), + ); + } catch (error) { + console.error("Stale chunk cleanup failed", error); + } + }); return Response.json({ status: "success" }); } catch (error: any) { diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index b8a645aa1..a853fa1cd 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -1,3 +1,4 @@ +import { after } from "next/server"; import { db } from "@/db"; import { uploadChunkTable } from "@/db/schema"; import { and, asc, eq } from "drizzle-orm"; @@ -144,10 +145,17 @@ export async function POST(request: Request) { ); } - await db.delete(uploadChunkTable).where(and( - eq(uploadChunkTable.uploadId, uploadId), - eq(uploadChunkTable.userId, user.id), - )); + // ponytail: chunks already pushed to GitHub; delete in background, TTL sweep catches any miss + after(async () => { + try { + await db.delete(uploadChunkTable).where(and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + )); + } catch (error) { + console.error("Chunk cleanup after finalize failed", error); + } + }); return Response.json({ status: "success", diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 40be00353..4fcc45277 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -66,6 +66,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const handleFiles = useCallback(async (files: File[]) => { const CHUNK_BYTES = 3 * 1024 * 1024; const MAX_TOTAL_BYTES = 50 * 1024 * 1024; + const CHUNK_CONCURRENCY = 4; try { for (const file of files) { @@ -83,7 +84,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); - for (let idx = 0; idx < totalChunks; idx++) { + const uploadChunk = async (idx: number) => { const start = idx * CHUNK_BYTES; const end = Math.min(start + CHUNK_BYTES, file.size); const blob = file.slice(start, end); @@ -93,6 +94,15 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple form.set("chunk", blob); const chunkResponse = await fetch("/api/upload/chunk", { method: "POST", body: form }); await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); + }; + + // ponytail: batched parallelism (4); switch to rolling pool if uneven chunk times matter + for (let i = 0; i < totalChunks; i += CHUNK_CONCURRENCY) { + const batch = []; + for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { + batch.push(uploadChunk(j)); + } + await Promise.all(batch); } const fullPath = joinPathSegments([path ?? "", uploadFilename]); From 3d4672abd3d3a59ae4cf0d9661147d5d537ac3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 11:48:39 +0200 Subject: [PATCH 03/17] Skip DB staging for media files under 3 MB When the file fits in Vercel's 4.5 MB request body (after base64 overhead and JSON envelope), upload directly via the existing files endpoint instead of the chunked path. Cuts DB write/read traffic by the share of small uploads, which is most of them in practice. --- components/media/media-upload.tsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 4fcc45277..97edcb02f 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -64,16 +64,26 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }, [extensions, configMedia?.extensions]); const handleFiles = useCallback(async (files: File[]) => { + // ponytail: 3 MB keeps base64 + JSON envelope under Vercel's 4.5 MB body limit + const DIRECT_UPLOAD_BYTES = 3 * 1024 * 1024; const CHUNK_BYTES = 3 * 1024 * 1024; const MAX_TOTAL_BYTES = 50 * 1024 * 1024; const CHUNK_CONCURRENCY = 4; + const readAsBase64 = (file: File) => new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve((reader.result as string).replace(/^(.+,)/, "")); + reader.onerror = () => reject(new Error("Failed to read file")); + reader.readAsDataURL(file); + }); + try { for (const file of files) { const uploadFilename = getUploadFileName( file.name, rename ?? configMedia?.rename, ); + const fullPath = joinPathSegments([path ?? "", uploadFilename]); const uploadPromise = (async () => { if (file.size === 0) throw new Error("File is empty"); @@ -81,6 +91,17 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); } + if (file.size <= DIRECT_UPLOAD_BYTES) { + const content = await readAsBase64(file); + const response = await fetch(`/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullPath)}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ type: "media", name: configMedia.name, content }), + }); + const data = await requireApiSuccess(response, "Failed to upload file"); + return data.data as FileSaveData; + } + const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); @@ -105,7 +126,6 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple await Promise.all(batch); } - const fullPath = joinPathSegments([path ?? "", uploadFilename]); const finalizeResponse = await fetch("/api/upload/finalize", { method: "POST", headers: { "Content-Type": "application/json" }, From 4bc2672ce8fffa606fae53d1bd716a5fbb55346d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 11:52:40 +0200 Subject: [PATCH 04/17] Inline last chunk in finalize request The last chunk is sent inline with the finalize metadata (multipart) instead of going through the DB, saving one INSERT+SELECT per upload. For a 4 MB file (2 chunks) this halves DB writes; for larger files the ratio drops but every saved chunk still helps under Neon Free quotas. --- app/api/upload/finalize/route.ts | 66 +++++++++++++++++++------------ components/media/media-upload.tsx | 28 +++++++------ 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index a853fa1cd..bc7ba9449 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -14,6 +14,7 @@ import { updateFileCache } from "@/lib/github-cache-file"; const MAX_TOTAL_BYTES = 50 * 1024 * 1024; const MAX_CHUNKS = 50; +const MAX_INLINE_LAST_CHUNK_BYTES = 4 * 1024 * 1024; export async function POST(request: Request) { try { @@ -21,24 +22,33 @@ export async function POST(request: Request) { if ("response" in sessionResult) return sessionResult.response; const user = sessionResult.user; - const data: any = await request.json(); - const uploadId = typeof data.uploadId === "string" ? data.uploadId : ""; - const totalChunks = Number.isInteger(data.totalChunks) ? data.totalChunks : -1; - const owner = typeof data.owner === "string" ? data.owner : ""; - const repo = typeof data.repo === "string" ? data.repo : ""; - const branch = typeof data.branch === "string" ? data.branch : ""; - const path = typeof data.path === "string" ? data.path : ""; - const name = typeof data.name === "string" ? data.name : ""; - const sha = typeof data.sha === "string" ? data.sha : undefined; - const onConflict = data.onConflict === "error" ? "error" : "rename"; + const form = await request.formData(); + const uploadId = typeof form.get("uploadId") === "string" ? form.get("uploadId") as string : ""; + const totalChunksRaw = form.get("totalChunks"); + const totalChunks = typeof totalChunksRaw === "string" ? parseInt(totalChunksRaw, 10) : NaN; + const owner = typeof form.get("owner") === "string" ? form.get("owner") as string : ""; + const repo = typeof form.get("repo") === "string" ? form.get("repo") as string : ""; + const branch = typeof form.get("branch") === "string" ? form.get("branch") as string : ""; + const path = typeof form.get("path") === "string" ? form.get("path") as string : ""; + const name = typeof form.get("name") === "string" ? form.get("name") as string : ""; + const shaRaw = form.get("sha"); + const sha = typeof shaRaw === "string" && shaRaw.length > 0 ? shaRaw : undefined; + const onConflict = form.get("onConflict") === "error" ? "error" : "rename"; + const lastChunk = form.get("lastChunk"); if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); - if (totalChunks < 1 || totalChunks > MAX_CHUNKS) { + if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); } if (!owner || !repo || !branch || !path || !name) { throw createHttpError(`Missing required fields.`, 400); } + if (!(lastChunk instanceof Blob) || lastChunk.size === 0) { + throw createHttpError(`Missing "lastChunk".`, 400); + } + if (lastChunk.size > MAX_INLINE_LAST_CHUNK_BYTES) { + throw createHttpError(`"lastChunk" too large.`, 413); + } const { token } = await getToken(user, owner, repo, true); if (!token) throw new Error("Token not found"); @@ -63,28 +73,34 @@ export async function POST(request: Request) { throw createHttpError(`Use the files endpoint to create empty folders.`, 400); } - const chunks = await db - .select({ chunkIdx: uploadChunkTable.chunkIdx, data: uploadChunkTable.data }) - .from(uploadChunkTable) - .where(and( - eq(uploadChunkTable.uploadId, uploadId), - eq(uploadChunkTable.userId, user.id), - )) - .orderBy(asc(uploadChunkTable.chunkIdx)); - - if (chunks.length !== totalChunks) { + const expectedFromDb = totalChunks - 1; + const chunksFromDb = expectedFromDb > 0 + ? await db + .select({ chunkIdx: uploadChunkTable.chunkIdx, data: uploadChunkTable.data }) + .from(uploadChunkTable) + .where(and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + )) + .orderBy(asc(uploadChunkTable.chunkIdx)) + : []; + + if (chunksFromDb.length !== expectedFromDb) { throw createHttpError( - `Expected ${totalChunks} chunks but found ${chunks.length}.`, + `Expected ${expectedFromDb} staged chunks but found ${chunksFromDb.length}.`, 400, ); } - for (let i = 0; i < chunks.length; i++) { - if (chunks[i].chunkIdx !== i) { + for (let i = 0; i < chunksFromDb.length; i++) { + if (chunksFromDb[i].chunkIdx !== i) { throw createHttpError(`Missing chunk at index ${i}.`, 400); } } - const buffers = chunks.map(c => Buffer.from(c.data, "base64")); + const buffers = [ + ...chunksFromDb.map(c => Buffer.from(c.data, "base64")), + Buffer.from(await lastChunk.arrayBuffer()), + ]; const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); if (totalSize > MAX_TOTAL_BYTES) { throw createHttpError( diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 97edcb02f..29e720198 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -104,6 +104,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); + const stagedChunks = totalChunks - 1; // ponytail: last chunk rides inline in finalize const uploadChunk = async (idx: number) => { const start = idx * CHUNK_BYTES; @@ -118,26 +119,29 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }; // ponytail: batched parallelism (4); switch to rolling pool if uneven chunk times matter - for (let i = 0; i < totalChunks; i += CHUNK_CONCURRENCY) { + for (let i = 0; i < stagedChunks; i += CHUNK_CONCURRENCY) { const batch = []; - for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { + for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, stagedChunks); j++) { batch.push(uploadChunk(j)); } await Promise.all(batch); } + const lastStart = (totalChunks - 1) * CHUNK_BYTES; + const lastBlob = file.slice(lastStart, file.size); + const finalizeForm = new FormData(); + finalizeForm.set("uploadId", uploadId); + finalizeForm.set("totalChunks", String(totalChunks)); + finalizeForm.set("owner", config.owner); + finalizeForm.set("repo", config.repo); + finalizeForm.set("branch", config.branch); + finalizeForm.set("path", fullPath); + finalizeForm.set("name", configMedia.name); + finalizeForm.set("lastChunk", lastBlob); + const finalizeResponse = await fetch("/api/upload/finalize", { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - uploadId, - totalChunks, - owner: config.owner, - repo: config.repo, - branch: config.branch, - path: fullPath, - name: configMedia.name, - }), + body: finalizeForm, }); const data = await requireApiSuccess(finalizeResponse, "Failed to upload file"); From bcb36be00e0e53346e90818479f1af2c1099d1f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 11:58:14 +0200 Subject: [PATCH 05/17] Store chunks as bytea instead of base64 text Eliminates the +33% base64 overhead in the upload_chunk table. The INSERT/SELECT bandwidth per chunk drops by ~25% with no client change and no CPU cost. Migration uses decode('base64') in USING so any chunks in flight at upgrade time are converted instead of dropped. For a 20 MB upload, total DB bandwidth goes from ~48 MB to ~36 MB (2.4x -> 1.8x file size). --- app/api/upload/chunk/route.ts | 5 +- app/api/upload/finalize/route.ts | 2 +- db/migrations/0014_upload_chunk_bytea.sql | 1 + db/migrations/meta/0014_snapshot.json | 1611 +++++++++++++++++++++ db/migrations/meta/_journal.json | 7 + db/schema.ts | 9 +- 6 files changed, 1630 insertions(+), 5 deletions(-) create mode 100644 db/migrations/0014_upload_chunk_bytea.sql create mode 100644 db/migrations/meta/0014_snapshot.json diff --git a/app/api/upload/chunk/route.ts b/app/api/upload/chunk/route.ts index b7c090568..49f03439f 100644 --- a/app/api/upload/chunk/route.ts +++ b/app/api/upload/chunk/route.ts @@ -34,16 +34,15 @@ export async function POST(request: Request) { } const buffer = Buffer.from(await chunk.arrayBuffer()); - const base64 = buffer.toString("base64"); await db.insert(uploadChunkTable).values({ uploadId, userId: user.id, chunkIdx: idx, - data: base64, + data: buffer, }).onConflictDoUpdate({ target: [uploadChunkTable.uploadId, uploadChunkTable.chunkIdx], - set: { data: base64, createdAt: new Date() }, + set: { data: buffer, createdAt: new Date() }, setWhere: eq(uploadChunkTable.userId, user.id), }); diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index bc7ba9449..d798d38b0 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -98,7 +98,7 @@ export async function POST(request: Request) { } const buffers = [ - ...chunksFromDb.map(c => Buffer.from(c.data, "base64")), + ...chunksFromDb.map(c => c.data), Buffer.from(await lastChunk.arrayBuffer()), ]; const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); diff --git a/db/migrations/0014_upload_chunk_bytea.sql b/db/migrations/0014_upload_chunk_bytea.sql new file mode 100644 index 000000000..fc45a3853 --- /dev/null +++ b/db/migrations/0014_upload_chunk_bytea.sql @@ -0,0 +1 @@ +ALTER TABLE "upload_chunk" ALTER COLUMN "data" SET DATA TYPE bytea USING decode("data", 'base64'); diff --git a/db/migrations/meta/0014_snapshot.json b/db/migrations/meta/0014_snapshot.json new file mode 100644 index 000000000..59e6ba31d --- /dev/null +++ b/db/migrations/meta/0014_snapshot.json @@ -0,0 +1,1611 @@ +{ + "id": "4f4718b4-6983-41d5-9f07-6826b21beaf6", + "prevId": "5147365d-5a58-4806-8db4-ec7e4d10989b", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_account_userId": { + "name": "idx_account_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_providerId": { + "name": "idx_account_providerId", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_run": { + "name": "action_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_ref": { + "name": "workflow_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_type": { + "name": "context_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_name": { + "name": "context_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context_path": { + "name": "context_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow": { + "name": "workflow", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_run_id": { + "name": "workflow_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conclusion": { + "name": "conclusion", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triggered_by": { + "name": "triggered_by", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "failure": { + "name": "failure", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_action_run_owner_repo_createdAt": { + "name": "idx_action_run_owner_repo_createdAt", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_owner_repo_actionName": { + "name": "idx_action_run_owner_repo_actionName", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_owner_repo_status": { + "name": "idx_action_run_owner_repo_status", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_context": { + "name": "idx_action_run_context", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_action_run_workflowRunId": { + "name": "idx_action_run_workflowRunId", + "columns": [ + { + "expression": "workflow_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_file_meta": { + "name": "cache_file_meta", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'branch'" + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_timestamp": { + "name": "commit_timestamp", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ok'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_cache_file_meta_owner_repo_branch_path_context": { + "name": "idx_cache_file_meta_owner_repo_branch_path_context", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_file": { + "name": "cache_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'collection'" + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_path": { + "name": "parent_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "download_url": { + "name": "download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_timestamp": { + "name": "commit_timestamp", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_cache_file_owner_repo_branch_parentPath": { + "name": "idx_cache_file_owner_repo_branch_parentPath", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_cache_file_owner_repo_branch_path": { + "name": "idx_cache_file_owner_repo_branch_path", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache_permission": { + "name": "cache_permission", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "github_id": { + "name": "github_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_updated": { + "name": "last_updated", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_cache_permission_githubId_owner_repo": { + "name": "idx_cache_permission_githubId_owner_repo", + "columns": [ + { + "expression": "github_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collaborator_invite": { + "name": "collaborator_invite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uq_collaborator_invite_token": { + "name": "uq_collaborator_invite_token", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_collaborator_invite_owner_repo_email": { + "name": "idx_collaborator_invite_owner_repo_email", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_collaborator_invite_owner_repo_email_ci": { + "name": "uq_collaborator_invite_owner_repo_email_ci", + "columns": [ + { + "expression": "lower(\"owner\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"repo\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collaborator": { + "name": "collaborator", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "repo_id": { + "name": "repo_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_collaborator_owner_repo_email": { + "name": "idx_collaborator_owner_repo_email", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_collaborator_userId": { + "name": "idx_collaborator_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_collaborator_owner_repo_email_ci": { + "name": "uq_collaborator_owner_repo_email_ci", + "columns": [ + { + "expression": "lower(\"owner\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"repo\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collaborator_user_id_user_id_fk": { + "name": "collaborator_user_id_user_id_fk", + "tableFrom": "collaborator", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "collaborator_invited_by_user_id_fk": { + "name": "collaborator_invited_by_user_id_fk", + "tableFrom": "collaborator", + "tableTo": "user", + "columnsFrom": [ + "invited_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.config": { + "name": "config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_checked_at": { + "name": "last_checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_config_owner_repo_branch": { + "name": "idx_config_owner_repo_branch", + "columns": [ + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "branch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.github_installation_token": { + "name": "github_installation_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "uq_github_installation_token_installationId": { + "name": "uq_github_installation_token_installationId", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_session_userId": { + "name": "idx_session_userId", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.upload_chunk": { + "name": "upload_chunk", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "upload_id": { + "name": "upload_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_idx": { + "name": "chunk_idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uq_upload_chunk_uploadId_chunkIdx": { + "name": "uq_upload_chunk_uploadId_chunkIdx", + "columns": [ + { + "expression": "upload_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_idx", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_upload_chunk_createdAt": { + "name": "idx_upload_chunk_createdAt", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "upload_chunk_user_id_user_id_fk": { + "name": "upload_chunk_user_id_user_id_fk", + "tableFrom": "upload_chunk", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_username": { + "name": "github_username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_verification_identifier": { + "name": "idx_verification_identifier", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/db/migrations/meta/_journal.json b/db/migrations/meta/_journal.json index e17433125..d897c366a 100644 --- a/db/migrations/meta/_journal.json +++ b/db/migrations/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1782289030662, "tag": "0013_upload_chunks", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1782295021459, + "tag": "0014_upload_chunk_bytea", + "breakpoints": true } ] } \ No newline at end of file diff --git a/db/schema.ts b/db/schema.ts index 2c80ff8f0..fadd0c4bf 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -9,9 +9,16 @@ import { index, uniqueIndex, jsonb, + customType, } from "drizzle-orm/pg-core"; import { sql } from "drizzle-orm"; +const bytea = customType<{ data: Buffer; driverData: Buffer; notNull: true; default: false }>({ + dataType() { + return "bytea"; + }, +}); + const userTable = pgTable("user", { id: text("id").notNull().primaryKey(), name: text("name").notNull(), @@ -214,7 +221,7 @@ const uploadChunkTable = pgTable("upload_chunk", { uploadId: text("upload_id").notNull(), userId: text("user_id").notNull().references(() => userTable.id, { onDelete: "cascade" }), chunkIdx: integer("chunk_idx").notNull(), - data: text("data").notNull(), + data: bytea("data").notNull(), createdAt: timestamp("created_at").notNull().defaultNow(), }, (table) => [ uniqueIndex("uq_upload_chunk_uploadId_chunkIdx").on(table.uploadId, table.chunkIdx), From 63359c7391038caf5e5b8a4fb3c899b88e49f4d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 12:01:21 +0200 Subject: [PATCH 06/17] Bump chunk size to 4 MB Multipart carries the chunk as raw binary, not base64, so 4 MB fits in Vercel's 4.5 MB body with about 500 KB of headroom. For a 20 MB upload this drops the chunk count from 7 to 5 (4 staged in DB, 1 inline) and total DB bandwidth from ~36 MB to ~32 MB. --- components/media/media-upload.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 29e720198..d92cbb253 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -66,7 +66,8 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const handleFiles = useCallback(async (files: File[]) => { // ponytail: 3 MB keeps base64 + JSON envelope under Vercel's 4.5 MB body limit const DIRECT_UPLOAD_BYTES = 3 * 1024 * 1024; - const CHUNK_BYTES = 3 * 1024 * 1024; + // ponytail: 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk + const CHUNK_BYTES = 4 * 1024 * 1024; const MAX_TOTAL_BYTES = 50 * 1024 * 1024; const CHUNK_CONCURRENCY = 4; From 363d13405cfb653f6e4501aa24115a9c241b3947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 12:05:13 +0200 Subject: [PATCH 07/17] Inline first chunk instead of last The first chunk is always CHUNK_BYTES (4 MB) for any multi-chunk upload; the last one can be smaller. Sending the first inline keeps the largest chunk out of the DB. For files whose size is a multiple of CHUNK_BYTES there is no change; for the rest, DB bandwidth drops by up to (CHUNK_BYTES - lastChunkSize) * 2. --- app/api/upload/finalize/route.ts | 18 +++++++++--------- components/media/media-upload.tsx | 11 +++++------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index d798d38b0..d9fa42259 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -14,7 +14,7 @@ import { updateFileCache } from "@/lib/github-cache-file"; const MAX_TOTAL_BYTES = 50 * 1024 * 1024; const MAX_CHUNKS = 50; -const MAX_INLINE_LAST_CHUNK_BYTES = 4 * 1024 * 1024; +const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; export async function POST(request: Request) { try { @@ -34,7 +34,7 @@ export async function POST(request: Request) { const shaRaw = form.get("sha"); const sha = typeof shaRaw === "string" && shaRaw.length > 0 ? shaRaw : undefined; const onConflict = form.get("onConflict") === "error" ? "error" : "rename"; - const lastChunk = form.get("lastChunk"); + const firstChunk = form.get("firstChunk"); if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { @@ -43,11 +43,11 @@ export async function POST(request: Request) { if (!owner || !repo || !branch || !path || !name) { throw createHttpError(`Missing required fields.`, 400); } - if (!(lastChunk instanceof Blob) || lastChunk.size === 0) { - throw createHttpError(`Missing "lastChunk".`, 400); + if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { + throw createHttpError(`Missing "firstChunk".`, 400); } - if (lastChunk.size > MAX_INLINE_LAST_CHUNK_BYTES) { - throw createHttpError(`"lastChunk" too large.`, 413); + if (firstChunk.size > MAX_INLINE_CHUNK_BYTES) { + throw createHttpError(`"firstChunk" too large.`, 413); } const { token } = await getToken(user, owner, repo, true); @@ -92,14 +92,14 @@ export async function POST(request: Request) { ); } for (let i = 0; i < chunksFromDb.length; i++) { - if (chunksFromDb[i].chunkIdx !== i) { - throw createHttpError(`Missing chunk at index ${i}.`, 400); + if (chunksFromDb[i].chunkIdx !== i + 1) { + throw createHttpError(`Missing chunk at index ${i + 1}.`, 400); } } const buffers = [ + Buffer.from(await firstChunk.arrayBuffer()), ...chunksFromDb.map(c => c.data), - Buffer.from(await lastChunk.arrayBuffer()), ]; const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); if (totalSize > MAX_TOTAL_BYTES) { diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index d92cbb253..4817becb6 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -105,7 +105,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); - const stagedChunks = totalChunks - 1; // ponytail: last chunk rides inline in finalize + // ponytail: chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes const uploadChunk = async (idx: number) => { const start = idx * CHUNK_BYTES; @@ -120,16 +120,15 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }; // ponytail: batched parallelism (4); switch to rolling pool if uneven chunk times matter - for (let i = 0; i < stagedChunks; i += CHUNK_CONCURRENCY) { + for (let i = 1; i < totalChunks; i += CHUNK_CONCURRENCY) { const batch = []; - for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, stagedChunks); j++) { + for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { batch.push(uploadChunk(j)); } await Promise.all(batch); } - const lastStart = (totalChunks - 1) * CHUNK_BYTES; - const lastBlob = file.slice(lastStart, file.size); + const firstBlob = file.slice(0, Math.min(CHUNK_BYTES, file.size)); const finalizeForm = new FormData(); finalizeForm.set("uploadId", uploadId); finalizeForm.set("totalChunks", String(totalChunks)); @@ -138,7 +137,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple finalizeForm.set("branch", config.branch); finalizeForm.set("path", fullPath); finalizeForm.set("name", configMedia.name); - finalizeForm.set("lastChunk", lastBlob); + finalizeForm.set("firstChunk", firstBlob); const finalizeResponse = await fetch("/api/upload/finalize", { method: "POST", From 9e644e4ab88026447ca0e4646bdc23721f419d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 12:18:09 +0200 Subject: [PATCH 08/17] Cap media uploads at 15 MB Limits per upload to 22 MB of DB bandwidth in the worst case, sized to fit Neon Free quotas with margin for other traffic. --- app/api/upload/finalize/route.ts | 4 ++-- components/media/media-upload.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index d9fa42259..208c78367 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -12,8 +12,8 @@ import { resolveCommitIdentity } from "@/lib/commit-message"; import { githubSaveFile } from "@/lib/utils/github-save-file"; import { updateFileCache } from "@/lib/github-cache-file"; -const MAX_TOTAL_BYTES = 50 * 1024 * 1024; -const MAX_CHUNKS = 50; +const MAX_TOTAL_BYTES = 15 * 1024 * 1024; +const MAX_CHUNKS = 4; const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; export async function POST(request: Request) { diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 4817becb6..33edd9b44 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -68,7 +68,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const DIRECT_UPLOAD_BYTES = 3 * 1024 * 1024; // ponytail: 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk const CHUNK_BYTES = 4 * 1024 * 1024; - const MAX_TOTAL_BYTES = 50 * 1024 * 1024; + const MAX_TOTAL_BYTES = 15 * 1024 * 1024; const CHUNK_CONCURRENCY = 4; const readAsBase64 = (file: File) => new Promise((resolve, reject) => { From 43d543a428d6f39b8327bde4cdddc3c7b11f3c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 12:48:12 +0200 Subject: [PATCH 09/17] Consolidate upload_chunk into a single migration Merges the old 0013 (CREATE TABLE with text data) and 0014 (ALTER to bytea) into a single 0013 that creates the table with bytea directly. Also drops the unused chunk-assembly self-check script. --- db/migrations/0013_upload_chunks.sql | 2 +- db/migrations/0014_upload_chunk_bytea.sql | 1 - db/migrations/meta/0013_snapshot.json | 4 +- db/migrations/meta/0014_snapshot.json | 1611 --------------------- db/migrations/meta/_journal.json | 9 +- scripts/check-chunk-assembly.mjs | 40 - 6 files changed, 4 insertions(+), 1663 deletions(-) delete mode 100644 db/migrations/0014_upload_chunk_bytea.sql delete mode 100644 db/migrations/meta/0014_snapshot.json delete mode 100644 scripts/check-chunk-assembly.mjs diff --git a/db/migrations/0013_upload_chunks.sql b/db/migrations/0013_upload_chunks.sql index 682a5ff42..5c5c75c29 100644 --- a/db/migrations/0013_upload_chunks.sql +++ b/db/migrations/0013_upload_chunks.sql @@ -3,7 +3,7 @@ CREATE TABLE "upload_chunk" ( "upload_id" text NOT NULL, "user_id" text NOT NULL, "chunk_idx" integer NOT NULL, - "data" text NOT NULL, + "data" bytea NOT NULL, "created_at" timestamp DEFAULT now() NOT NULL ); --> statement-breakpoint diff --git a/db/migrations/0014_upload_chunk_bytea.sql b/db/migrations/0014_upload_chunk_bytea.sql deleted file mode 100644 index fc45a3853..000000000 --- a/db/migrations/0014_upload_chunk_bytea.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "upload_chunk" ALTER COLUMN "data" SET DATA TYPE bytea USING decode("data", 'base64'); diff --git a/db/migrations/meta/0013_snapshot.json b/db/migrations/meta/0013_snapshot.json index 72da85a09..ad3148d18 100644 --- a/db/migrations/meta/0013_snapshot.json +++ b/db/migrations/meta/0013_snapshot.json @@ -1,5 +1,5 @@ { - "id": "5147365d-5a58-4806-8db4-ec7e4d10989b", + "id": "cd186a8b-62c3-4425-9190-ae26f6e46d0b", "prevId": "4655a2e1-f322-41b1-b587-9fa5d038d98a", "version": "7", "dialect": "postgresql", @@ -1386,7 +1386,7 @@ }, "data": { "name": "data", - "type": "text", + "type": "bytea", "primaryKey": false, "notNull": true }, diff --git a/db/migrations/meta/0014_snapshot.json b/db/migrations/meta/0014_snapshot.json deleted file mode 100644 index 59e6ba31d..000000000 --- a/db/migrations/meta/0014_snapshot.json +++ /dev/null @@ -1,1611 +0,0 @@ -{ - "id": "4f4718b4-6983-41d5-9f07-6826b21beaf6", - "prevId": "5147365d-5a58-4806-8db4-ec7e4d10989b", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.account": { - "name": "account", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "account_id": { - "name": "account_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "provider_id": { - "name": "provider_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "access_token": { - "name": "access_token", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "refresh_token": { - "name": "refresh_token", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "id_token": { - "name": "id_token", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "access_token_expires_at": { - "name": "access_token_expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "refresh_token_expires_at": { - "name": "refresh_token_expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "scope": { - "name": "scope", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "idx_account_userId": { - "name": "idx_account_userId", - "columns": [ - { - "expression": "user_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_account_providerId": { - "name": "idx_account_providerId", - "columns": [ - { - "expression": "provider_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "account_user_id_user_id_fk": { - "name": "account_user_id_user_id_fk", - "tableFrom": "account", - "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.action_run": { - "name": "action_run", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "ref": { - "name": "ref", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "workflow_ref": { - "name": "workflow_ref", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "sha": { - "name": "sha", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "action_name": { - "name": "action_name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "context_type": { - "name": "context_type", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "context_name": { - "name": "context_name", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "context_path": { - "name": "context_path", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "workflow": { - "name": "workflow", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "workflow_run_id": { - "name": "workflow_run_id", - "type": "bigint", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "conclusion": { - "name": "conclusion", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "html_url": { - "name": "html_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "triggered_by": { - "name": "triggered_by", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "failure": { - "name": "failure", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "payload": { - "name": "payload", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "completed_at": { - "name": "completed_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_action_run_owner_repo_createdAt": { - "name": "idx_action_run_owner_repo_createdAt", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "created_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_action_run_owner_repo_actionName": { - "name": "idx_action_run_owner_repo_actionName", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "action_name", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_action_run_owner_repo_status": { - "name": "idx_action_run_owner_repo_status", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_action_run_context": { - "name": "idx_action_run_context", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "context_type", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "context_name", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "context_path", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_action_run_workflowRunId": { - "name": "idx_action_run_workflowRunId", - "columns": [ - { - "expression": "workflow_run_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.cache_file_meta": { - "name": "cache_file_meta", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "branch": { - "name": "branch", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "path": { - "name": "path", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "''" - }, - "context": { - "name": "context", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "'branch'" - }, - "commit_sha": { - "name": "commit_sha", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "commit_timestamp": { - "name": "commit_timestamp", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "'ok'" - }, - "error": { - "name": "error", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "last_checked_at": { - "name": "last_checked_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "idx_cache_file_meta_owner_repo_branch_path_context": { - "name": "idx_cache_file_meta_owner_repo_branch_path_context", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "branch", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "path", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "context", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.cache_file": { - "name": "cache_file", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "context": { - "name": "context", - "type": "text", - "primaryKey": false, - "notNull": true, - "default": "'collection'" - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "branch": { - "name": "branch", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "parent_path": { - "name": "parent_path", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "path": { - "name": "path", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "sha": { - "name": "sha", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "download_url": { - "name": "download_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "commit_sha": { - "name": "commit_sha", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "commit_timestamp": { - "name": "commit_timestamp", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "idx_cache_file_owner_repo_branch_parentPath": { - "name": "idx_cache_file_owner_repo_branch_parentPath", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "branch", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "parent_path", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_cache_file_owner_repo_branch_path": { - "name": "idx_cache_file_owner_repo_branch_path", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "branch", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "path", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.cache_permission": { - "name": "cache_permission", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "github_id": { - "name": "github_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "last_updated": { - "name": "last_updated", - "type": "timestamp", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "idx_cache_permission_githubId_owner_repo": { - "name": "idx_cache_permission_githubId_owner_repo", - "columns": [ - { - "expression": "github_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.collaborator_invite": { - "name": "collaborator_invite", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "token": { - "name": "token", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "expires_at": { - "name": "expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "uq_collaborator_invite_token": { - "name": "uq_collaborator_invite_token", - "columns": [ - { - "expression": "token", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_collaborator_invite_owner_repo_email": { - "name": "idx_collaborator_invite_owner_repo_email", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "email", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "uq_collaborator_invite_owner_repo_email_ci": { - "name": "uq_collaborator_invite_owner_repo_email_ci", - "columns": [ - { - "expression": "lower(\"owner\")", - "asc": true, - "isExpression": true, - "nulls": "last" - }, - { - "expression": "lower(\"repo\")", - "asc": true, - "isExpression": true, - "nulls": "last" - }, - { - "expression": "lower(\"email\")", - "asc": true, - "isExpression": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.collaborator": { - "name": "collaborator", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "installation_id": { - "name": "installation_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "owner_id": { - "name": "owner_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "repo_id": { - "name": "repo_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "branch": { - "name": "branch", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "invited_by": { - "name": "invited_by", - "type": "text", - "primaryKey": false, - "notNull": false - } - }, - "indexes": { - "idx_collaborator_owner_repo_email": { - "name": "idx_collaborator_owner_repo_email", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "email", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_collaborator_userId": { - "name": "idx_collaborator_userId", - "columns": [ - { - "expression": "user_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "uq_collaborator_owner_repo_email_ci": { - "name": "uq_collaborator_owner_repo_email_ci", - "columns": [ - { - "expression": "lower(\"owner\")", - "asc": true, - "isExpression": true, - "nulls": "last" - }, - { - "expression": "lower(\"repo\")", - "asc": true, - "isExpression": true, - "nulls": "last" - }, - { - "expression": "lower(\"email\")", - "asc": true, - "isExpression": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "collaborator_user_id_user_id_fk": { - "name": "collaborator_user_id_user_id_fk", - "tableFrom": "collaborator", - "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "collaborator_invited_by_user_id_fk": { - "name": "collaborator_invited_by_user_id_fk", - "tableFrom": "collaborator", - "tableTo": "user", - "columnsFrom": [ - "invited_by" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.config": { - "name": "config", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "repo": { - "name": "repo", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "branch": { - "name": "branch", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "sha": { - "name": "sha", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "version": { - "name": "version", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "object": { - "name": "object", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "last_checked_at": { - "name": "last_checked_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "idx_config_owner_repo_branch": { - "name": "idx_config_owner_repo_branch", - "columns": [ - { - "expression": "owner", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "repo", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "branch", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.github_installation_token": { - "name": "github_installation_token", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "ciphertext": { - "name": "ciphertext", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "iv": { - "name": "iv", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "installation_id": { - "name": "installation_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "expires_at": { - "name": "expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "uq_github_installation_token_installationId": { - "name": "uq_github_installation_token_installationId", - "columns": [ - { - "expression": "installation_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.session": { - "name": "session", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "expires_at": { - "name": "expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true - }, - "token": { - "name": "token", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "ip_address": { - "name": "ip_address", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "user_agent": { - "name": "user_agent", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": true - } - }, - "indexes": { - "idx_session_userId": { - "name": "idx_session_userId", - "columns": [ - { - "expression": "user_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "session_user_id_user_id_fk": { - "name": "session_user_id_user_id_fk", - "tableFrom": "session", - "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "session_token_unique": { - "name": "session_token_unique", - "nullsNotDistinct": false, - "columns": [ - "token" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.upload_chunk": { - "name": "upload_chunk", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "upload_id": { - "name": "upload_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "chunk_idx": { - "name": "chunk_idx", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "data": { - "name": "data", - "type": "bytea", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "uq_upload_chunk_uploadId_chunkIdx": { - "name": "uq_upload_chunk_uploadId_chunkIdx", - "columns": [ - { - "expression": "upload_id", - "isExpression": false, - "asc": true, - "nulls": "last" - }, - { - "expression": "chunk_idx", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": true, - "concurrently": false, - "method": "btree", - "with": {} - }, - "idx_upload_chunk_createdAt": { - "name": "idx_upload_chunk_createdAt", - "columns": [ - { - "expression": "created_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "upload_chunk_user_id_user_id_fk": { - "name": "upload_chunk_user_id_user_id_fk", - "tableFrom": "upload_chunk", - "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.user": { - "name": "user", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "image": { - "name": "image", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "github_username": { - "name": "github_username", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "email_verified": { - "name": "email_verified", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "user_email_unique": { - "name": "user_email_unique", - "nullsNotDistinct": false, - "columns": [ - "email" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.verification": { - "name": "verification", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "identifier": { - "name": "identifier", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "value": { - "name": "value", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "expires_at": { - "name": "expires_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "idx_verification_identifier": { - "name": "idx_verification_identifier", - "columns": [ - { - "expression": "identifier", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": {}, - "schemas": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "views": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} \ No newline at end of file diff --git a/db/migrations/meta/_journal.json b/db/migrations/meta/_journal.json index d897c366a..5dd089df7 100644 --- a/db/migrations/meta/_journal.json +++ b/db/migrations/meta/_journal.json @@ -96,16 +96,9 @@ { "idx": 13, "version": "7", - "when": 1782289030662, + "when": 1782298049440, "tag": "0013_upload_chunks", "breakpoints": true - }, - { - "idx": 14, - "version": "7", - "when": 1782295021459, - "tag": "0014_upload_chunk_bytea", - "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/check-chunk-assembly.mjs b/scripts/check-chunk-assembly.mjs deleted file mode 100644 index 87022e35b..000000000 --- a/scripts/check-chunk-assembly.mjs +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env node -// Self-check for chunk split + base64 round-trip + concat. -// Run: node scripts/check-chunk-assembly.mjs - -import { randomBytes } from "node:crypto"; -import { strict as assert } from "node:assert"; - -const CHUNK_BYTES = 3 * 1024 * 1024; - -function splitToBase64Chunks(buffer, chunkBytes) { - const chunks = []; - for (let start = 0; start < buffer.length; start += chunkBytes) { - const slice = buffer.subarray(start, Math.min(start + chunkBytes, buffer.length)); - chunks.push(slice.toString("base64")); - } - return chunks; -} - -function assembleFromBase64Chunks(chunks) { - return Buffer.concat(chunks.map((c) => Buffer.from(c, "base64"))); -} - -function check(label, sizeBytes) { - const original = randomBytes(sizeBytes); - const chunks = splitToBase64Chunks(original, CHUNK_BYTES); - const expectedChunks = Math.max(1, Math.ceil(sizeBytes / CHUNK_BYTES)); - assert.equal(chunks.length, expectedChunks, `${label}: chunk count`); - const reassembled = assembleFromBase64Chunks(chunks); - assert.equal(reassembled.length, original.length, `${label}: length`); - assert.ok(reassembled.equals(original), `${label}: bytes match`); - console.log(`ok ${label} (${sizeBytes} bytes, ${chunks.length} chunks)`); -} - -check("single chunk", 1024); -check("exactly one chunk", CHUNK_BYTES); -check("two chunks, second partial", CHUNK_BYTES + 1); -check("many chunks", CHUNK_BYTES * 7 + 123); -check("size 1", 1); - -console.log("\nAll chunk-assembly checks passed."); From a98b3aac85794b701671b9f044c2ff6b190d5590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 12:50:22 +0200 Subject: [PATCH 10/17] Drop ponytail tag from comments --- app/api/upload/chunk/route.ts | 1 - app/api/upload/finalize/route.ts | 1 - components/media/media-upload.tsx | 8 ++++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/api/upload/chunk/route.ts b/app/api/upload/chunk/route.ts index 49f03439f..5d3c56591 100644 --- a/app/api/upload/chunk/route.ts +++ b/app/api/upload/chunk/route.ts @@ -46,7 +46,6 @@ export async function POST(request: Request) { setWhere: eq(uploadChunkTable.userId, user.id), }); - // ponytail: oportunistic stale-chunk cleanup runs after the response; cron-free housekeeping for Hobby after(async () => { try { await db.delete(uploadChunkTable).where( diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index 208c78367..e23119160 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -161,7 +161,6 @@ export async function POST(request: Request) { ); } - // ponytail: chunks already pushed to GitHub; delete in background, TTL sweep catches any miss after(async () => { try { await db.delete(uploadChunkTable).where(and( diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 33edd9b44..42c77a306 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -64,9 +64,9 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }, [extensions, configMedia?.extensions]); const handleFiles = useCallback(async (files: File[]) => { - // ponytail: 3 MB keeps base64 + JSON envelope under Vercel's 4.5 MB body limit + // 3 MB keeps base64 + JSON envelope under Vercel's 4.5 MB body limit const DIRECT_UPLOAD_BYTES = 3 * 1024 * 1024; - // ponytail: 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk + // 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk const CHUNK_BYTES = 4 * 1024 * 1024; const MAX_TOTAL_BYTES = 15 * 1024 * 1024; const CHUNK_CONCURRENCY = 4; @@ -105,7 +105,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); - // ponytail: chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes + // chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes const uploadChunk = async (idx: number) => { const start = idx * CHUNK_BYTES; @@ -119,7 +119,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); }; - // ponytail: batched parallelism (4); switch to rolling pool if uneven chunk times matter + // batched parallelism (4); switch to rolling pool if uneven chunk times matter for (let i = 1; i < totalChunks; i += CHUNK_CONCURRENCY) { const batch = []; for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { From e85ab114803dd613f98f32ce512eaf058d1280ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 13:07:10 +0200 Subject: [PATCH 11/17] Drop direct upload branch for small media files Files <=3 MB previously skipped the chunked path and posted base64+JSON to the files endpoint. Removed because the chunked path already short-circuits to inline-only when totalChunks=1, no DB rows are created, and the request body is smaller (binary multipart vs base64 JSON). One code path now handles every size up to 15 MB. --- components/media/media-upload.tsx | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 42c77a306..377ec71d6 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -64,20 +64,11 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }, [extensions, configMedia?.extensions]); const handleFiles = useCallback(async (files: File[]) => { - // 3 MB keeps base64 + JSON envelope under Vercel's 4.5 MB body limit - const DIRECT_UPLOAD_BYTES = 3 * 1024 * 1024; // 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk const CHUNK_BYTES = 4 * 1024 * 1024; const MAX_TOTAL_BYTES = 15 * 1024 * 1024; const CHUNK_CONCURRENCY = 4; - const readAsBase64 = (file: File) => new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve((reader.result as string).replace(/^(.+,)/, "")); - reader.onerror = () => reject(new Error("Failed to read file")); - reader.readAsDataURL(file); - }); - try { for (const file of files) { const uploadFilename = getUploadFileName( @@ -92,17 +83,6 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); } - if (file.size <= DIRECT_UPLOAD_BYTES) { - const content = await readAsBase64(file); - const response = await fetch(`/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullPath)}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ type: "media", name: configMedia.name, content }), - }); - const data = await requireApiSuccess(response, "Failed to upload file"); - return data.data as FileSaveData; - } - const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); // chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes From 543c2d1dbb7dcb511ac2fca61a2c63fd08448898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Wed, 24 Jun 2026 16:09:54 +0200 Subject: [PATCH 12/17] Move stale chunk cleanup into finalize and shorten TTL to 10m --- app/api/upload/chunk/route.ts | 14 +------------- app/api/upload/finalize/route.ts | 12 ++++++++---- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/app/api/upload/chunk/route.ts b/app/api/upload/chunk/route.ts index 5d3c56591..d42ceae66 100644 --- a/app/api/upload/chunk/route.ts +++ b/app/api/upload/chunk/route.ts @@ -1,12 +1,10 @@ -import { after } from "next/server"; import { db } from "@/db"; import { uploadChunkTable } from "@/db/schema"; -import { eq, lt } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { requireApiUserSession } from "@/lib/session-server"; import { createHttpError, toErrorResponse } from "@/lib/api-error"; const MAX_CHUNK_BYTES = 4 * 1024 * 1024; -const STALE_CHUNK_AGE_MS = 60 * 60 * 1000; export async function POST(request: Request) { try { @@ -46,16 +44,6 @@ export async function POST(request: Request) { setWhere: eq(uploadChunkTable.userId, user.id), }); - after(async () => { - try { - await db.delete(uploadChunkTable).where( - lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), - ); - } catch (error) { - console.error("Stale chunk cleanup failed", error); - } - }); - return Response.json({ status: "success" }); } catch (error: any) { if (!error?.status || error.status >= 500) console.error(error); diff --git a/app/api/upload/finalize/route.ts b/app/api/upload/finalize/route.ts index e23119160..6d3837c78 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/upload/finalize/route.ts @@ -1,7 +1,7 @@ import { after } from "next/server"; import { db } from "@/db"; import { uploadChunkTable } from "@/db/schema"; -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, lt, or } from "drizzle-orm"; import { requireApiUserSession } from "@/lib/session-server"; import { createHttpError, toErrorResponse } from "@/lib/api-error"; import { getToken } from "@/lib/token"; @@ -15,6 +15,7 @@ import { updateFileCache } from "@/lib/github-cache-file"; const MAX_TOTAL_BYTES = 15 * 1024 * 1024; const MAX_CHUNKS = 4; const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; +const STALE_CHUNK_AGE_MS = 10 * 60 * 1000; export async function POST(request: Request) { try { @@ -163,9 +164,12 @@ export async function POST(request: Request) { after(async () => { try { - await db.delete(uploadChunkTable).where(and( - eq(uploadChunkTable.uploadId, uploadId), - eq(uploadChunkTable.userId, user.id), + await db.delete(uploadChunkTable).where(or( + and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + ), + lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), )); } catch (error) { console.error("Chunk cleanup after finalize failed", error); From 73dcda9973bab735b4ca9d62a6cb4e1ce6d27831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Fri, 26 Jun 2026 07:49:15 +0200 Subject: [PATCH 13/17] Move chunk and finalize routes under files path prefix --- .../[repo]/[branch]/files/[path]}/chunk/route.ts | 0 .../[branch]/files/[path]}/finalize/route.ts | 15 +++++++-------- components/media/media-upload.tsx | 9 +++------ 3 files changed, 10 insertions(+), 14 deletions(-) rename app/api/{upload => [owner]/[repo]/[branch]/files/[path]}/chunk/route.ts (100%) rename app/api/{upload => [owner]/[repo]/[branch]/files/[path]}/finalize/route.ts (93%) diff --git a/app/api/upload/chunk/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/chunk/route.ts similarity index 100% rename from app/api/upload/chunk/route.ts rename to app/api/[owner]/[repo]/[branch]/files/[path]/chunk/route.ts diff --git a/app/api/upload/finalize/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts similarity index 93% rename from app/api/upload/finalize/route.ts rename to app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts index 6d3837c78..86f07c994 100644 --- a/app/api/upload/finalize/route.ts +++ b/app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts @@ -17,20 +17,21 @@ const MAX_CHUNKS = 4; const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; const STALE_CHUNK_AGE_MS = 10 * 60 * 1000; -export async function POST(request: Request) { +export async function POST( + request: Request, + context: { params: Promise<{ owner: string, repo: string, branch: string, path: string }> } +) { try { const sessionResult = await requireApiUserSession(); if ("response" in sessionResult) return sessionResult.response; const user = sessionResult.user; + const { owner, repo, branch, path } = await context.params; + const form = await request.formData(); const uploadId = typeof form.get("uploadId") === "string" ? form.get("uploadId") as string : ""; const totalChunksRaw = form.get("totalChunks"); const totalChunks = typeof totalChunksRaw === "string" ? parseInt(totalChunksRaw, 10) : NaN; - const owner = typeof form.get("owner") === "string" ? form.get("owner") as string : ""; - const repo = typeof form.get("repo") === "string" ? form.get("repo") as string : ""; - const branch = typeof form.get("branch") === "string" ? form.get("branch") as string : ""; - const path = typeof form.get("path") === "string" ? form.get("path") as string : ""; const name = typeof form.get("name") === "string" ? form.get("name") as string : ""; const shaRaw = form.get("sha"); const sha = typeof shaRaw === "string" && shaRaw.length > 0 ? shaRaw : undefined; @@ -41,9 +42,7 @@ export async function POST(request: Request) { if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); } - if (!owner || !repo || !branch || !path || !name) { - throw createHttpError(`Missing required fields.`, 400); - } + if (!name) throw createHttpError(`Missing "name".`, 400); if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { throw createHttpError(`Missing "firstChunk".`, 400); } diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 377ec71d6..f318dd7cf 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -85,6 +85,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); + const baseUrl = `/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullPath)}`; // chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes const uploadChunk = async (idx: number) => { @@ -95,7 +96,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple form.set("uploadId", uploadId); form.set("idx", String(idx)); form.set("chunk", blob); - const chunkResponse = await fetch("/api/upload/chunk", { method: "POST", body: form }); + const chunkResponse = await fetch(`${baseUrl}/chunk`, { method: "POST", body: form }); await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); }; @@ -112,14 +113,10 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const finalizeForm = new FormData(); finalizeForm.set("uploadId", uploadId); finalizeForm.set("totalChunks", String(totalChunks)); - finalizeForm.set("owner", config.owner); - finalizeForm.set("repo", config.repo); - finalizeForm.set("branch", config.branch); - finalizeForm.set("path", fullPath); finalizeForm.set("name", configMedia.name); finalizeForm.set("firstChunk", firstBlob); - const finalizeResponse = await fetch("/api/upload/finalize", { + const finalizeResponse = await fetch(`${baseUrl}/finalize`, { method: "POST", body: finalizeForm, }); From e592289a22c8fea2c3f7e9d8489574ca488107af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Fri, 26 Jun 2026 08:04:12 +0200 Subject: [PATCH 14/17] Move chunk and finalize routes under media path prefix --- .../[branch]/files/[path]/finalize/route.ts | 197 ----------------- .../[name]}/[path]/chunk/route.ts | 0 .../[branch]/media/[name]/[path]/route.ts | 205 +++++++++++++++++- components/media/media-upload.tsx | 5 +- 4 files changed, 202 insertions(+), 205 deletions(-) delete mode 100644 app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts rename app/api/[owner]/[repo]/[branch]/{files => media/[name]}/[path]/chunk/route.ts (100%) diff --git a/app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts deleted file mode 100644 index 86f07c994..000000000 --- a/app/api/[owner]/[repo]/[branch]/files/[path]/finalize/route.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { after } from "next/server"; -import { db } from "@/db"; -import { uploadChunkTable } from "@/db/schema"; -import { and, asc, eq, lt, or } from "drizzle-orm"; -import { requireApiUserSession } from "@/lib/session-server"; -import { createHttpError, toErrorResponse } from "@/lib/api-error"; -import { getToken } from "@/lib/token"; -import { getConfig } from "@/lib/config-store"; -import { getSchemaByName } from "@/lib/schema"; -import { getFileExtension, getFileName, normalizePath } from "@/lib/utils/file"; -import { resolveCommitIdentity } from "@/lib/commit-message"; -import { githubSaveFile } from "@/lib/utils/github-save-file"; -import { updateFileCache } from "@/lib/github-cache-file"; - -const MAX_TOTAL_BYTES = 15 * 1024 * 1024; -const MAX_CHUNKS = 4; -const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; -const STALE_CHUNK_AGE_MS = 10 * 60 * 1000; - -export async function POST( - request: Request, - context: { params: Promise<{ owner: string, repo: string, branch: string, path: string }> } -) { - try { - const sessionResult = await requireApiUserSession(); - if ("response" in sessionResult) return sessionResult.response; - const user = sessionResult.user; - - const { owner, repo, branch, path } = await context.params; - - const form = await request.formData(); - const uploadId = typeof form.get("uploadId") === "string" ? form.get("uploadId") as string : ""; - const totalChunksRaw = form.get("totalChunks"); - const totalChunks = typeof totalChunksRaw === "string" ? parseInt(totalChunksRaw, 10) : NaN; - const name = typeof form.get("name") === "string" ? form.get("name") as string : ""; - const shaRaw = form.get("sha"); - const sha = typeof shaRaw === "string" && shaRaw.length > 0 ? shaRaw : undefined; - const onConflict = form.get("onConflict") === "error" ? "error" : "rename"; - const firstChunk = form.get("firstChunk"); - - if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); - if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { - throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); - } - if (!name) throw createHttpError(`Missing "name".`, 400); - if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { - throw createHttpError(`Missing "firstChunk".`, 400); - } - if (firstChunk.size > MAX_INLINE_CHUNK_BYTES) { - throw createHttpError(`"firstChunk" too large.`, 413); - } - - const { token } = await getToken(user, owner, repo, true); - if (!token) throw new Error("Token not found"); - - const normalizedPath = normalizePath(path); - - const config = await getConfig(owner, repo, branch, { getToken: async () => token }); - if (!config) throw new Error(`Configuration not found for ${owner}/${repo}/${branch}.`); - - const schema = getSchemaByName(config.object, name, "media"); - if (!schema) throw new Error(`Media schema not found for ${name}.`); - if (!normalizedPath.startsWith(schema.input)) { - throw new Error(`Invalid path "${path}" for media "${name}".`); - } - if ( - schema.extensions?.length > 0 && - !schema.extensions.includes(getFileExtension(normalizedPath)) - ) { - throw new Error(`Invalid extension "${getFileExtension(normalizedPath)}" for media.`); - } - if (getFileName(normalizedPath) === ".gitkeep") { - throw createHttpError(`Use the files endpoint to create empty folders.`, 400); - } - - const expectedFromDb = totalChunks - 1; - const chunksFromDb = expectedFromDb > 0 - ? await db - .select({ chunkIdx: uploadChunkTable.chunkIdx, data: uploadChunkTable.data }) - .from(uploadChunkTable) - .where(and( - eq(uploadChunkTable.uploadId, uploadId), - eq(uploadChunkTable.userId, user.id), - )) - .orderBy(asc(uploadChunkTable.chunkIdx)) - : []; - - if (chunksFromDb.length !== expectedFromDb) { - throw createHttpError( - `Expected ${expectedFromDb} staged chunks but found ${chunksFromDb.length}.`, - 400, - ); - } - for (let i = 0; i < chunksFromDb.length; i++) { - if (chunksFromDb[i].chunkIdx !== i + 1) { - throw createHttpError(`Missing chunk at index ${i + 1}.`, 400); - } - } - - const buffers = [ - Buffer.from(await firstChunk.arrayBuffer()), - ...chunksFromDb.map(c => c.data), - ]; - const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); - if (totalSize > MAX_TOTAL_BYTES) { - throw createHttpError( - `File too large (${totalSize} bytes). Max ${MAX_TOTAL_BYTES} bytes.`, - 413, - ); - } - const contentBase64 = Buffer.concat(buffers).toString("base64"); - - const schemaCommitTemplates = schema?.commit?.templates; - const schemaCommitIdentity = schema?.commit?.identity; - const commitIdentity = resolveCommitIdentity({ - configObject: config.object, - identityOverride: schemaCommitIdentity, - }); - const committer = (commitIdentity === "user" && user.email) - ? { name: user.name?.trim() || user.email, email: user.email } - : undefined; - - const response = await githubSaveFile( - token, - owner, - repo, - branch, - normalizedPath, - contentBase64, - sha, - { - configObject: config.object, - templatesOverride: schemaCommitTemplates, - contentName: name, - user: user.email || user.name || String(user.id || ""), - onConflict, - committer, - } - ); - - const savedPath = response?.data.content?.path; - - if (response?.data.content && response?.data.commit) { - await updateFileCache( - 'media', - owner, - repo, - branch, - { - type: sha ? 'modify' : 'add', - path: response.data.content.path!, - sha: response.data.content.sha!, - content: Buffer.from(contentBase64, 'base64').toString('utf-8'), - size: response.data.content.size, - downloadUrl: response.data.content.download_url, - commit: { - sha: response.data.commit.sha!, - timestamp: new Date(response.data.commit.committer?.date ?? new Date().toISOString()).getTime() - } - } - ); - } - - after(async () => { - try { - await db.delete(uploadChunkTable).where(or( - and( - eq(uploadChunkTable.uploadId, uploadId), - eq(uploadChunkTable.userId, user.id), - ), - lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), - )); - } catch (error) { - console.error("Chunk cleanup after finalize failed", error); - } - }); - - return Response.json({ - status: "success", - message: savedPath !== normalizedPath - ? `File "${normalizedPath}" saved successfully but renamed to "${savedPath}" to avoid naming conflict.` - : `File "${normalizedPath}" saved successfully.`, - data: { - type: response?.data.content?.type, - sha: response?.data.content?.sha, - name: response?.data.content?.name, - path: savedPath, - extension: getFileExtension(response?.data.content?.name || ""), - size: response?.data.content?.size, - url: response?.data.content?.download_url, - } - }); - } catch (error: any) { - if (!error?.status || error.status >= 500) console.error(error); - return toErrorResponse(error); - } -} diff --git a/app/api/[owner]/[repo]/[branch]/files/[path]/chunk/route.ts b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts similarity index 100% rename from app/api/[owner]/[repo]/[branch]/files/[path]/chunk/route.ts rename to app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts diff --git a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts index 64f13ee1b..17c908053 100644 --- a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts @@ -1,13 +1,30 @@ +import { after } from "next/server"; +import { db } from "@/db"; +import { uploadChunkTable } from "@/db/schema"; +import { and, asc, eq, lt, or } from "drizzle-orm"; import { getRepoReadContext } from "@/lib/api-repo-context"; -import { getFileExtension, normalizePath } from "@/lib/utils/file"; -import { getMediaCache } from "@/lib/github-cache-file"; +import { getFileExtension, getFileName, normalizePath } from "@/lib/utils/file"; +import { getMediaCache, updateFileCache } from "@/lib/github-cache-file"; import { createHttpError, toErrorResponse } from "@/lib/api-error"; +import { requireApiUserSession } from "@/lib/session-server"; +import { getToken } from "@/lib/token"; +import { getConfig } from "@/lib/config-store"; +import { getSchemaByName } from "@/lib/schema"; +import { resolveCommitIdentity } from "@/lib/commit-message"; +import { githubSaveFile } from "@/lib/utils/github-save-file"; + +const MAX_TOTAL_BYTES = 15 * 1024 * 1024; +const MAX_CHUNKS = 4; +const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; +const STALE_CHUNK_AGE_MS = 10 * 60 * 1000; /** - * Get the list of media files in a directory. + * GET: list media files in a directory. + * POST: finalize a chunked media upload (chunks staged via the /chunk sub-route). + * + * GET /api/[owner]/[repo]/[branch]/media/[name]/[path] + * POST /api/[owner]/[repo]/[branch]/media/[name]/[path] * - * GET /api/[owner]/[repo]/[branch]/media/[path] - * * Requires authentication. */ @@ -84,6 +101,184 @@ export async function GET( } } +export async function POST( + request: Request, + context: { params: Promise<{ owner: string, repo: string, branch: string, name: string, path: string }> } +) { + try { + const sessionResult = await requireApiUserSession(); + if ("response" in sessionResult) return sessionResult.response; + const user = sessionResult.user; + + const { owner, repo, branch, name, path } = await context.params; + + const form = await request.formData(); + const uploadId = typeof form.get("uploadId") === "string" ? form.get("uploadId") as string : ""; + const totalChunksRaw = form.get("totalChunks"); + const totalChunks = typeof totalChunksRaw === "string" ? parseInt(totalChunksRaw, 10) : NaN; + const shaRaw = form.get("sha"); + const sha = typeof shaRaw === "string" && shaRaw.length > 0 ? shaRaw : undefined; + const onConflict = form.get("onConflict") === "error" ? "error" : "rename"; + const firstChunk = form.get("firstChunk"); + + if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); + if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { + throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); + } + if (!name) throw createHttpError(`Missing "name".`, 400); + if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { + throw createHttpError(`Missing "firstChunk".`, 400); + } + if (firstChunk.size > MAX_INLINE_CHUNK_BYTES) { + throw createHttpError(`"firstChunk" too large.`, 413); + } + + const { token } = await getToken(user, owner, repo, true); + if (!token) throw new Error("Token not found"); + + const normalizedPath = normalizePath(path); + + const config = await getConfig(owner, repo, branch, { getToken: async () => token }); + if (!config) throw new Error(`Configuration not found for ${owner}/${repo}/${branch}.`); + + const schema = getSchemaByName(config.object, name, "media"); + if (!schema) throw new Error(`Media schema not found for ${name}.`); + if (!normalizedPath.startsWith(schema.input)) { + throw new Error(`Invalid path "${path}" for media "${name}".`); + } + if ( + schema.extensions?.length > 0 && + !schema.extensions.includes(getFileExtension(normalizedPath)) + ) { + throw new Error(`Invalid extension "${getFileExtension(normalizedPath)}" for media.`); + } + if (getFileName(normalizedPath) === ".gitkeep") { + throw createHttpError(`Use the files endpoint to create empty folders.`, 400); + } + + const expectedFromDb = totalChunks - 1; + const chunksFromDb = expectedFromDb > 0 + ? await db + .select({ chunkIdx: uploadChunkTable.chunkIdx, data: uploadChunkTable.data }) + .from(uploadChunkTable) + .where(and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + )) + .orderBy(asc(uploadChunkTable.chunkIdx)) + : []; + + if (chunksFromDb.length !== expectedFromDb) { + throw createHttpError( + `Expected ${expectedFromDb} staged chunks but found ${chunksFromDb.length}.`, + 400, + ); + } + for (let i = 0; i < chunksFromDb.length; i++) { + if (chunksFromDb[i].chunkIdx !== i + 1) { + throw createHttpError(`Missing chunk at index ${i + 1}.`, 400); + } + } + + const buffers = [ + Buffer.from(await firstChunk.arrayBuffer()), + ...chunksFromDb.map(c => c.data), + ]; + const totalSize = buffers.reduce((acc, b) => acc + b.length, 0); + if (totalSize > MAX_TOTAL_BYTES) { + throw createHttpError( + `File too large (${totalSize} bytes). Max ${MAX_TOTAL_BYTES} bytes.`, + 413, + ); + } + const contentBase64 = Buffer.concat(buffers).toString("base64"); + + const schemaCommitTemplates = schema?.commit?.templates; + const schemaCommitIdentity = schema?.commit?.identity; + const commitIdentity = resolveCommitIdentity({ + configObject: config.object, + identityOverride: schemaCommitIdentity, + }); + const committer = (commitIdentity === "user" && user.email) + ? { name: user.name?.trim() || user.email, email: user.email } + : undefined; + + const response = await githubSaveFile( + token, + owner, + repo, + branch, + normalizedPath, + contentBase64, + sha, + { + configObject: config.object, + templatesOverride: schemaCommitTemplates, + contentName: name, + user: user.email || user.name || String(user.id || ""), + onConflict, + committer, + } + ); + + const savedPath = response?.data.content?.path; + + if (response?.data.content && response?.data.commit) { + await updateFileCache( + 'media', + owner, + repo, + branch, + { + type: sha ? 'modify' : 'add', + path: response.data.content.path!, + sha: response.data.content.sha!, + content: Buffer.from(contentBase64, 'base64').toString('utf-8'), + size: response.data.content.size, + downloadUrl: response.data.content.download_url, + commit: { + sha: response.data.commit.sha!, + timestamp: new Date(response.data.commit.committer?.date ?? new Date().toISOString()).getTime() + } + } + ); + } + + after(async () => { + try { + await db.delete(uploadChunkTable).where(or( + and( + eq(uploadChunkTable.uploadId, uploadId), + eq(uploadChunkTable.userId, user.id), + ), + lt(uploadChunkTable.createdAt, new Date(Date.now() - STALE_CHUNK_AGE_MS)), + )); + } catch (error) { + console.error("Chunk cleanup after finalize failed", error); + } + }); + + return Response.json({ + status: "success", + message: savedPath !== normalizedPath + ? `File "${normalizedPath}" saved successfully but renamed to "${savedPath}" to avoid naming conflict.` + : `File "${normalizedPath}" saved successfully.`, + data: { + type: response?.data.content?.type, + sha: response?.data.content?.sha, + name: response?.data.content?.name, + path: savedPath, + extension: getFileExtension(response?.data.content?.name || ""), + size: response?.data.content?.size, + url: response?.data.content?.download_url, + } + }); + } catch (error: any) { + if (!error?.status || error.status >= 500) console.error(error); + return toErrorResponse(error); + } +} + const normalizeMediaPath = ( rawPath: string, owner: string, diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index f318dd7cf..5a74b4859 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -85,7 +85,7 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const uploadId = crypto.randomUUID(); const totalChunks = Math.ceil(file.size / CHUNK_BYTES); - const baseUrl = `/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullPath)}`; + const baseUrl = `/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/media/${encodeURIComponent(configMedia.name)}/${encodeURIComponent(fullPath)}`; // chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes const uploadChunk = async (idx: number) => { @@ -113,10 +113,9 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple const finalizeForm = new FormData(); finalizeForm.set("uploadId", uploadId); finalizeForm.set("totalChunks", String(totalChunks)); - finalizeForm.set("name", configMedia.name); finalizeForm.set("firstChunk", firstBlob); - const finalizeResponse = await fetch(`${baseUrl}/finalize`, { + const finalizeResponse = await fetch(baseUrl, { method: "POST", body: finalizeForm, }); From a9b00c7610e1d3331669593494ca3eae70dddd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Fri, 26 Jun 2026 08:20:28 +0200 Subject: [PATCH 15/17] Migrate rich-text image upload to chunked media endpoint --- .../[repo]/[branch]/files/[path]/route.ts | 15 ++--- components/media/media-upload.tsx | 63 +++---------------- fields/core/rich-text/edit-component.tsx | 42 +++---------- lib/utils/upload-media.ts | 60 ++++++++++++++++++ 4 files changed, 84 insertions(+), 96 deletions(-) create mode 100644 lib/utils/upload-media.ts diff --git a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts index df6e2704c..793ca3428 100644 --- a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts @@ -163,6 +163,9 @@ export async function POST( break; case "media": if (!data.name) throw new Error(`"name" is required for media.`); + if (getFileName(normalizedPath) !== ".gitkeep") { + throw new Error(`Media uploads must use POST /api/.../media/[name]/[path].`); + } schema = getSchemaByName(config?.object, data.name, "media"); if (!schema) throw new Error(`Media schema not found for ${data.name}.`); @@ -170,18 +173,8 @@ export async function POST( schemaCommitIdentity = schema?.commit?.identity; if (!normalizedPath.startsWith(schema.input)) throw new Error(`Invalid path "${params.path}" for media "${data.name}".`); - - if (getFileName(normalizedPath) === ".gitkeep") { - // Folder creation - contentBase64 = ""; - } else { - if ( - schema.extensions?.length > 0 && - !schema.extensions.includes(getFileExtension(normalizedPath)) - ) throw new Error(`Invalid extension "${getFileExtension(normalizedPath)}" for media.`); - contentBase64 = data.content; - } + contentBase64 = ""; break; case "settings": assertGithubIdentity(user, "Only GitHub users can manage settings."); diff --git a/components/media/media-upload.tsx b/components/media/media-upload.tsx index 5a74b4859..ffa275fe3 100644 --- a/components/media/media-upload.tsx +++ b/components/media/media-upload.tsx @@ -6,7 +6,7 @@ import { getUploadFileName, joinPathSegments } from "@/lib/utils/file"; import { toast } from "sonner"; import { getSchemaByName } from "@/lib/schema"; import { cn } from "@/lib/utils"; -import { requireApiSuccess } from "@/lib/api-client"; +import { uploadMediaChunked } from "@/lib/utils/upload-media"; import type { FileSaveData } from "@/types/api"; interface MediaUploadContextValue { @@ -64,11 +64,6 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple }, [extensions, configMedia?.extensions]); const handleFiles = useCallback(async (files: File[]) => { - // 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk - const CHUNK_BYTES = 4 * 1024 * 1024; - const MAX_TOTAL_BYTES = 15 * 1024 * 1024; - const CHUNK_CONCURRENCY = 4; - try { for (const file of files) { const uploadFilename = getUploadFileName( @@ -77,56 +72,18 @@ function MediaUploadRoot({ children, path, onUpload, media, extensions, multiple ); const fullPath = joinPathSegments([path ?? "", uploadFilename]); - const uploadPromise = (async () => { - if (file.size === 0) throw new Error("File is empty"); - if (file.size > MAX_TOTAL_BYTES) { - throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); - } - - const uploadId = crypto.randomUUID(); - const totalChunks = Math.ceil(file.size / CHUNK_BYTES); - const baseUrl = `/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/media/${encodeURIComponent(configMedia.name)}/${encodeURIComponent(fullPath)}`; - // chunk 0 is always CHUNK_BYTES (or the whole file if N=1); riding it inline maximizes savings on non-multiple sizes - - const uploadChunk = async (idx: number) => { - const start = idx * CHUNK_BYTES; - const end = Math.min(start + CHUNK_BYTES, file.size); - const blob = file.slice(start, end); - const form = new FormData(); - form.set("uploadId", uploadId); - form.set("idx", String(idx)); - form.set("chunk", blob); - const chunkResponse = await fetch(`${baseUrl}/chunk`, { method: "POST", body: form }); - await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); - }; - - // batched parallelism (4); switch to rolling pool if uneven chunk times matter - for (let i = 1; i < totalChunks; i += CHUNK_CONCURRENCY) { - const batch = []; - for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { - batch.push(uploadChunk(j)); - } - await Promise.all(batch); - } - - const firstBlob = file.slice(0, Math.min(CHUNK_BYTES, file.size)); - const finalizeForm = new FormData(); - finalizeForm.set("uploadId", uploadId); - finalizeForm.set("totalChunks", String(totalChunks)); - finalizeForm.set("firstChunk", firstBlob); - - const finalizeResponse = await fetch(baseUrl, { - method: "POST", - body: finalizeForm, - }); - - const data = await requireApiSuccess(finalizeResponse, "Failed to upload file"); - return data.data as FileSaveData; - })(); + const uploadPromise = uploadMediaChunked({ + file, + owner: config.owner, + repo: config.repo, + branch: config.branch, + mediaName: configMedia.name, + targetPath: fullPath, + }); await toast.promise(uploadPromise, { loading: `Uploading ${file.name}`, - success: (savedEntry) => { + success: (savedEntry: FileSaveData) => { onUpload?.(savedEntry); return `Uploaded ${file.name}`; }, diff --git a/fields/core/rich-text/edit-component.tsx b/fields/core/rich-text/edit-component.tsx index e18e67cce..77b31b6d2 100644 --- a/fields/core/rich-text/edit-component.tsx +++ b/fields/core/rich-text/edit-component.tsx @@ -38,7 +38,7 @@ import { normalizeMediaPath, normalizePath, } from "@/lib/utils/file"; -import type { ApiResponse, FileSaveData } from "@/types/api"; +import { uploadMediaChunked } from "@/lib/utils/upload-media"; import type { Field } from "@/types/field"; import "./edit-component.css"; @@ -772,14 +772,6 @@ const EditComponent = forwardRef( ); } - const dataUrl = await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(String(reader.result ?? "")); - reader.onerror = () => - reject(new Error("Failed to read image file.")); - reader.readAsDataURL(file); - }); - const content = dataUrl.replace(/^(.+,)/, ""); const uploadFilename = getUploadFileName( file.name, options.rename ?? mediaConfig.rename, @@ -789,30 +781,16 @@ const EditComponent = forwardRef( uploadFilename, ]); - const response = await fetch( - `/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(targetPath)}`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - type: "media", - name: mediaConfig.name, - content, - }), - }, - ); - if (!response.ok) { - throw new Error( - `Failed to upload file: ${response.status} ${response.statusText}`, - ); - } - - const payload = (await response.json()) as ApiResponse; - if (payload.status !== "success") { - throw new Error(payload.message); - } + const saved = await uploadMediaChunked({ + file, + owner: config.owner, + repo: config.repo, + branch: config.branch, + mediaName: mediaConfig.name, + targetPath, + }); - const uploadedPath = payload.data.path || targetPath; + const uploadedPath = saved.path || targetPath; const src = await toDisplayImageUrl(uploadedPath); return { src, diff --git a/lib/utils/upload-media.ts b/lib/utils/upload-media.ts new file mode 100644 index 000000000..ebc3672e9 --- /dev/null +++ b/lib/utils/upload-media.ts @@ -0,0 +1,60 @@ +import { requireApiSuccess } from "@/lib/api-client"; +import type { FileSaveData } from "@/types/api"; + +// 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk +const CHUNK_BYTES = 4 * 1024 * 1024; +const MAX_TOTAL_BYTES = 15 * 1024 * 1024; +const CHUNK_CONCURRENCY = 4; + +export const MAX_MEDIA_UPLOAD_BYTES = MAX_TOTAL_BYTES; + +export async function uploadMediaChunked(opts: { + file: File; + owner: string; + repo: string; + branch: string; + mediaName: string; + targetPath: string; +}): Promise { + const { file, owner, repo, branch, mediaName, targetPath } = opts; + + if (file.size === 0) throw new Error("File is empty"); + if (file.size > MAX_TOTAL_BYTES) { + throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); + } + + const uploadId = crypto.randomUUID(); + const totalChunks = Math.ceil(file.size / CHUNK_BYTES); + const baseUrl = `/api/${owner}/${repo}/${encodeURIComponent(branch)}/media/${encodeURIComponent(mediaName)}/${encodeURIComponent(targetPath)}`; + + const uploadChunk = async (idx: number) => { + const start = idx * CHUNK_BYTES; + const end = Math.min(start + CHUNK_BYTES, file.size); + const blob = file.slice(start, end); + const form = new FormData(); + form.set("uploadId", uploadId); + form.set("idx", String(idx)); + form.set("chunk", blob); + const chunkResponse = await fetch(`${baseUrl}/chunk`, { method: "POST", body: form }); + await requireApiSuccess(chunkResponse, `Failed to upload chunk ${idx + 1}/${totalChunks}`); + }; + + // batched parallelism (4); switch to rolling pool if uneven chunk times matter + for (let i = 1; i < totalChunks; i += CHUNK_CONCURRENCY) { + const batch = []; + for (let j = i; j < Math.min(i + CHUNK_CONCURRENCY, totalChunks); j++) { + batch.push(uploadChunk(j)); + } + await Promise.all(batch); + } + + const firstBlob = file.slice(0, Math.min(CHUNK_BYTES, file.size)); + const finalizeForm = new FormData(); + finalizeForm.set("uploadId", uploadId); + finalizeForm.set("totalChunks", String(totalChunks)); + finalizeForm.set("firstChunk", firstBlob); + + const response = await fetch(baseUrl, { method: "POST", body: finalizeForm }); + const data = await requireApiSuccess(response, "Failed to upload file"); + return data.data as FileSaveData; +} From 7f80f7e78b4f47936eb761ed14228d93562a163c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Fri, 26 Jun 2026 09:14:32 +0200 Subject: [PATCH 16/17] Share upload-media constants and tighten chunk idx cap --- .../[branch]/media/[name]/[path]/chunk/route.ts | 12 +++++++----- .../[repo]/[branch]/media/[name]/[path]/route.ts | 10 ++++------ lib/utils/upload-media.ts | 7 +++---- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts index d42ceae66..58ecba5fc 100644 --- a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts +++ b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/chunk/route.ts @@ -3,8 +3,10 @@ import { uploadChunkTable } from "@/db/schema"; import { eq } from "drizzle-orm"; import { requireApiUserSession } from "@/lib/session-server"; import { createHttpError, toErrorResponse } from "@/lib/api-error"; +import { CHUNK_BYTES, MAX_TOTAL_BYTES } from "@/lib/utils/upload-media"; -const MAX_CHUNK_BYTES = 4 * 1024 * 1024; +// chunk 0 rides inline in finalize, chunks 1..MAX_CHUNK_IDX go here +const MAX_CHUNK_IDX = Math.ceil(MAX_TOTAL_BYTES / CHUNK_BYTES) - 1; export async function POST(request: Request) { try { @@ -21,14 +23,14 @@ export async function POST(request: Request) { throw createHttpError(`Invalid "uploadId".`, 400); } const idx = typeof idxRaw === "string" ? parseInt(idxRaw, 10) : NaN; - if (!Number.isInteger(idx) || idx < 0 || idx > 9999) { - throw createHttpError(`Invalid "idx".`, 400); + if (!Number.isInteger(idx) || idx < 1 || idx > MAX_CHUNK_IDX) { + throw createHttpError(`"idx" must be between 1 and ${MAX_CHUNK_IDX}.`, 400); } if (!(chunk instanceof Blob)) { throw createHttpError(`Invalid "chunk".`, 400); } - if (chunk.size === 0 || chunk.size > MAX_CHUNK_BYTES) { - throw createHttpError(`Chunk size must be between 1 and ${MAX_CHUNK_BYTES} bytes.`, 413); + if (chunk.size === 0 || chunk.size > CHUNK_BYTES) { + throw createHttpError(`Chunk size must be between 1 and ${CHUNK_BYTES} bytes.`, 413); } const buffer = Buffer.from(await chunk.arrayBuffer()); diff --git a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts index 17c908053..12bf62b4a 100644 --- a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts @@ -12,10 +12,8 @@ import { getConfig } from "@/lib/config-store"; import { getSchemaByName } from "@/lib/schema"; import { resolveCommitIdentity } from "@/lib/commit-message"; import { githubSaveFile } from "@/lib/utils/github-save-file"; +import { CHUNK_BYTES, MAX_TOTAL_BYTES } from "@/lib/utils/upload-media"; -const MAX_TOTAL_BYTES = 15 * 1024 * 1024; -const MAX_CHUNKS = 4; -const MAX_INLINE_CHUNK_BYTES = 4 * 1024 * 1024; const STALE_CHUNK_AGE_MS = 10 * 60 * 1000; /** @@ -122,14 +120,14 @@ export async function POST( const firstChunk = form.get("firstChunk"); if (!uploadId || uploadId.length > 64) throw createHttpError(`Invalid "uploadId".`, 400); - if (!Number.isInteger(totalChunks) || totalChunks < 1 || totalChunks > MAX_CHUNKS) { - throw createHttpError(`"totalChunks" must be between 1 and ${MAX_CHUNKS}.`, 400); + if (!Number.isInteger(totalChunks) || totalChunks < 1) { + throw createHttpError(`"totalChunks" must be a positive integer.`, 400); } if (!name) throw createHttpError(`Missing "name".`, 400); if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { throw createHttpError(`Missing "firstChunk".`, 400); } - if (firstChunk.size > MAX_INLINE_CHUNK_BYTES) { + if (firstChunk.size > CHUNK_BYTES) { throw createHttpError(`"firstChunk" too large.`, 413); } diff --git a/lib/utils/upload-media.ts b/lib/utils/upload-media.ts index ebc3672e9..698baf0e5 100644 --- a/lib/utils/upload-media.ts +++ b/lib/utils/upload-media.ts @@ -2,11 +2,10 @@ import { requireApiSuccess } from "@/lib/api-client"; import type { FileSaveData } from "@/types/api"; // 4 MB binary fits in multipart body (overhead < 1 KB); raise above 4 MB at your own risk -const CHUNK_BYTES = 4 * 1024 * 1024; -const MAX_TOTAL_BYTES = 15 * 1024 * 1024; -const CHUNK_CONCURRENCY = 4; +export const CHUNK_BYTES = 4 * 1024 * 1024; +export const MAX_TOTAL_BYTES = 15 * 1024 * 1024; -export const MAX_MEDIA_UPLOAD_BYTES = MAX_TOTAL_BYTES; +const CHUNK_CONCURRENCY = 4; export async function uploadMediaChunked(opts: { file: File; From db6ffd801069b8ad414b7c084828830c5e473606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Jim=C3=A9nez=20Delgado?= Date: Fri, 26 Jun 2026 10:23:19 +0200 Subject: [PATCH 17/17] Route media .gitkeep folder creation through chunked endpoint --- .../[repo]/[branch]/files/[path]/route.ts | 15 ----- .../[branch]/media/[name]/[path]/route.ts | 11 ++-- components/folder-create.tsx | 59 ++++++++++++------- lib/utils/upload-media.ts | 7 ++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts index 793ca3428..d02a7e45a 100644 --- a/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/files/[path]/route.ts @@ -161,21 +161,6 @@ export async function POST( } } break; - case "media": - if (!data.name) throw new Error(`"name" is required for media.`); - if (getFileName(normalizedPath) !== ".gitkeep") { - throw new Error(`Media uploads must use POST /api/.../media/[name]/[path].`); - } - - schema = getSchemaByName(config?.object, data.name, "media"); - if (!schema) throw new Error(`Media schema not found for ${data.name}.`); - schemaCommitTemplates = schema?.commit?.templates; - schemaCommitIdentity = schema?.commit?.identity; - - if (!normalizedPath.startsWith(schema.input)) throw new Error(`Invalid path "${params.path}" for media "${data.name}".`); - - contentBase64 = ""; - break; case "settings": assertGithubIdentity(user, "Only GitHub users can manage settings."); if (normalizedPath !== ".pages.yml") throw new Error(`Invalid path "${params.path}" for settings.`); diff --git a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts index 12bf62b4a..aefe91c5d 100644 --- a/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts +++ b/app/api/[owner]/[repo]/[branch]/media/[name]/[path]/route.ts @@ -124,7 +124,7 @@ export async function POST( throw createHttpError(`"totalChunks" must be a positive integer.`, 400); } if (!name) throw createHttpError(`Missing "name".`, 400); - if (!(firstChunk instanceof Blob) || firstChunk.size === 0) { + if (!(firstChunk instanceof Blob)) { throw createHttpError(`Missing "firstChunk".`, 400); } if (firstChunk.size > CHUNK_BYTES) { @@ -135,6 +135,11 @@ export async function POST( if (!token) throw new Error("Token not found"); const normalizedPath = normalizePath(path); + const isFolderMarker = getFileName(normalizedPath) === ".gitkeep"; + + if (firstChunk.size === 0 && !isFolderMarker) { + throw createHttpError(`Empty "firstChunk" only allowed for .gitkeep folder markers.`, 400); + } const config = await getConfig(owner, repo, branch, { getToken: async () => token }); if (!config) throw new Error(`Configuration not found for ${owner}/${repo}/${branch}.`); @@ -145,14 +150,12 @@ export async function POST( throw new Error(`Invalid path "${path}" for media "${name}".`); } if ( + !isFolderMarker && schema.extensions?.length > 0 && !schema.extensions.includes(getFileExtension(normalizedPath)) ) { throw new Error(`Invalid extension "${getFileExtension(normalizedPath)}" for media.`); } - if (getFileName(normalizedPath) === ".gitkeep") { - throw createHttpError(`Use the files endpoint to create empty folders.`, 400); - } const expectedFromDb = totalChunks - 1; const chunksFromDb = expectedFromDb > 0 diff --git a/components/folder-create.tsx b/components/folder-create.tsx index a26075d10..d6bda3b23 100644 --- a/components/folder-create.tsx +++ b/components/folder-create.tsx @@ -16,6 +16,7 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; +import { uploadMediaChunked } from "@/lib/utils/upload-media"; type FolderCreateResult = { path: string; @@ -56,35 +57,49 @@ const FolderCreate = ({ setIsSubmitting(true); try { - const createPromise: Promise<{ + let createPromise: Promise<{ status: string; message?: string; data: FolderCreateResult; - }> = fetch(`/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullNewPath + "/.gitkeep")}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - type, - name, - content: "", + }>; + + if (type === "media") { + createPromise = uploadMediaChunked({ + file: new File([], ".gitkeep"), + owner: config.owner, + repo: config.repo, + branch: config.branch, + mediaName: name!, + targetPath: `${fullNewPath}/.gitkeep`, onConflict: "error", - }), - }).then(async (response) => { - const payload = await response.json().catch(() => null); - if (!response.ok) { - if (response.status === 409) { - throw new Error(`Folder \"${fullNewPath}\" already exists.`); - } + }).then((data) => ({ status: "success", data: data as FolderCreateResult })); + } else { + createPromise = fetch(`/api/${config.owner}/${config.repo}/${encodeURIComponent(config.branch)}/files/${encodeURIComponent(fullNewPath + "/.gitkeep")}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + type, + name, + content: "", + onConflict: "error", + }), + }).then(async (response) => { + const payload = await response.json().catch(() => null); + if (!response.ok) { + if (response.status === 409) { + throw new Error(`Folder \"${fullNewPath}\" already exists.`); + } - throw new Error(payload?.message || "Failed to create folder"); - } + throw new Error(payload?.message || "Failed to create folder"); + } - if (!payload || payload.status !== "success") { - throw new Error(payload?.message || "Failed to create folder"); - } + if (!payload || payload.status !== "success") { + throw new Error(payload?.message || "Failed to create folder"); + } - return payload; - }); + return payload; + }); + } await toast.promise(createPromise, { loading: `Creating folder "${fullNewPath}"`, diff --git a/lib/utils/upload-media.ts b/lib/utils/upload-media.ts index 698baf0e5..64578ea24 100644 --- a/lib/utils/upload-media.ts +++ b/lib/utils/upload-media.ts @@ -14,16 +14,16 @@ export async function uploadMediaChunked(opts: { branch: string; mediaName: string; targetPath: string; + onConflict?: "error" | "rename"; }): Promise { - const { file, owner, repo, branch, mediaName, targetPath } = opts; + const { file, owner, repo, branch, mediaName, targetPath, onConflict } = opts; - if (file.size === 0) throw new Error("File is empty"); if (file.size > MAX_TOTAL_BYTES) { throw new Error(`File too large. Max ${Math.floor(MAX_TOTAL_BYTES / 1024 / 1024)} MB.`); } const uploadId = crypto.randomUUID(); - const totalChunks = Math.ceil(file.size / CHUNK_BYTES); + const totalChunks = Math.max(1, Math.ceil(file.size / CHUNK_BYTES)); const baseUrl = `/api/${owner}/${repo}/${encodeURIComponent(branch)}/media/${encodeURIComponent(mediaName)}/${encodeURIComponent(targetPath)}`; const uploadChunk = async (idx: number) => { @@ -52,6 +52,7 @@ export async function uploadMediaChunked(opts: { finalizeForm.set("uploadId", uploadId); finalizeForm.set("totalChunks", String(totalChunks)); finalizeForm.set("firstChunk", firstBlob); + if (onConflict) finalizeForm.set("onConflict", onConflict); const response = await fetch(baseUrl, { method: "POST", body: finalizeForm }); const data = await requireApiSuccess(response, "Failed to upload file");