diff --git a/apps/web/src/components/uploader.tsx b/apps/web/src/components/uploader.tsx index 766e208..10d36d0 100644 --- a/apps/web/src/components/uploader.tsx +++ b/apps/web/src/components/uploader.tsx @@ -9,9 +9,73 @@ import { getImportSummary, type ImportSummary } from "@/lib/api"; import { bulkUpload, isArchive, type BulkProgress } from "@/lib/bulk-upload"; import { useAuth } from "@/lib/auth"; -const ACCEPT = ".txt,.zip,.rar,.7z,.tar,.gz,.tgz,.bz2,.xz"; +const ARCHIVE_ACCEPT = ".zip,.rar,.7z,.tar,.gz,.tgz,.bz2,.xz"; -/** Folder / files / archive upload with live import report. Calls onComplete when done. */ +// readEntries yields a directory in batches; keep calling until it returns an empty batch. +async function readAllEntries(reader: FileSystemDirectoryReader): Promise { + const out: FileSystemEntry[] = []; + for (;;) { + const batch = await new Promise((res, rej) => reader.readEntries(res, rej)); + if (batch.length === 0) break; + out.push(...batch); + } + return out; +} + +// Resolve file entries in bounded batches: firing thousands of FileSystemFileEntry.file() calls at +// once (a folder of 10k+ files) overwhelms Chromium and the whole drop silently fails. +const FILE_RESOLVE_BATCH = 64; + +// Collect dropped folders and archives only — loose individual files are intentionally ignored +// (uploading is folder- or archive-based). Folder entries are walked breadth-first with bounded +// concurrency; an archive is taken via getAsFile(). We decide archive-vs-file from entry.name +// (sync, no file resolution) so dropping a huge loose-file selection never touches the files. +// `onCount` reports progress on large folder drops. +async function filesFromDrop(dt: DataTransfer, onCount?: (n: number) => void): Promise { + const items = Array.from(dt.items).filter((i) => i.kind === "file"); + // Snapshot the entries synchronously (they must be read before the event yields). + const pairs = items.map((item) => ({ item, entry: item.webkitGetAsEntry?.() ?? null })); + + // Browser without the entry API: keep only archives from the flat list. + if (pairs.every((p) => p.entry == null)) { + const archives = Array.from(dt.files).filter((f) => isArchive(f.name)); + onCount?.(archives.length); + return archives; + } + + const files: File[] = []; + const dirQueue: FileSystemDirectoryEntry[] = []; + for (const p of pairs) { + if (p.entry?.isDirectory) { + dirQueue.push(p.entry as FileSystemDirectoryEntry); + } else if (p.entry && isArchive(p.entry.name)) { + const f = p.item.getAsFile(); // archive — grab it now, while the item is valid + if (f) files.push(f); + } + // loose non-archive files are ignored + } + onCount?.(files.length); + + while (dirQueue.length > 0) { + const children = await readAllEntries(dirQueue.shift()!.createReader()); + const fileEntries: FileSystemFileEntry[] = []; + for (const c of children) { + if (c.isDirectory) dirQueue.push(c as FileSystemDirectoryEntry); + else fileEntries.push(c as FileSystemFileEntry); + } + for (let i = 0; i < fileEntries.length; i += FILE_RESOLVE_BATCH) { + const slice = fileEntries.slice(i, i + FILE_RESOLVE_BATCH); + const got = await Promise.all( + slice.map((fe) => new Promise((res, rej) => fe.file(res, rej))), + ); + files.push(...got); + onCount?.(files.length); + } + } + return files; +} + +/** Folder / files / archive upload (button or drag-and-drop) with live import report. */ export function Uploader({ onComplete }: { onComplete?: () => void }) { const { session } = useAuth(); const token = session?.access_token ?? ""; @@ -22,12 +86,45 @@ export function Uploader({ onComplete }: { onComplete?: () => void }) { const [progress, setProgress] = React.useState(null); const [report, setReport] = React.useState(null); const [error, setError] = React.useState(null); + const [dragOver, setDragOver] = React.useState(false); + const [scanCount, setScanCount] = React.useState(null); // reading a dropped folder + const dragDepth = React.useRef(0); // ignore dragleave from child elements (enter/leave flicker) - function pick(list: FileList | null) { - if (!list) return; - setFiles(Array.from(list).filter((f) => f.name.endsWith(".txt") || isArchive(f.name))); + function pickFiles(list: File[]) { + setFiles(list.filter((f) => f.name.endsWith(".txt") || isArchive(f.name))); setReport(null); setProgress(null); + setError(null); + } + + function pick(list: FileList | null) { + if (list) pickFiles(Array.from(list)); + } + + async function onDrop(e: React.DragEvent) { + e.preventDefault(); + dragDepth.current = 0; + setDragOver(false); + if (busy) return; + setScanCount(0); + try { + pickFiles(await filesFromDrop(e.dataTransfer, setScanCount)); + } catch (err) { + setError(`Could not read the dropped items: ${String(err)}`); + } finally { + setScanCount(null); + } + } + + function onDragEnter(e: React.DragEvent) { + e.preventDefault(); + dragDepth.current += 1; + if (!busy) setDragOver(true); + } + + function onDragLeave() { + dragDepth.current -= 1; + if (dragDepth.current <= 0) setDragOver(false); } async function start() { @@ -43,6 +140,7 @@ export function Uploader({ onComplete }: { onComplete?: () => void }) { if (s.complete) break; await new Promise((r) => setTimeout(r, 1500)); } + setFiles([]); // consumed — require a fresh selection before Upload is enabled again onComplete?.(); } catch (e) { setError(String(e)); @@ -52,23 +150,60 @@ export function Uploader({ onComplete }: { onComplete?: () => void }) { } const pct = - progress && progress.filesTotal > 0 - ? Math.round((progress.filesRead / progress.filesTotal) * 100) + progress && progress.bytesTotal > 0 + ? Math.min(100, Math.round((progress.bytesUploaded / progress.bytesTotal) * 100)) : 0; return ( - Upload hand history — folder, files, or an archive (.zip/.rar/.7z/…) + Upload hand history — a folder or an archive (.zip/.rar/.7z/…) -
- - +
e.preventDefault()} + onDragEnter={onDragEnter} + onDragLeave={onDragLeave} + className={`flex flex-col items-center gap-3 rounded-lg border-2 border-dashed p-6 text-center transition-colors ${ + dragOver ? "border-primary bg-primary/5" : "border-border" + }`} + > + +

+ Drag & drop a folder or an archive (.zip/.rar/.7z/…) here — or +

+
+ + + +
+ {scanCount !== null ? ( +

+ Reading {scanCount.toLocaleString("en-US")} files… +

+ ) : ( + files.length > 0 && + !busy && ( +

+ {files.length.toLocaleString("en-US")} file{files.length === 1 ? "" : "s"} ready to upload +

+ ) + )} void }) { ref={filesRef} type="file" multiple - accept={ACCEPT} + accept={ARCHIVE_ACCEPT} className="hidden" onChange={(e) => pick(e.target.files)} /> - {files.length > 0 && ( - - )}
{progress && ( diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index d928b69..3ff788b 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -56,24 +56,36 @@ export async function uploadBulk( return gzipped.byteLength; } -/** Upload one archive (zip/rar/7z/…) for server-side extraction; returns its byte count. */ -export async function uploadArchive( +/** + * Upload one archive (zip/rar/7z/…) for server-side extraction; returns its byte count. + * Uses XHR so the caller can track real upload progress via `onUploadProgress(loadedBytes)` — + * fetch() does not expose request-body upload progress. + */ +export function uploadArchive( token: string, file: File, sessionId: string, + onUploadProgress?: (loaded: number) => void, ): Promise { - const res = await fetch(`${API_URL}/ingest/archive`, { - method: "POST", - body: file, - headers: { - Authorization: `Bearer ${token}`, - "Content-Type": "application/octet-stream", - "X-Upload-Session": sessionId, - "X-Filename": encodeURIComponent(file.name), - }, + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("POST", `${API_URL}/ingest/archive`); + xhr.setRequestHeader("Authorization", `Bearer ${token}`); + xhr.setRequestHeader("Content-Type", "application/octet-stream"); + xhr.setRequestHeader("X-Upload-Session", sessionId); + xhr.setRequestHeader("X-Filename", encodeURIComponent(file.name)); + if (onUploadProgress) { + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) onUploadProgress(e.loaded); + }; + } + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) resolve(file.size); + else reject(new Error(`API ${xhr.status}: ${xhr.responseText}`)); + }; + xhr.onerror = () => reject(new Error("Network error during archive upload")); + xhr.send(file); }); - if (!res.ok) throw new Error(`API ${res.status}: ${await res.text()}`); - return file.size; } /** Aggregated added/skipped report for one upload session. */ diff --git a/apps/web/src/lib/bulk-upload.ts b/apps/web/src/lib/bulk-upload.ts index 1ca987d..80a36eb 100644 --- a/apps/web/src/lib/bulk-upload.ts +++ b/apps/web/src/lib/bulk-upload.ts @@ -21,7 +21,8 @@ export interface BulkProgress { filesSkipped: number; chunksUploaded: number; chunksStarted: number; - bytesUploaded: number; + bytesUploaded: number; // original bytes sent so far (drives the progress bar) + bytesTotal: number; // sum of selected file sizes, known upfront } function gzipAsync(data: Uint8Array): Promise { @@ -72,18 +73,20 @@ export async function bulkUpload( chunksUploaded: 0, chunksStarted: 0, bytesUploaded: 0, + bytesTotal: files.reduce((s, f) => s + f.size, 0), }; - // Run an upload task under backpressure (at most CONCURRENCY in flight). - const run = (taskFn: () => Promise) => { + // Run a text-bundle upload under backpressure (at most CONCURRENCY in flight). `bytes` is the + // original (uncompressed) size of the bundle, credited to the bar once the chunk lands. + const run = (bytes: number, taskFn: () => Promise) => { p.chunksStarted += 1; onProgress({ ...p }); const task = (async () => { await sem.acquire(); try { - const n = await taskFn(); + await taskFn(); p.chunksUploaded += 1; - p.bytesUploaded += n; + p.bytesUploaded += bytes; onProgress({ ...p }); } finally { sem.release(); @@ -92,44 +95,72 @@ export async function bulkUpload( inFlight.push(task); }; - const ship = async (text: string) => - run(async () => uploadBulk(token, await gzipAsync(encoder.encode(text)), sessionId)); + const ship = async (text: string, bytes: number) => + run(bytes, async () => uploadBulk(token, await gzipAsync(encoder.encode(text)), sessionId)); + + // Stream one archive to the server, crediting the bar from real upload progress (XHR). + const shipArchive = (a: File) => { + p.chunksStarted += 1; + onProgress({ ...p }); + const task = (async () => { + await sem.acquire(); + try { + let last = 0; + await uploadArchive(token, a, sessionId, (loaded) => { + p.bytesUploaded += loaded - last; + last = loaded; + onProgress({ ...p }); + }); + p.bytesUploaded += a.size - last; // settle any rounding from the last progress event + p.filesRead += 1; + p.chunksUploaded += 1; + onProgress({ ...p }); + } finally { + sem.release(); + } + })(); + inFlight.push(task); + }; let hh: string[] = []; let hhSize = 0; + let hhBytes = 0; let sm: string[] = []; let smSize = 0; + let smBytes = 0; // Read files with bounded concurrency (disk I/O parallelism) while bundling sequentially. const READ_CONCURRENCY = 16; for (let i = 0; i < files.length; i += READ_CONCURRENCY) { const batch = files.slice(i, i + READ_CONCURRENCY); // Archives go straight to the server (extracted there); the rest are read as text. - const archives = batch.filter((f) => isArchive(f.name)); - for (const a of archives) { - p.filesRead += 1; - run(() => uploadArchive(token, a, sessionId)); - } + for (const a of batch.filter((f) => isArchive(f.name))) shipArchive(a); const textFiles = batch.filter((f) => !isArchive(f.name)); const texts = await Promise.all(textFiles.map((f) => f.text())); - for (const text of texts) { + for (let j = 0; j < textFiles.length; j++) { + const text = texts[j]; + const fileBytes = textFiles[j].size; p.filesRead += 1; const kind = classify(text); if (kind === "hh") { hh.push(text); hhSize += text.length; + hhBytes += fileBytes; if (hhSize >= CHUNK_BYTES) { - await ship(hh.join("\n")); + await ship(hh.join("\n"), hhBytes); hh = []; hhSize = 0; + hhBytes = 0; } } else if (kind === "summary") { sm.push(text); smSize += text.length; + smBytes += fileBytes; if (smSize >= CHUNK_BYTES) { - await ship(sm.join("\n")); + await ship(sm.join("\n"), smBytes); sm = []; smSize = 0; + smBytes = 0; } } else { p.filesSkipped += 1; @@ -138,9 +169,10 @@ export async function bulkUpload( onProgress({ ...p }); } - if (hh.length) await ship(hh.join("\n")); - if (sm.length) await ship(sm.join("\n")); + if (hh.length) await ship(hh.join("\n"), hhBytes); + if (sm.length) await ship(sm.join("\n"), smBytes); await Promise.all(inFlight); + p.bytesUploaded = p.bytesTotal; // settle the bar to 100% (skipped files never shipped bytes) onProgress({ ...p }); return { sessionId, progress: p }; }