From 6cae426821a3add67f7ff2d3f216c2a640d883ea Mon Sep 17 00:00:00 2001 From: grenkoff Date: Mon, 15 Jun 2026 03:21:31 +0500 Subject: [PATCH] Add archive upload (zip/rar/7z/... via libarchive) and move upload UI onto the dashboard --- .github/workflows/ci.yml | 3 + apps/api/Dockerfile | 3 + apps/api/pyproject.toml | 1 + apps/api/src/spin2note_api/ingest/archives.py | 33 ++++ apps/api/src/spin2note_api/ingest/router.py | 36 ++++- apps/api/src/spin2note_api/worker.py | 57 ++++++- apps/api/tests/test_archives.py | 39 +++++ apps/api/uv.lock | 11 ++ apps/web/app/page.tsx | 7 +- apps/web/app/upload/page.tsx | 144 ------------------ apps/web/src/components/app-shell.tsx | 7 +- apps/web/src/components/uploader.tsx | 132 ++++++++++++++++ apps/web/src/lib/api.ts | 20 +++ apps/web/src/lib/api/schema.ts | 55 +++++++ apps/web/src/lib/bulk-upload.ts | 30 +++- 15 files changed, 416 insertions(+), 162 deletions(-) create mode 100644 apps/api/src/spin2note_api/ingest/archives.py create mode 100644 apps/api/tests/test_archives.py delete mode 100644 apps/web/app/upload/page.tsx create mode 100644 apps/web/src/components/uploader.tsx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 604b9fd..d8490f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install system libs (libarchive for archive extraction) + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libarchive13 + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index a8bec3e..1675427 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -25,6 +25,9 @@ RUN cd apps/api && uv sync FROM python:3.12-slim-bookworm AS runtime WORKDIR /app/apps/api +# libarchive13: archive extraction (zip/rar/7z/tar/...) via libarchive-c. +RUN apt-get update && apt-get install -y --no-install-recommends libarchive13 \ + && rm -rf /var/lib/apt/lists/* ENV PATH="/app/apps/api/.venv/bin:${PATH}" COPY --from=builder /app/apps/api/.venv /app/apps/api/.venv COPY --from=builder /app/apps/api/src /app/apps/api/src diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index d8610d5..496e5ef 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "asynch>=0.2.5", "redis>=5.2", "minio>=7.2", + "libarchive-c>=5.1", "sqlalchemy>=2.0", "sqlmodel>=0.0.22", "alembic>=1.14", diff --git a/apps/api/src/spin2note_api/ingest/archives.py b/apps/api/src/spin2note_api/ingest/archives.py new file mode 100644 index 0000000..05dfd0e --- /dev/null +++ b/apps/api/src/spin2note_api/ingest/archives.py @@ -0,0 +1,33 @@ +"""Archive extraction for bulk upload. + +One engine (``libarchive``) reads zip, rar, 7z, tar and tar.gz/bz2/xz, so users can upload a +single archive of their whole base instead of selecting files. We pull out the ``.txt`` members +(hand histories + tournament summaries) and hand them to the normal parse pipeline. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +# Extensions routed to server-side extraction (vs the client-side .txt bundling path). +ARCHIVE_SUFFIXES = (".zip", ".rar", ".7z", ".tar", ".tar.gz", ".tgz", ".gz", ".bz2", ".xz") + + +def is_archive(filename: str) -> bool: + name = filename.lower() + return any(name.endswith(s) for s in ARCHIVE_SUFFIXES) + + +def extract_text_members(data: bytes) -> Iterator[tuple[str, str]]: + """Yield (name, text) for every ``.txt`` file inside an archive blob.""" + import libarchive # imported lazily; needs the system libarchive shared lib + + with libarchive.memory_reader(data) as archive: + for entry in archive: + if not entry.isfile: + continue + name = str(entry.pathname) + if not name.lower().endswith(".txt"): + continue + raw = b"".join(entry.get_blocks()) + yield name, raw.decode("utf-8", errors="replace") diff --git a/apps/api/src/spin2note_api/ingest/router.py b/apps/api/src/spin2note_api/ingest/router.py index 80391f3..a0447af 100644 --- a/apps/api/src/spin2note_api/ingest/router.py +++ b/apps/api/src/spin2note_api/ingest/router.py @@ -40,7 +40,7 @@ def _session_id(header: str | None) -> UUID: async def _register_and_enqueue( - settings: Settings, key: str, user_id: str, session_id: UUID + settings: Settings, key: str, user_id: str, session_id: UUID, kind: str = "file" ) -> UUID: import_id = new_id() await create_import( @@ -49,11 +49,12 @@ async def _register_and_enqueue( # Imported lazily so the module has no hard Redis dependency at import time. from ..cache import RedisQueue + job = {"object_key": key, "user_id": user_id, "import_id": str(import_id)} + if kind != "file": + job["kind"] = kind queue = RedisQueue(settings.redis_url) try: - await queue.enqueue( - {"object_key": key, "user_id": user_id, "import_id": str(import_id)} - ) + await queue.enqueue(job) finally: await queue.close() return import_id @@ -114,3 +115,30 @@ async def upload_bulk( object_key=key, bytes=len(body), queued=True, import_id=str(import_id), session_id=str(session_id), ) + + +@router.post("/archive", response_model=UploadResult) +async def upload_archive( + request: Request, + settings: Settings = Depends(get_settings), + user_id: str = Depends(current_user_id), + x_upload_session: str | None = Header(default=None), + x_filename: str | None = Header(default=None), +) -> UploadResult: + """Accept a single archive (zip/rar/7z/tar/gz/…) and extract its .txt members server-side. + + The raw archive is staged as-is (already compressed); the worker uses libarchive to pull out + the hand-history/summary files and feeds them through the normal parse + dedup pipeline. + """ + body = await request.body() + stamp = datetime.now(UTC).strftime("%Y/%m/%d") + name = (x_filename or "upload.zip").rsplit("/", 1)[-1] + key = f"{user_id}/archive/{stamp}/{new_id()}-{name}" + + session_id = _session_id(x_upload_session) + _stage(settings, key, body) + import_id = await _register_and_enqueue(settings, key, user_id, session_id, kind="archive") + return UploadResult( + object_key=key, bytes=len(body), queued=True, + import_id=str(import_id), session_id=str(session_id), + ) diff --git a/apps/api/src/spin2note_api/worker.py b/apps/api/src/spin2note_api/worker.py index 56ff977..cbccd7d 100644 --- a/apps/api/src/spin2note_api/worker.py +++ b/apps/api/src/spin2note_api/worker.py @@ -25,6 +25,7 @@ from .clickhouse.queries import existing_hand_ids, existing_tournament_ids, user_has_rows from .config import get_settings from .db.imports import ImportCounts, finish_import +from .ingest.archives import extract_text_members from .ingest.storage import RawStorage from .logging_config import bind_log_context, configure_logging, reset_log_context from .parser import ( @@ -87,6 +88,55 @@ async def handle_raw( ) +async def _ingest_chunked( + parts: list[str], user_id: UUID, batcher: ClickHouseBatcher, client: AsyncClient +) -> ImportCounts: + """Parse same-blob parts in ~32 MB chunks (bounds memory for huge archives).""" + out = ImportCounts() + buf: list[str] = [] + size = 0 + + async def flush() -> None: + nonlocal buf, size + if not buf: + return + c = await handle_raw("\n".join(buf), user_id, batcher, client) + out.hands_added += c.hands_added + out.hands_skipped += c.hands_skipped + out.tournaments_added += c.tournaments_added + out.tournaments_skipped += c.tournaments_skipped + buf, size = [], 0 + + for part in parts: + buf.append(part) + size += len(part) + if size >= 32 * 1024 * 1024: + await flush() + await flush() + return out + + +async def handle_archive( + data: bytes, user_id: UUID, batcher: ClickHouseBatcher, client: AsyncClient +) -> ImportCounts: + """Extract an archive's .txt members, split into hand-history vs summary, and ingest both.""" + hh_parts: list[str] = [] + summary_parts: list[str] = [] + for _name, text in extract_text_members(data): + if "Poker Hand #" in text: + hh_parts.append(text) + elif "Buy-in:" in text or text.startswith("Tournament #"): + summary_parts.append(text) + counts = await _ingest_chunked(hh_parts, user_id, batcher, client) + sums = await _ingest_chunked(summary_parts, user_id, batcher, client) + counts.kind = "archive" + counts.tournaments_added += sums.tournaments_added + counts.tournaments_skipped += sums.tournaments_skipped + counts.hands_added += sums.hands_added + counts.hands_skipped += sums.hands_skipped + return counts + + async def _handle( job: dict[str, Any], storage: RawStorage, batcher: ClickHouseBatcher, client: AsyncClient ) -> None: @@ -97,8 +147,11 @@ async def _handle( reset_log_context() bind_log_context(user_id=str(user_id), object_key=key, import_id=str(import_id or "")) try: - raw = _decode_object(key, storage.get(key)) - counts = await handle_raw(raw, user_id, batcher, client) + data = storage.get(key) + if job.get("kind") == "archive": + counts = await handle_archive(data, user_id, batcher, client) + else: + counts = await handle_raw(_decode_object(key, data), user_id, batcher, client) await batcher.flush_all() # durable in ClickHouse before dropping raw / reporting except ParserUnavailable: logger.error("hh_parser not built; leaving %s for retry", key) diff --git a/apps/api/tests/test_archives.py b/apps/api/tests/test_archives.py new file mode 100644 index 0000000..9f65d7c --- /dev/null +++ b/apps/api/tests/test_archives.py @@ -0,0 +1,39 @@ +"""Archive extraction: pull .txt members out of an archive blob.""" + +from __future__ import annotations + +import io +import zipfile + +import pytest + +pytest.importorskip("libarchive") # needs the system libarchive shared lib + +from spin2note_api.ingest.archives import extract_text_members, is_archive # noqa: E402 + + +def _zip(files: dict[str, bytes]) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, content in files.items(): + zf.writestr(name, content) + return buf.getvalue() + + +def test_extracts_only_txt_members() -> None: + blob = _zip( + { + "hands/GG1.txt": b"Poker Hand #SG1: ...", + "summary/T1.txt": b"Tournament #1, Spin&Gold", + "readme.md": b"not a hand history", + "nested/dir/GG2.txt": b"Poker Hand #SG2: ...", + } + ) + members = dict(extract_text_members(blob)) + assert set(members) == {"hands/GG1.txt", "summary/T1.txt", "nested/dir/GG2.txt"} + assert members["hands/GG1.txt"].startswith("Poker Hand #") + + +def test_is_archive_by_extension() -> None: + assert is_archive("base.zip") and is_archive("HISTORY.RAR") and is_archive("x.tar.gz") + assert not is_archive("GG20260101.txt") diff --git a/apps/api/uv.lock b/apps/api/uv.lock index 5495dc8..585bb81 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -766,6 +766,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/f6/62cd379fe8527c6d685013ebed11c85fd4fced125bde9b3c80ebd5759850/leb128-1.0.9-py2.py3-none-any.whl", hash = "sha256:fef16ef20aca33dfdd2f4841d8004ec4acb7ed8545b63a7bc1183292c9a3e594", size = 3700, upload-time = "2026-01-09T08:29:37.446Z" }, ] +[[package]] +name = "libarchive-c" +version = "5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/23/e72434d5457c24113e0c22605cbf7dd806a2561294a335047f5aa8ddc1ca/libarchive_c-5.3.tar.gz", hash = "sha256:5ddb42f1a245c927e7686545da77159859d5d4c6d00163c59daff4df314dae82", size = 54349, upload-time = "2025-05-22T08:08:04.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/3f/ff00c588ebd7eae46a9d6223389f5ae28a3af4b6d975c0f2a6d86b1342b9/libarchive_c-5.3-py3-none-any.whl", hash = "sha256:651550a6ec39266b78f81414140a1e04776c935e72dfc70f1d7c8e0a3672ffba", size = 17035, upload-time = "2025-05-22T08:08:03.045Z" }, +] + [[package]] name = "librt" version = "0.11.0" @@ -1622,6 +1631,7 @@ dependencies = [ { name = "clickhouse-connect" }, { name = "fastapi" }, { name = "httpx" }, + { name = "libarchive-c" }, { name = "minio" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, @@ -1652,6 +1662,7 @@ requires-dist = [ { name = "clickhouse-connect", specifier = ">=0.8" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.28" }, + { name = "libarchive-c", specifier = ">=5.1" }, { name = "minio", specifier = ">=7.2" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.9" }, diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 04bb45d..cbb51f2 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -4,6 +4,7 @@ import * as React from "react"; import { ResultByStack } from "@/components/charts/result-by-stack"; import { AppShell } from "@/components/app-shell"; +import { Uploader } from "@/components/uploader"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -41,7 +42,7 @@ function Dashboard() { const [hands, setHands] = React.useState([]); const [error, setError] = React.useState(null); - React.useEffect(() => { + const load = React.useCallback(() => { if (!token) return; setError(null); Promise.all([getOverview(token, format), getRecentHands(token, 25)]) @@ -52,6 +53,8 @@ function Dashboard() { .catch((e) => setError(String(e))); }, [token, format]); + React.useEffect(load, [load]); + return (
@@ -72,6 +75,8 @@ function Dashboard() { {error &&

{error}

} + +
diff --git a/apps/web/app/upload/page.tsx b/apps/web/app/upload/page.tsx deleted file mode 100644 index c76573c..0000000 --- a/apps/web/app/upload/page.tsx +++ /dev/null @@ -1,144 +0,0 @@ -"use client"; - -import * as React from "react"; -import { FolderUp, UploadCloud } from "lucide-react"; - -import { AppShell } from "@/components/app-shell"; -import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { getImportSummary, type ImportSummary } from "@/lib/api"; -import { bulkUpload, type BulkProgress } from "@/lib/bulk-upload"; -import { useAuth } from "@/lib/auth"; - -export default function UploadPage() { - return ( - - - - ); -} - -function Uploader() { - const { session } = useAuth(); - const token = session?.access_token ?? ""; - const folderRef = React.useRef(null); - const filesRef = React.useRef(null); - const [files, setFiles] = React.useState([]); - const [busy, setBusy] = React.useState(false); - const [progress, setProgress] = React.useState(null); - const [report, setReport] = React.useState(null); - const [error, setError] = React.useState(null); - - function pick(list: FileList | null) { - if (!list) return; - setFiles(Array.from(list).filter((f) => f.name.endsWith(".txt"))); - setReport(null); - setProgress(null); - } - - async function start() { - if (files.length === 0) return; - setBusy(true); - setError(null); - setReport(null); - try { - const { sessionId } = await bulkUpload(files, token, setProgress); - // Parsing is async in the worker — poll the import report until all chunks finish. - for (let i = 0; i < 600; i++) { - const s = await getImportSummary(token, sessionId); - setReport(s); - if (s.complete) break; - await new Promise((r) => setTimeout(r, 1500)); - } - } catch (e) { - setError(String(e)); - } finally { - setBusy(false); - } - } - - const pct = progress && progress.filesTotal > 0 - ? Math.round((progress.filesRead / progress.filesTotal) * 100) - : 0; - - return ( -
-

Upload your hand history

- - - Select a whole folder or many .txt files — even 100k+ - - -
- - - pick(e.target.files)} - /> - pick(e.target.files)} - /> -
- - {files.length > 0 && ( -

{files.length.toLocaleString("en-US")} .txt files selected

- )} - - - - {progress && ( -
-
-
-
-
- Read: {progress.filesRead.toLocaleString("en-US")} / {progress.filesTotal.toLocaleString("en-US")} - Chunks: {progress.chunksUploaded} / {progress.chunksStarted} - Sent: {(progress.bytesUploaded / 1e6).toFixed(1)} MB (gzip) - {progress.filesSkipped > 0 && Skipped: {progress.filesSkipped}} -
-
- )} - - {error &&

{error}

} - - {report && ( -
-
- - {report.complete ? "Import complete" : "Importing…"} - - - {report.done}/{report.chunks} chunks{report.failed > 0 ? ` · ${report.failed} failed` : ""} - -
-
- +{report.hands_added.toLocaleString("en-US")} hands - {report.hands_skipped.toLocaleString("en-US")} duplicate hands skipped - +{report.tournaments_added.toLocaleString("en-US")} tournaments - {report.tournaments_skipped.toLocaleString("en-US")} duplicate tournaments skipped -
-
- )} - - -
- ); -} diff --git a/apps/web/src/components/app-shell.tsx b/apps/web/src/components/app-shell.tsx index f9eb8aa..bc0ddda 100644 --- a/apps/web/src/components/app-shell.tsx +++ b/apps/web/src/components/app-shell.tsx @@ -3,16 +3,13 @@ import Link from "next/link"; import { usePathname, useRouter } from "next/navigation"; import * as React from "react"; -import { LogOut, LayoutDashboard, Upload } from "lucide-react"; +import { LogOut, LayoutDashboard } from "lucide-react"; import { Button } from "@/components/ui/button"; import { useAuth } from "@/lib/auth"; import { cn } from "@/lib/utils"; -const NAV = [ - { href: "/", label: "Dashboard", icon: LayoutDashboard }, - { href: "/upload", label: "Upload", icon: Upload }, -]; +const NAV = [{ href: "/", label: "Dashboard", icon: LayoutDashboard }]; /** Auth-gated layout: redirects to /login when there is no session. */ export function AppShell({ children }: { children: React.ReactNode }) { diff --git a/apps/web/src/components/uploader.tsx b/apps/web/src/components/uploader.tsx new file mode 100644 index 0000000..766e208 --- /dev/null +++ b/apps/web/src/components/uploader.tsx @@ -0,0 +1,132 @@ +"use client"; + +import * as React from "react"; +import { FolderUp, UploadCloud } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { getImportSummary, type ImportSummary } from "@/lib/api"; +import { bulkUpload, isArchive, type BulkProgress } from "@/lib/bulk-upload"; +import { useAuth } from "@/lib/auth"; + +const ACCEPT = ".txt,.zip,.rar,.7z,.tar,.gz,.tgz,.bz2,.xz"; + +/** Folder / files / archive upload with live import report. Calls onComplete when done. */ +export function Uploader({ onComplete }: { onComplete?: () => void }) { + const { session } = useAuth(); + const token = session?.access_token ?? ""; + const folderRef = React.useRef(null); + const filesRef = React.useRef(null); + const [files, setFiles] = React.useState([]); + const [busy, setBusy] = React.useState(false); + const [progress, setProgress] = React.useState(null); + const [report, setReport] = React.useState(null); + const [error, setError] = React.useState(null); + + function pick(list: FileList | null) { + if (!list) return; + setFiles(Array.from(list).filter((f) => f.name.endsWith(".txt") || isArchive(f.name))); + setReport(null); + setProgress(null); + } + + async function start() { + if (files.length === 0) return; + setBusy(true); + setError(null); + setReport(null); + try { + const { sessionId } = await bulkUpload(files, token, setProgress); + for (let i = 0; i < 1200; i++) { + const s = await getImportSummary(token, sessionId); + setReport(s); + if (s.complete) break; + await new Promise((r) => setTimeout(r, 1500)); + } + onComplete?.(); + } catch (e) { + setError(String(e)); + } finally { + setBusy(false); + } + } + + const pct = + progress && progress.filesTotal > 0 + ? Math.round((progress.filesRead / progress.filesTotal) * 100) + : 0; + + return ( + + + Upload hand history — folder, files, or an archive (.zip/.rar/.7z/…) + + +
+ + + pick(e.target.files)} + /> + pick(e.target.files)} + /> + {files.length > 0 && ( + + )} +
+ + {progress && ( +
+
+
+
+
+ Read: {progress.filesRead.toLocaleString("en-US")} / {progress.filesTotal.toLocaleString("en-US")} + Uploads: {progress.chunksUploaded} / {progress.chunksStarted} + Sent: {(progress.bytesUploaded / 1e6).toFixed(1)} MB +
+
+ )} + + {error &&

{error}

} + + {report && ( +
+
+ + {report.complete ? "Import complete" : "Importing…"} + + + {report.done}/{report.chunks}{report.failed > 0 ? ` · ${report.failed} failed` : ""} + +
+
+ +{report.hands_added.toLocaleString("en-US")} hands + {report.hands_skipped.toLocaleString("en-US")} duplicate hands skipped + +{report.tournaments_added.toLocaleString("en-US")} tournaments + {report.tournaments_skipped.toLocaleString("en-US")} duplicate tournaments skipped +
+
+ )} + + + ); +} diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 3cba67f..d928b69 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -56,6 +56,26 @@ export async function uploadBulk( return gzipped.byteLength; } +/** Upload one archive (zip/rar/7z/…) for server-side extraction; returns its byte count. */ +export async function uploadArchive( + token: string, + file: File, + sessionId: string, +): Promise { + const res = await fetch(`${API_URL}/ingest/archive`, { + method: "POST", + body: file, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/octet-stream", + "X-Upload-Session": sessionId, + "X-Filename": encodeURIComponent(file.name), + }, + }); + if (!res.ok) throw new Error(`API ${res.status}: ${await res.text()}`); + return file.size; +} + /** Aggregated added/skipped report for one upload session. */ export function getImportSummary(token: string, sessionId: string): Promise { return apiFetch(`/imports/${sessionId}`, token); diff --git a/apps/web/src/lib/api/schema.ts b/apps/web/src/lib/api/schema.ts index 84449ab..2291f54 100644 --- a/apps/web/src/lib/api/schema.ts +++ b/apps/web/src/lib/api/schema.ts @@ -71,6 +71,29 @@ export interface paths { patch?: never; trace?: never; }; + "/ingest/archive": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Upload Archive + * @description Accept a single archive (zip/rar/7z/tar/gz/…) and extract its .txt members server-side. + * + * The raw archive is staged as-is (already compressed); the worker uses libarchive to pull out + * the hand-history/summary files and feeds them through the normal parse + dedup pipeline. + */ + post: operations["upload_archive_ingest_archive_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/stats/overview": { parameters: { query?: never; @@ -324,6 +347,38 @@ export interface operations { }; }; }; + upload_archive_ingest_archive_post: { + parameters: { + query?: never; + header?: { + "x-upload-session"?: string | null; + "x-filename"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UploadResult"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; overview_stats_overview_get: { parameters: { query?: { diff --git a/apps/web/src/lib/bulk-upload.ts b/apps/web/src/lib/bulk-upload.ts index 452c686..1ca987d 100644 --- a/apps/web/src/lib/bulk-upload.ts +++ b/apps/web/src/lib/bulk-upload.ts @@ -1,6 +1,13 @@ import { gzip } from "fflate"; -import { uploadBulk } from "./api"; +import { uploadArchive, uploadBulk } from "./api"; + +// Archives are extracted server-side (one engine for zip/rar/7z/tar/…); plain .txt files are +// bundled client-side. +const ARCHIVE_RE = /\.(zip|rar|7z|tar|tar\.gz|tgz|gz|bz2|xz)$/i; +export function isArchive(name: string): boolean { + return ARCHIVE_RE.test(name); +} // The browser concatenates many small files into ~16 MB chunks and gzips them, so a base of // hundreds of thousands of files becomes a few dozen requests. The backend parsers split by @@ -67,14 +74,14 @@ export async function bulkUpload( bytesUploaded: 0, }; - const ship = async (text: string) => { + // Run an upload task under backpressure (at most CONCURRENCY in flight). + const run = (taskFn: () => Promise) => { p.chunksStarted += 1; onProgress({ ...p }); - await sem.acquire(); // backpressure: at most CONCURRENCY chunks in flight const task = (async () => { + await sem.acquire(); try { - const gz = await gzipAsync(encoder.encode(text)); - const n = await uploadBulk(token, gz, sessionId); + const n = await taskFn(); p.chunksUploaded += 1; p.bytesUploaded += n; onProgress({ ...p }); @@ -85,6 +92,9 @@ export async function bulkUpload( inFlight.push(task); }; + const ship = async (text: string) => + run(async () => uploadBulk(token, await gzipAsync(encoder.encode(text)), sessionId)); + let hh: string[] = []; let hhSize = 0; let sm: string[] = []; @@ -93,7 +103,15 @@ export async function bulkUpload( // Read files with bounded concurrency (disk I/O parallelism) while bundling sequentially. const READ_CONCURRENCY = 16; for (let i = 0; i < files.length; i += READ_CONCURRENCY) { - const texts = await Promise.all(files.slice(i, i + READ_CONCURRENCY).map((f) => f.text())); + const batch = files.slice(i, i + READ_CONCURRENCY); + // Archives go straight to the server (extracted there); the rest are read as text. + const archives = batch.filter((f) => isArchive(f.name)); + for (const a of archives) { + p.filesRead += 1; + run(() => uploadArchive(token, a, sessionId)); + } + const textFiles = batch.filter((f) => !isArchive(f.name)); + const texts = await Promise.all(textFiles.map((f) => f.text())); for (const text of texts) { p.filesRead += 1; const kind = classify(text);