Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Install system libs (libarchive for archive extraction)
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libarchive13

- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable

Expand Down
3 changes: 3 additions & 0 deletions apps/api/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ RUN cd apps/api && uv sync

FROM python:3.12-slim-bookworm AS runtime
WORKDIR /app/apps/api
# libarchive13: archive extraction (zip/rar/7z/tar/...) via libarchive-c.
RUN apt-get update && apt-get install -y --no-install-recommends libarchive13 \
&& rm -rf /var/lib/apt/lists/*
ENV PATH="/app/apps/api/.venv/bin:${PATH}"
COPY --from=builder /app/apps/api/.venv /app/apps/api/.venv
COPY --from=builder /app/apps/api/src /app/apps/api/src
Expand Down
1 change: 1 addition & 0 deletions apps/api/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ dependencies = [
"asynch>=0.2.5",
"redis>=5.2",
"minio>=7.2",
"libarchive-c>=5.1",
"sqlalchemy>=2.0",
"sqlmodel>=0.0.22",
"alembic>=1.14",
Expand Down
33 changes: 33 additions & 0 deletions apps/api/src/spin2note_api/ingest/archives.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""Archive extraction for bulk upload.

One engine (``libarchive``) reads zip, rar, 7z, tar and tar.gz/bz2/xz, so users can upload a
single archive of their whole base instead of selecting files. We pull out the ``.txt`` members
(hand histories + tournament summaries) and hand them to the normal parse pipeline.
"""

from __future__ import annotations

from collections.abc import Iterator

# Extensions routed to server-side extraction (vs the client-side .txt bundling path).
ARCHIVE_SUFFIXES = (".zip", ".rar", ".7z", ".tar", ".tar.gz", ".tgz", ".gz", ".bz2", ".xz")


def is_archive(filename: str) -> bool:
name = filename.lower()
return any(name.endswith(s) for s in ARCHIVE_SUFFIXES)


def extract_text_members(data: bytes) -> Iterator[tuple[str, str]]:
"""Yield (name, text) for every ``.txt`` file inside an archive blob."""
import libarchive # imported lazily; needs the system libarchive shared lib

with libarchive.memory_reader(data) as archive:
for entry in archive:
if not entry.isfile:
continue
name = str(entry.pathname)
if not name.lower().endswith(".txt"):
continue
raw = b"".join(entry.get_blocks())
yield name, raw.decode("utf-8", errors="replace")
36 changes: 32 additions & 4 deletions apps/api/src/spin2note_api/ingest/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def _session_id(header: str | None) -> UUID:


async def _register_and_enqueue(
settings: Settings, key: str, user_id: str, session_id: UUID
settings: Settings, key: str, user_id: str, session_id: UUID, kind: str = "file"
) -> UUID:
import_id = new_id()
await create_import(
Expand All @@ -49,11 +49,12 @@ async def _register_and_enqueue(
# Imported lazily so the module has no hard Redis dependency at import time.
from ..cache import RedisQueue

job = {"object_key": key, "user_id": user_id, "import_id": str(import_id)}
if kind != "file":
job["kind"] = kind
queue = RedisQueue(settings.redis_url)
try:
await queue.enqueue(
{"object_key": key, "user_id": user_id, "import_id": str(import_id)}
)
await queue.enqueue(job)
finally:
await queue.close()
return import_id
Expand Down Expand Up @@ -114,3 +115,30 @@ async def upload_bulk(
object_key=key, bytes=len(body), queued=True,
import_id=str(import_id), session_id=str(session_id),
)


@router.post("/archive", response_model=UploadResult)
async def upload_archive(
request: Request,
settings: Settings = Depends(get_settings),
user_id: str = Depends(current_user_id),
x_upload_session: str | None = Header(default=None),
x_filename: str | None = Header(default=None),
) -> UploadResult:
"""Accept a single archive (zip/rar/7z/tar/gz/…) and extract its .txt members server-side.

The raw archive is staged as-is (already compressed); the worker uses libarchive to pull out
the hand-history/summary files and feeds them through the normal parse + dedup pipeline.
"""
body = await request.body()
stamp = datetime.now(UTC).strftime("%Y/%m/%d")
name = (x_filename or "upload.zip").rsplit("/", 1)[-1]
key = f"{user_id}/archive/{stamp}/{new_id()}-{name}"

session_id = _session_id(x_upload_session)
_stage(settings, key, body)
import_id = await _register_and_enqueue(settings, key, user_id, session_id, kind="archive")
return UploadResult(
object_key=key, bytes=len(body), queued=True,
import_id=str(import_id), session_id=str(session_id),
)
57 changes: 55 additions & 2 deletions apps/api/src/spin2note_api/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from .clickhouse.queries import existing_hand_ids, existing_tournament_ids, user_has_rows
from .config import get_settings
from .db.imports import ImportCounts, finish_import
from .ingest.archives import extract_text_members
from .ingest.storage import RawStorage
from .logging_config import bind_log_context, configure_logging, reset_log_context
from .parser import (
Expand Down Expand Up @@ -87,6 +88,55 @@ async def handle_raw(
)


async def _ingest_chunked(
parts: list[str], user_id: UUID, batcher: ClickHouseBatcher, client: AsyncClient
) -> ImportCounts:
"""Parse same-blob parts in ~32 MB chunks (bounds memory for huge archives)."""
out = ImportCounts()
buf: list[str] = []
size = 0

async def flush() -> None:
nonlocal buf, size
if not buf:
return
c = await handle_raw("\n".join(buf), user_id, batcher, client)
out.hands_added += c.hands_added
out.hands_skipped += c.hands_skipped
out.tournaments_added += c.tournaments_added
out.tournaments_skipped += c.tournaments_skipped
buf, size = [], 0

for part in parts:
buf.append(part)
size += len(part)
if size >= 32 * 1024 * 1024:
await flush()
await flush()
return out


async def handle_archive(
data: bytes, user_id: UUID, batcher: ClickHouseBatcher, client: AsyncClient
) -> ImportCounts:
"""Extract an archive's .txt members, split into hand-history vs summary, and ingest both."""
hh_parts: list[str] = []
summary_parts: list[str] = []
for _name, text in extract_text_members(data):
if "Poker Hand #" in text:
hh_parts.append(text)
elif "Buy-in:" in text or text.startswith("Tournament #"):
summary_parts.append(text)
counts = await _ingest_chunked(hh_parts, user_id, batcher, client)
sums = await _ingest_chunked(summary_parts, user_id, batcher, client)
counts.kind = "archive"
counts.tournaments_added += sums.tournaments_added
counts.tournaments_skipped += sums.tournaments_skipped
counts.hands_added += sums.hands_added
counts.hands_skipped += sums.hands_skipped
return counts


async def _handle(
job: dict[str, Any], storage: RawStorage, batcher: ClickHouseBatcher, client: AsyncClient
) -> None:
Expand All @@ -97,8 +147,11 @@ async def _handle(
reset_log_context()
bind_log_context(user_id=str(user_id), object_key=key, import_id=str(import_id or ""))
try:
raw = _decode_object(key, storage.get(key))
counts = await handle_raw(raw, user_id, batcher, client)
data = storage.get(key)
if job.get("kind") == "archive":
counts = await handle_archive(data, user_id, batcher, client)
else:
counts = await handle_raw(_decode_object(key, data), user_id, batcher, client)
await batcher.flush_all() # durable in ClickHouse before dropping raw / reporting
except ParserUnavailable:
logger.error("hh_parser not built; leaving %s for retry", key)
Expand Down
39 changes: 39 additions & 0 deletions apps/api/tests/test_archives.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Archive extraction: pull .txt members out of an archive blob."""

from __future__ import annotations

import io
import zipfile

import pytest

pytest.importorskip("libarchive") # needs the system libarchive shared lib

from spin2note_api.ingest.archives import extract_text_members, is_archive # noqa: E402


def _zip(files: dict[str, bytes]) -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
for name, content in files.items():
zf.writestr(name, content)
return buf.getvalue()


def test_extracts_only_txt_members() -> None:
blob = _zip(
{
"hands/GG1.txt": b"Poker Hand #SG1: ...",
"summary/T1.txt": b"Tournament #1, Spin&Gold",
"readme.md": b"not a hand history",
"nested/dir/GG2.txt": b"Poker Hand #SG2: ...",
}
)
members = dict(extract_text_members(blob))
assert set(members) == {"hands/GG1.txt", "summary/T1.txt", "nested/dir/GG2.txt"}
assert members["hands/GG1.txt"].startswith("Poker Hand #")


def test_is_archive_by_extension() -> None:
assert is_archive("base.zip") and is_archive("HISTORY.RAR") and is_archive("x.tar.gz")
assert not is_archive("GG20260101.txt")
11 changes: 11 additions & 0 deletions apps/api/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion apps/web/app/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import * as React from "react";

import { ResultByStack } from "@/components/charts/result-by-stack";
import { AppShell } from "@/components/app-shell";
import { Uploader } from "@/components/uploader";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
Expand Down Expand Up @@ -41,7 +42,7 @@ function Dashboard() {
const [hands, setHands] = React.useState<RecentHand[]>([]);
const [error, setError] = React.useState<string | null>(null);

React.useEffect(() => {
const load = React.useCallback(() => {
if (!token) return;
setError(null);
Promise.all([getOverview(token, format), getRecentHands(token, 25)])
Expand All @@ -52,6 +53,8 @@ function Dashboard() {
.catch((e) => setError(String(e)));
}, [token, format]);

React.useEffect(load, [load]);

return (
<div className="flex flex-col gap-6">
<div className="flex items-center justify-between">
Expand All @@ -72,6 +75,8 @@ function Dashboard() {

{error && <p className="text-sm text-destructive">{error}</p>}

<Uploader onComplete={load} />

<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
<Stat title="Hands" value={(overview?.total_hands ?? 0).toLocaleString("en-US")} />
<Stat title="Tournaments" value={(overview?.total_tournaments ?? 0).toLocaleString("en-US")} />
Expand Down
Loading
Loading