Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/api/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ dependencies = [
"pydantic-settings>=2.6",
"clickhouse-connect>=0.8",
"aiohttp>=3.10",
"asynch>=0.2.5",
"redis>=5.2",
"minio>=7.2",
"sqlalchemy>=2.0",
Expand Down
18 changes: 18 additions & 0 deletions apps/api/src/spin2note_api/clickhouse/batcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,24 @@ async def submit_many(self, table: str, rows: Sequence[Any]) -> None:
for row in rows:
await self.submit(table, row)

async def submit_block(self, table: str, rows: Sequence[Any]) -> None:
"""Append many rows at once (one lock), flushing full ``max_rows`` blocks.

Far cheaper than per-row ``submit`` for bulk ingestion — the hot path adds a whole
chunk's rows in a single critical section.
"""
if not rows:
return
async with self._lock:
buf = self._buffers.setdefault(table, [])
buf.extend(rows)
ready: list[list[Any]] = []
while len(buf) >= self._max_rows:
ready.append(buf[: self._max_rows])
del buf[: self._max_rows]
for block in ready:
await self._insert_fn(table, block)

async def flush_all(self) -> None:
async with self._lock:
pending = {t: rows for t, rows in self._buffers.items() if rows}
Expand Down
41 changes: 41 additions & 0 deletions apps/api/src/spin2note_api/clickhouse/native.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""Native-protocol ClickHouse inserts (asynch) for the worker's hot path.

The HTTP client (clickhouse-connect) is kept for reads; bulk inserts go over the native TCP
protocol, which is markedly faster for large column blocks. Rows are tuples in
``TABLE_COLUMNS`` order; Enum8 columns receive their label strings (the driver maps them).
"""

from __future__ import annotations

from collections.abc import Awaitable, Callable, Sequence
from typing import Any

from asynch import Connection

from ..config import Settings


async def make_native_conn(settings: Settings) -> Any:
host = settings.clickhouse_url.split("://", 1)[-1].split(":")[0]
conn = Connection(
host=host,
port=settings.clickhouse_native_port,
user=settings.clickhouse_user,
password=settings.clickhouse_password,
database=settings.clickhouse_database,
)
await conn.connect()
return conn


def make_native_insert_fn(
conn: Any, columns: dict[str, list[str]]
) -> Callable[[str, Sequence[Any]], Awaitable[None]]:
"""Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn."""

async def insert_fn(table: str, rows: Sequence[Any]) -> None:
cols = ", ".join(columns[table])
async with conn.cursor() as cur:
await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows))

return insert_fn
11 changes: 11 additions & 0 deletions apps/api/src/spin2note_api/clickhouse/queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,17 @@ def _batched(items: list[Any], size: int = _ID_BATCH) -> list[list[Any]]:
return [items[i : i + size] for i in range(0, len(items), size)]


async def user_has_rows(client: AsyncClient, table: str, user_id: UUID) -> bool:
"""Cheap check used to skip membership queries entirely on a user's first import."""
if table not in {"hands", "tournaments"}:
raise ValueError(table)
res = await client.query(
f"SELECT 1 FROM {table} WHERE user_id = {{u:UUID}} LIMIT 1",
parameters={"u": str(user_id)},
)
return bool(res.result_rows)


async def existing_hand_ids(client: AsyncClient, user_id: UUID, ids: list[UUID]) -> set[UUID]:
"""Return which of ``ids`` already exist for this user (for input deduplication)."""
found: set[UUID] = set()
Expand Down
8 changes: 5 additions & 3 deletions apps/api/src/spin2note_api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ class Settings(BaseSettings):
environment: str = "development"

# ClickHouse (analytics store)
clickhouse_url: str = "http://localhost:8123"
clickhouse_url: str = "http://localhost:8123" # HTTP, used for reads (clickhouse-connect)
clickhouse_native_port: int = 9000 # native TCP, used for fast inserts (asynch)
clickhouse_user: str = "default"
clickhouse_password: str = ""
clickhouse_database: str = "spin2note"
Expand All @@ -46,8 +47,9 @@ class Settings(BaseSettings):
# Fallback HS256 secret for local dev when JWKS is unavailable.
supabase_jwt_secret: str = ""

# ClickHouse batching (hard rule: never single-row inserts)
batch_max_rows: int = 1000
# ClickHouse batching (hard rule: never single-row inserts; bigger blocks = far fewer
# native round-trips, which dominates bulk-insert throughput).
batch_max_rows: int = 50000
batch_max_interval_seconds: float = 1.0

# Fallback owner id for uploads without an authenticated user (local dev / bulk import).
Expand Down
11 changes: 10 additions & 1 deletion apps/api/src/spin2note_api/parser/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
"""Hand-history parsing: Rust ``hh_parser`` wrapper + domain mapping."""

from .mapping import build_hand, build_hands, build_tournament, deterministic_hand_id
from .mapping import (
build_chunk_rows,
build_hand,
build_hands,
build_tournament,
build_tournament_rows,
deterministic_hand_id,
)
from .wrapper import ParserUnavailable, detect_format, parse, parse_summaries, parse_summary

__all__ = [
Expand All @@ -12,5 +19,7 @@
"build_hand",
"build_hands",
"build_tournament",
"build_chunk_rows",
"build_tournament_rows",
"deterministic_hand_id",
]
49 changes: 49 additions & 0 deletions apps/api/src/spin2note_api/parser/mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,55 @@ def build_hands(parsed_list: list[dict[str, Any]], user_id: UUID) -> list[Hand]:
return [build_hand(p, user_id) for p in parsed_list]


# --- Fast path: build column-ordered tuples directly (no Pydantic) for native inserts. ---
# Tuple order MUST match clickhouse.client.TABLE_COLUMNS for each table.

_EPOCH = datetime(1970, 1, 1)


def build_chunk_rows(
hands: list[dict[str, Any]], user_id: UUID, parsed_at: datetime
) -> dict[str, list[tuple[Any, ...]]]:
"""Project parsed hand dicts into column-ordered row tuples for hands/hand_players/actions."""
hand_rows: list[tuple[Any, ...]] = []
player_rows: list[tuple[Any, ...]] = []
action_rows: list[tuple[Any, ...]] = []
for h in hands:
hid = UUID(h["hand_id"]) # deterministic id computed in Rust
fmt = h["format"] # Enum8 label ("3max"/"6max")
eff = int(h["effective_stack_bb"])
played = _dt(h["played_at"]) or _EPOCH
hand_rows.append((
hid, user_id, fmt, eff, h["source_hand_id"], h["tournament_id"], h["level"],
h["button_seat"], played, 0.0, h["small_blind"], h["big_blind"], h["board"],
h["pot"], h["rake"], parsed_at,
))
for p in h["players"]:
player_rows.append((
hid, user_id, fmt, eff, p["seat"], int(p["is_hero"]), p["villain_hash"],
p["position"], p["starting_stack"], p["hole_cards"], p["won"], p["result"],
parsed_at,
))
for a in h["actions"]:
action_rows.append((
hid, user_id, fmt, eff, a["street"], a["seat"], a["action_index"],
a["action_type"], a["amount"], 0.0, a["to_amount"], int(a["all_in"]),
))
return {"hands": hand_rows, "hand_players": player_rows, "actions": action_rows}


def build_tournament_rows(
summaries: list[dict[str, Any]], user_id: UUID, parsed_at: datetime
) -> list[tuple[Any, ...]]:
return [
(
s["tournament_id"], user_id, s["name"], s["buy_in"], s["currency"], s["players"],
s["prize_pool"], s["multiplier"], _dt(s["started_at"]), s["hero_place"], parsed_at,
)
for s in summaries
]


def build_tournament(summary: dict[str, Any], user_id: UUID) -> Tournament:
return Tournament(
tournament_id=summary["tournament_id"],
Expand Down
76 changes: 41 additions & 35 deletions apps/api/src/spin2note_api/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,27 @@
import asyncio
import gzip
import logging
from datetime import UTC, datetime
from typing import Any
from uuid import UUID

from clickhouse_connect.driver.asyncclient import AsyncClient

from .cache import RedisQueue
from .clickhouse import ClickHouseBatcher
from .clickhouse.client import make_client, make_insert_fn
from .clickhouse.queries import existing_hand_ids, existing_tournament_ids
from .clickhouse.client import TABLE_COLUMNS, make_client
from .clickhouse.native import make_native_conn, make_native_insert_fn
from .clickhouse.queries import existing_hand_ids, existing_tournament_ids, user_has_rows
from .config import get_settings
from .db.imports import ImportCounts, finish_import
from .ingest.storage import RawStorage
from .parser import ParserUnavailable, build_hands, build_tournament, parse, parse_summaries
from .parser import (
ParserUnavailable,
build_chunk_rows,
build_tournament_rows,
parse,
parse_summaries,
)
from .parser.wrapper import looks_like_summary

logger = logging.getLogger("spin2note.worker")
Expand All @@ -46,40 +54,36 @@ async def handle_raw(
current batch and what already exists for the user — so re-uploading is a no-op that simply
reports how many were skipped.
"""
now = datetime.now(UTC).replace(tzinfo=None)
if looks_like_summary(raw):
tournaments = [build_tournament(s, user_id) for s in parse_summaries(raw)]
total = len(tournaments)
unique: dict[str, Any] = {}
for t in tournaments:
unique.setdefault(t.tournament_id, t)
existing = await existing_tournament_ids(client, user_id, unique.keys())
added = 0
for tid, t in unique.items():
if tid in existing:
continue
await batcher.submit("tournaments", t)
added += 1
unique: dict[str, dict[str, Any]] = {}
for s in parse_summaries(raw):
unique.setdefault(s["tournament_id"], s)
total = len(unique)
existing: set[str] = set()
if await user_has_rows(client, "tournaments", user_id):
existing = await existing_tournament_ids(client, user_id, unique.keys())
new = [s for tid, s in unique.items() if tid not in existing]
await batcher.submit_block("tournaments", build_tournament_rows(new, user_id, now))
return ImportCounts(
kind="summary", tournaments_added=added, tournaments_skipped=total - added
kind="summary", tournaments_added=len(new), tournaments_skipped=total - len(new)
)

hands = build_hands(parse(raw), user_id)
total = len(hands)
unique_h: dict[UUID, Any] = {}
for h in hands:
unique_h.setdefault(h.hand_id, h)
existing_h = await existing_hand_ids(client, user_id, list(unique_h.keys()))
added = 0
for hid, hand in unique_h.items():
if hid in existing_h:
continue
await batcher.submit("hands", hand)
for player in hand.players:
await batcher.submit("hand_players", player)
for action in hand.actions:
await batcher.submit("actions", action)
added += 1
return ImportCounts(kind="hands", hands_added=added, hands_skipped=total - added)
unique_h: dict[UUID, dict[str, Any]] = {}
for h in parse(raw):
unique_h.setdefault(UUID(h["hand_id"]), h)
total = len(unique_h)
existing_h: set[UUID] = set()
if await user_has_rows(client, "hands", user_id):
existing_h = await existing_hand_ids(client, user_id, list(unique_h.keys()))
new_hands = [h for hid, h in unique_h.items() if hid not in existing_h]
blocks = build_chunk_rows(new_hands, user_id, now)
await batcher.submit_block("hands", blocks["hands"])
await batcher.submit_block("hand_players", blocks["hand_players"])
await batcher.submit_block("actions", blocks["actions"])
return ImportCounts(
kind="hands", hands_added=len(new_hands), hands_skipped=total - len(new_hands)
)


async def _handle(
Expand Down Expand Up @@ -112,9 +116,10 @@ async def run() -> None:
settings = get_settings()
queue = RedisQueue(settings.redis_url)
storage = RawStorage(settings)
client = await make_client(settings)
client = await make_client(settings) # HTTP, reads (dedup)
native = await make_native_conn(settings) # native TCP, fast inserts
batcher = ClickHouseBatcher(
make_insert_fn(client),
make_native_insert_fn(native, TABLE_COLUMNS),
max_rows=settings.batch_max_rows,
max_interval_seconds=settings.batch_max_interval_seconds,
)
Expand All @@ -129,6 +134,7 @@ async def run() -> None:
finally:
await batcher.stop()
await queue.close()
await native.close()


def main() -> None:
Expand Down
9 changes: 9 additions & 0 deletions apps/api/tests/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,15 @@ def test_six_max_positions() -> None:
assert positions == {"BTN", "SB", "BB", "UTG", "HJ", "CO"}


def test_rust_hand_id_matches_python() -> None:
# The Rust parser computes hand_id (uuid5); it must equal the Python derivation, or dedup
# would key differently between the two.
for raw in (_read("3max/sample.txt"), _read("6max/sample.txt")):
for h in parse(raw):
expected = deterministic_hand_id(h["tournament_id"], h["source_hand_id"])
assert UUID(h["hand_id"]) == expected


def test_deterministic_hand_id_is_stable() -> None:
a = deterministic_hand_id("900000001", "SG3000000001")
b = deterministic_hand_id("900000001", "SG3000000001")
Expand Down
7 changes: 6 additions & 1 deletion apps/api/tests/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ async def _insert(table: str, rows) -> None: # pragma: no cover - not used
async def submit(self, table: str, row) -> None: # type: ignore[override]
self.rows.setdefault(table, []).append(row)

async def submit_block(self, table: str, rows) -> None: # type: ignore[override]
if not rows:
return
self.rows.setdefault(table, []).extend(rows)


class FakeCH:
"""Stub ClickHouse client returning a fixed set of already-existing ids."""
Expand Down Expand Up @@ -65,7 +70,7 @@ async def test_dedup_skips_existing_hands() -> None:
# First import: nothing exists yet -> both hands added.
first = CollectBatcher()
await handle_raw(raw, USER, first, FakeCH())
seen_ids = {h.hand_id for h in first.rows["hands"]}
seen_ids = {row[0] for row in first.rows["hands"]} # row[0] = hand_id (column order)
assert len(seen_ids) == 2

# Re-import: those hand_ids already exist -> all skipped, nothing submitted.
Expand Down
Loading
Loading