From d39078c717d871fe5fe68d9f3e2f6f0156b8ecfe Mon Sep 17 00:00:00 2001 From: grenkoff Date: Sun, 14 Jun 2026 17:26:04 +0500 Subject: [PATCH] Speed up ingestion: parallel workers, native inserts, big blocks, Rust hand_id, dedup skip/projection, parallel reads --- apps/api/pyproject.toml | 1 + .../src/spin2note_api/clickhouse/batcher.py | 18 +++ .../src/spin2note_api/clickhouse/native.py | 41 ++++++ .../src/spin2note_api/clickhouse/queries.py | 11 ++ apps/api/src/spin2note_api/config.py | 8 +- apps/api/src/spin2note_api/parser/__init__.py | 11 +- apps/api/src/spin2note_api/parser/mapping.py | 49 +++++++ apps/api/src/spin2note_api/worker.py | 76 ++++++----- apps/api/tests/test_parser.py | 9 ++ apps/api/tests/test_worker.py | 7 +- apps/api/uv.lock | 129 ++++++++++++++++++ apps/web/src/lib/bulk-upload.ts | 48 ++++--- crates/hh-parser/Cargo.toml | 1 + crates/hh-parser/src/lib.rs | 1 + crates/hh-parser/src/model.rs | 14 ++ crates/hh-parser/src/parser.rs | 7 +- docker-compose.yml | 8 +- .../clickhouse/0003_dedup_projection.sql | 15 ++ 18 files changed, 390 insertions(+), 64 deletions(-) create mode 100644 apps/api/src/spin2note_api/clickhouse/native.py create mode 100644 migrations/clickhouse/0003_dedup_projection.sql diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 7be54cd..d8610d5 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pydantic-settings>=2.6", "clickhouse-connect>=0.8", "aiohttp>=3.10", + "asynch>=0.2.5", "redis>=5.2", "minio>=7.2", "sqlalchemy>=2.0", diff --git a/apps/api/src/spin2note_api/clickhouse/batcher.py b/apps/api/src/spin2note_api/clickhouse/batcher.py index 8978b58..547100e 100644 --- a/apps/api/src/spin2note_api/clickhouse/batcher.py +++ b/apps/api/src/spin2note_api/clickhouse/batcher.py @@ -71,6 +71,24 @@ async def submit_many(self, table: str, rows: Sequence[Any]) -> None: for row in rows: await self.submit(table, row) + async def submit_block(self, table: str, rows: Sequence[Any]) -> None: + """Append many rows at once (one lock), flushing full ``max_rows`` blocks. + + Far cheaper than per-row ``submit`` for bulk ingestion — the hot path adds a whole + chunk's rows in a single critical section. + """ + if not rows: + return + async with self._lock: + buf = self._buffers.setdefault(table, []) + buf.extend(rows) + ready: list[list[Any]] = [] + while len(buf) >= self._max_rows: + ready.append(buf[: self._max_rows]) + del buf[: self._max_rows] + for block in ready: + await self._insert_fn(table, block) + async def flush_all(self) -> None: async with self._lock: pending = {t: rows for t, rows in self._buffers.items() if rows} diff --git a/apps/api/src/spin2note_api/clickhouse/native.py b/apps/api/src/spin2note_api/clickhouse/native.py new file mode 100644 index 0000000..88204a1 --- /dev/null +++ b/apps/api/src/spin2note_api/clickhouse/native.py @@ -0,0 +1,41 @@ +"""Native-protocol ClickHouse inserts (asynch) for the worker's hot path. + +The HTTP client (clickhouse-connect) is kept for reads; bulk inserts go over the native TCP +protocol, which is markedly faster for large column blocks. Rows are tuples in +``TABLE_COLUMNS`` order; Enum8 columns receive their label strings (the driver maps them). +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Sequence +from typing import Any + +from asynch import Connection + +from ..config import Settings + + +async def make_native_conn(settings: Settings) -> Any: + host = settings.clickhouse_url.split("://", 1)[-1].split(":")[0] + conn = Connection( + host=host, + port=settings.clickhouse_native_port, + user=settings.clickhouse_user, + password=settings.clickhouse_password, + database=settings.clickhouse_database, + ) + await conn.connect() + return conn + + +def make_native_insert_fn( + conn: Any, columns: dict[str, list[str]] +) -> Callable[[str, Sequence[Any]], Awaitable[None]]: + """Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn.""" + + async def insert_fn(table: str, rows: Sequence[Any]) -> None: + cols = ", ".join(columns[table]) + async with conn.cursor() as cur: + await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows)) + + return insert_fn diff --git a/apps/api/src/spin2note_api/clickhouse/queries.py b/apps/api/src/spin2note_api/clickhouse/queries.py index eb93648..0b2a0d3 100644 --- a/apps/api/src/spin2note_api/clickhouse/queries.py +++ b/apps/api/src/spin2note_api/clickhouse/queries.py @@ -23,6 +23,17 @@ def _batched(items: list[Any], size: int = _ID_BATCH) -> list[list[Any]]: return [items[i : i + size] for i in range(0, len(items), size)] +async def user_has_rows(client: AsyncClient, table: str, user_id: UUID) -> bool: + """Cheap check used to skip membership queries entirely on a user's first import.""" + if table not in {"hands", "tournaments"}: + raise ValueError(table) + res = await client.query( + f"SELECT 1 FROM {table} WHERE user_id = {{u:UUID}} LIMIT 1", + parameters={"u": str(user_id)}, + ) + return bool(res.result_rows) + + async def existing_hand_ids(client: AsyncClient, user_id: UUID, ids: list[UUID]) -> set[UUID]: """Return which of ``ids`` already exist for this user (for input deduplication).""" found: set[UUID] = set() diff --git a/apps/api/src/spin2note_api/config.py b/apps/api/src/spin2note_api/config.py index 6566191..9274a1a 100644 --- a/apps/api/src/spin2note_api/config.py +++ b/apps/api/src/spin2note_api/config.py @@ -19,7 +19,8 @@ class Settings(BaseSettings): environment: str = "development" # ClickHouse (analytics store) - clickhouse_url: str = "http://localhost:8123" + clickhouse_url: str = "http://localhost:8123" # HTTP, used for reads (clickhouse-connect) + clickhouse_native_port: int = 9000 # native TCP, used for fast inserts (asynch) clickhouse_user: str = "default" clickhouse_password: str = "" clickhouse_database: str = "spin2note" @@ -46,8 +47,9 @@ class Settings(BaseSettings): # Fallback HS256 secret for local dev when JWKS is unavailable. supabase_jwt_secret: str = "" - # ClickHouse batching (hard rule: never single-row inserts) - batch_max_rows: int = 1000 + # ClickHouse batching (hard rule: never single-row inserts; bigger blocks = far fewer + # native round-trips, which dominates bulk-insert throughput). + batch_max_rows: int = 50000 batch_max_interval_seconds: float = 1.0 # Fallback owner id for uploads without an authenticated user (local dev / bulk import). diff --git a/apps/api/src/spin2note_api/parser/__init__.py b/apps/api/src/spin2note_api/parser/__init__.py index 8675f28..401eb2a 100644 --- a/apps/api/src/spin2note_api/parser/__init__.py +++ b/apps/api/src/spin2note_api/parser/__init__.py @@ -1,6 +1,13 @@ """Hand-history parsing: Rust ``hh_parser`` wrapper + domain mapping.""" -from .mapping import build_hand, build_hands, build_tournament, deterministic_hand_id +from .mapping import ( + build_chunk_rows, + build_hand, + build_hands, + build_tournament, + build_tournament_rows, + deterministic_hand_id, +) from .wrapper import ParserUnavailable, detect_format, parse, parse_summaries, parse_summary __all__ = [ @@ -12,5 +19,7 @@ "build_hand", "build_hands", "build_tournament", + "build_chunk_rows", + "build_tournament_rows", "deterministic_hand_id", ] diff --git a/apps/api/src/spin2note_api/parser/mapping.py b/apps/api/src/spin2note_api/parser/mapping.py index 837698b..a99559f 100644 --- a/apps/api/src/spin2note_api/parser/mapping.py +++ b/apps/api/src/spin2note_api/parser/mapping.py @@ -96,6 +96,55 @@ def build_hands(parsed_list: list[dict[str, Any]], user_id: UUID) -> list[Hand]: return [build_hand(p, user_id) for p in parsed_list] +# --- Fast path: build column-ordered tuples directly (no Pydantic) for native inserts. --- +# Tuple order MUST match clickhouse.client.TABLE_COLUMNS for each table. + +_EPOCH = datetime(1970, 1, 1) + + +def build_chunk_rows( + hands: list[dict[str, Any]], user_id: UUID, parsed_at: datetime +) -> dict[str, list[tuple[Any, ...]]]: + """Project parsed hand dicts into column-ordered row tuples for hands/hand_players/actions.""" + hand_rows: list[tuple[Any, ...]] = [] + player_rows: list[tuple[Any, ...]] = [] + action_rows: list[tuple[Any, ...]] = [] + for h in hands: + hid = UUID(h["hand_id"]) # deterministic id computed in Rust + fmt = h["format"] # Enum8 label ("3max"/"6max") + eff = int(h["effective_stack_bb"]) + played = _dt(h["played_at"]) or _EPOCH + hand_rows.append(( + hid, user_id, fmt, eff, h["source_hand_id"], h["tournament_id"], h["level"], + h["button_seat"], played, 0.0, h["small_blind"], h["big_blind"], h["board"], + h["pot"], h["rake"], parsed_at, + )) + for p in h["players"]: + player_rows.append(( + hid, user_id, fmt, eff, p["seat"], int(p["is_hero"]), p["villain_hash"], + p["position"], p["starting_stack"], p["hole_cards"], p["won"], p["result"], + parsed_at, + )) + for a in h["actions"]: + action_rows.append(( + hid, user_id, fmt, eff, a["street"], a["seat"], a["action_index"], + a["action_type"], a["amount"], 0.0, a["to_amount"], int(a["all_in"]), + )) + return {"hands": hand_rows, "hand_players": player_rows, "actions": action_rows} + + +def build_tournament_rows( + summaries: list[dict[str, Any]], user_id: UUID, parsed_at: datetime +) -> list[tuple[Any, ...]]: + return [ + ( + s["tournament_id"], user_id, s["name"], s["buy_in"], s["currency"], s["players"], + s["prize_pool"], s["multiplier"], _dt(s["started_at"]), s["hero_place"], parsed_at, + ) + for s in summaries + ] + + def build_tournament(summary: dict[str, Any], user_id: UUID) -> Tournament: return Tournament( tournament_id=summary["tournament_id"], diff --git a/apps/api/src/spin2note_api/worker.py b/apps/api/src/spin2note_api/worker.py index 2f86f01..32c0a38 100644 --- a/apps/api/src/spin2note_api/worker.py +++ b/apps/api/src/spin2note_api/worker.py @@ -12,6 +12,7 @@ import asyncio import gzip import logging +from datetime import UTC, datetime from typing import Any from uuid import UUID @@ -19,12 +20,19 @@ from .cache import RedisQueue from .clickhouse import ClickHouseBatcher -from .clickhouse.client import make_client, make_insert_fn -from .clickhouse.queries import existing_hand_ids, existing_tournament_ids +from .clickhouse.client import TABLE_COLUMNS, make_client +from .clickhouse.native import make_native_conn, make_native_insert_fn +from .clickhouse.queries import existing_hand_ids, existing_tournament_ids, user_has_rows from .config import get_settings from .db.imports import ImportCounts, finish_import from .ingest.storage import RawStorage -from .parser import ParserUnavailable, build_hands, build_tournament, parse, parse_summaries +from .parser import ( + ParserUnavailable, + build_chunk_rows, + build_tournament_rows, + parse, + parse_summaries, +) from .parser.wrapper import looks_like_summary logger = logging.getLogger("spin2note.worker") @@ -46,40 +54,36 @@ async def handle_raw( current batch and what already exists for the user — so re-uploading is a no-op that simply reports how many were skipped. """ + now = datetime.now(UTC).replace(tzinfo=None) if looks_like_summary(raw): - tournaments = [build_tournament(s, user_id) for s in parse_summaries(raw)] - total = len(tournaments) - unique: dict[str, Any] = {} - for t in tournaments: - unique.setdefault(t.tournament_id, t) - existing = await existing_tournament_ids(client, user_id, unique.keys()) - added = 0 - for tid, t in unique.items(): - if tid in existing: - continue - await batcher.submit("tournaments", t) - added += 1 + unique: dict[str, dict[str, Any]] = {} + for s in parse_summaries(raw): + unique.setdefault(s["tournament_id"], s) + total = len(unique) + existing: set[str] = set() + if await user_has_rows(client, "tournaments", user_id): + existing = await existing_tournament_ids(client, user_id, unique.keys()) + new = [s for tid, s in unique.items() if tid not in existing] + await batcher.submit_block("tournaments", build_tournament_rows(new, user_id, now)) return ImportCounts( - kind="summary", tournaments_added=added, tournaments_skipped=total - added + kind="summary", tournaments_added=len(new), tournaments_skipped=total - len(new) ) - hands = build_hands(parse(raw), user_id) - total = len(hands) - unique_h: dict[UUID, Any] = {} - for h in hands: - unique_h.setdefault(h.hand_id, h) - existing_h = await existing_hand_ids(client, user_id, list(unique_h.keys())) - added = 0 - for hid, hand in unique_h.items(): - if hid in existing_h: - continue - await batcher.submit("hands", hand) - for player in hand.players: - await batcher.submit("hand_players", player) - for action in hand.actions: - await batcher.submit("actions", action) - added += 1 - return ImportCounts(kind="hands", hands_added=added, hands_skipped=total - added) + unique_h: dict[UUID, dict[str, Any]] = {} + for h in parse(raw): + unique_h.setdefault(UUID(h["hand_id"]), h) + total = len(unique_h) + existing_h: set[UUID] = set() + if await user_has_rows(client, "hands", user_id): + existing_h = await existing_hand_ids(client, user_id, list(unique_h.keys())) + new_hands = [h for hid, h in unique_h.items() if hid not in existing_h] + blocks = build_chunk_rows(new_hands, user_id, now) + await batcher.submit_block("hands", blocks["hands"]) + await batcher.submit_block("hand_players", blocks["hand_players"]) + await batcher.submit_block("actions", blocks["actions"]) + return ImportCounts( + kind="hands", hands_added=len(new_hands), hands_skipped=total - len(new_hands) + ) async def _handle( @@ -112,9 +116,10 @@ async def run() -> None: settings = get_settings() queue = RedisQueue(settings.redis_url) storage = RawStorage(settings) - client = await make_client(settings) + client = await make_client(settings) # HTTP, reads (dedup) + native = await make_native_conn(settings) # native TCP, fast inserts batcher = ClickHouseBatcher( - make_insert_fn(client), + make_native_insert_fn(native, TABLE_COLUMNS), max_rows=settings.batch_max_rows, max_interval_seconds=settings.batch_max_interval_seconds, ) @@ -129,6 +134,7 @@ async def run() -> None: finally: await batcher.stop() await queue.close() + await native.close() def main() -> None: diff --git a/apps/api/tests/test_parser.py b/apps/api/tests/test_parser.py index 274ae8a..ddbec8a 100644 --- a/apps/api/tests/test_parser.py +++ b/apps/api/tests/test_parser.py @@ -74,6 +74,15 @@ def test_six_max_positions() -> None: assert positions == {"BTN", "SB", "BB", "UTG", "HJ", "CO"} +def test_rust_hand_id_matches_python() -> None: + # The Rust parser computes hand_id (uuid5); it must equal the Python derivation, or dedup + # would key differently between the two. + for raw in (_read("3max/sample.txt"), _read("6max/sample.txt")): + for h in parse(raw): + expected = deterministic_hand_id(h["tournament_id"], h["source_hand_id"]) + assert UUID(h["hand_id"]) == expected + + def test_deterministic_hand_id_is_stable() -> None: a = deterministic_hand_id("900000001", "SG3000000001") b = deterministic_hand_id("900000001", "SG3000000001") diff --git a/apps/api/tests/test_worker.py b/apps/api/tests/test_worker.py index 25d4533..128e6b0 100644 --- a/apps/api/tests/test_worker.py +++ b/apps/api/tests/test_worker.py @@ -30,6 +30,11 @@ async def _insert(table: str, rows) -> None: # pragma: no cover - not used async def submit(self, table: str, row) -> None: # type: ignore[override] self.rows.setdefault(table, []).append(row) + async def submit_block(self, table: str, rows) -> None: # type: ignore[override] + if not rows: + return + self.rows.setdefault(table, []).extend(rows) + class FakeCH: """Stub ClickHouse client returning a fixed set of already-existing ids.""" @@ -65,7 +70,7 @@ async def test_dedup_skips_existing_hands() -> None: # First import: nothing exists yet -> both hands added. first = CollectBatcher() await handle_raw(raw, USER, first, FakeCH()) - seen_ids = {h.hand_id for h in first.rows["hands"]} + seen_ids = {row[0] for row in first.rows["hands"]} # row[0] = hand_id (column order) assert len(seen_ids) == 2 # Re-import: those hand_ids already exist -> all skipped, nothing submitted. diff --git a/apps/api/uv.lock b/apps/api/uv.lock index b232d0e..5495dc8 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -257,6 +257,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/19/cc8bd127d28a43da249aa955cfd164cf8fd534e79e42cea96c4854d72fd0/ast_serialize-0.5.0-cp39-abi3-win_arm64.whl", hash = "sha256:92a31c9c20d25a076edaeec76b128a3535d74a24f340b9a8a7e96c9b86dc9642", size = 1081181, upload-time = "2026-05-17T17:48:28.122Z" }, ] +[[package]] +name = "asynch" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ciso8601" }, + { name = "leb128" }, + { name = "lz4" }, + { name = "pytz" }, + { name = "tzlocal" }, + { name = "zstd" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/ce/f11ee2cafa6010d295db3ddee089f14cfc5fa9b8a19d0a21356c18a49267/asynch-0.3.1.tar.gz", hash = "sha256:d8bb3c1793a74a2e9d44cc7e8406ead3cf3818c11ecd6d8405b702b23148c584", size = 59825, upload-time = "2025-11-11T15:20:22.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/b0/9547aeadb577cb0c590162661b31110f7c3c02c1b23653a1b33222c7ff70/asynch-0.3.1-py3-none-any.whl", hash = "sha256:32b30a4409b70514077f7bf8ef713a248158fe5e63504c2769a4c7f772828412", size = 76502, upload-time = "2025-11-11T15:20:21.46Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -332,6 +349,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] +[[package]] +name = "ciso8601" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/8a/075724aea06c98626109bfd670c27c248c87b9ba33e637f069bf46e8c4c3/ciso8601-2.3.3.tar.gz", hash = "sha256:db5d78d9fb0de8686fbad1c1c2d168ed52efb6e8bf8774ae26226e5034a46dae", size = 31909, upload-time = "2025-08-20T16:31:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/aa/b723a6981cfc42bbe992da23179f5dd1556e9054067985108ec6cbe34dd3/ciso8601-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e7ef14610446211c4102bf6c67f32619ab341e56db15bad6884385b43c12b064", size = 16111, upload-time = "2025-08-20T16:30:36.781Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e9/e547ec4dd75f28d8d217488130fa07767bc42fd643d61a18870487133c0e/ciso8601-2.3.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:523901aec6b0ccdf255c863ef161f476197f177c5cd33f2fbb35955c5f97fdb4", size = 24193, upload-time = "2025-08-20T16:30:38.067Z" }, + { url = "https://files.pythonhosted.org/packages/14/c8/801b78e30667cb31b4524e9dc26cbc2c03c012f9aa3f5ae21676461dc622/ciso8601-2.3.3-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:45f8254d1fb0a41e20f98e93075db7b56504adddf65e4c8b397671feba4861ca", size = 15917, upload-time = "2025-08-20T16:30:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/44/6b/dfc56a2a4e572a2a3f8c88a66dea6a9186a8e10da7c36cc84abc31bf795c/ciso8601-2.3.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:202ca99077577683e6a84d394ff2677ec19d9f406fbf35734f68be85d2bcd3f1", size = 41324, upload-time = "2025-08-20T16:30:40.321Z" }, + { url = "https://files.pythonhosted.org/packages/7c/57/cf66171cb5807fe345b03ce9e32fd91b3a8b6e5bd95710618a9a1b0f3fab/ciso8601-2.3.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7cec4e31c363e87221f2561e7083ce055a82de041e822e7c3775f8ce6250a7e", size = 41804, upload-time = "2025-08-20T16:30:41.204Z" }, + { url = "https://files.pythonhosted.org/packages/75/91/15e8871d7ae2ff0f756128e246348bdede58c08edba13cd886450ceeb304/ciso8601-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:389fef3ccc3065fa21cb6ef7d03aee63ab980591b5d87b9f0bbe349f52b16bdc", size = 41209, upload-time = "2025-08-20T16:30:42.46Z" }, + { url = "https://files.pythonhosted.org/packages/30/54/7563e20a158a4bdf3e8d13c63e02b71f9b73c662edc83cb4d5ab67171a7d/ciso8601-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c4499cfbe4da092dea95ab81aefc78b98e2d7464518e6e80107cf2b9b1f65fa2", size = 41368, upload-time = "2025-08-20T16:30:43.397Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d5/6182006dd86365bb21d1f658f70c41e266ce0f97eaf353f9d7069c51851f/ciso8601-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:1df1ca3791c6f2d543f091d88e728a60a31681ff900d9eb02f1403cf31e9c177", size = 17566, upload-time = "2025-08-20T16:30:44.706Z" }, + { url = "https://files.pythonhosted.org/packages/01/16/88154fe8247e4dcfdbaed8c6b8ccf32b1dd4389c6c95b1986bf31649eb00/ciso8601-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8afa073802c926c3244e1e5fcc5818afd3acb90fb7826a90f91ddbda0636ea70", size = 16109, upload-time = "2025-08-20T16:30:45.655Z" }, + { url = "https://files.pythonhosted.org/packages/be/46/8d46372b3802c7201c20c8b316569f27253aaafba0cdd2cd033985e8b77e/ciso8601-2.3.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:8a04e518b4adf8e35e030feaecdb4a835d39b9bb44d207e926aea8ce3447ad7c", size = 24189, upload-time = "2025-08-20T16:30:46.958Z" }, + { url = "https://files.pythonhosted.org/packages/13/80/1890e097cb76e41995de82f29c0289ca590d7135e0be3707e5b78f54350d/ciso8601-2.3.3-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:f79ad8372463ba4265981016d1648bc05f4922bc8044c4243fcbaef7a12ee9f7", size = 15925, upload-time = "2025-08-20T16:30:48.082Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e9/690a2a6beefd9d982c20adde3f09ff54a23291a699b0df7cf0c59027d9cf/ciso8601-2.3.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d5894a33f119b5ac1082df187dc58c74fe13c9c092e19ba36495c2b7cee3540b", size = 41352, upload-time = "2025-08-20T16:30:49.294Z" }, + { url = "https://files.pythonhosted.org/packages/2f/34/9a498ceb0ebd23f538e6685721c9fc4666701372c651874ed22ec46b1423/ciso8601-2.3.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09deebf3e326ec59d80019b4ad35175c90b99cde789c644b1496811fe3340587", size = 41866, upload-time = "2025-08-20T16:30:50.262Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0a/ee0981502aa1c9f28f7e89cf6cee08bdff2c6ed9d4289b00cceb8a1c500e/ciso8601-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3aa43ed59b2117baccc5bb760e5e53dad77cacba671d757c1e82e0a367b1f42a", size = 41271, upload-time = "2025-08-20T16:30:51.198Z" }, + { url = "https://files.pythonhosted.org/packages/fb/65/24a888240324188d8350bc24fb58a6d759c0ca43adfa77210f3d60370b56/ciso8601-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:289515aa3a3b86a9c3450bf482f634138b98788332d136751507bfdfe46e6031", size = 41411, upload-time = "2025-08-20T16:30:52.439Z" }, + { url = "https://files.pythonhosted.org/packages/3d/1f/febc9de191acb461e02e616e5366bc2b7757277a11b4bf215d4fb79516a8/ciso8601-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:e7288068a5bffbcc50cbe9cdaf3971f541fcd209c194fa6a59ad06066a3dcff0", size = 17573, upload-time = "2025-08-20T16:30:53.759Z" }, + { url = "https://files.pythonhosted.org/packages/ef/3a/54ad0ae2257870076b4990545a8f16221470fecea0aa7a4e1f39506db8c5/ciso8601-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82db4047d74d8b1d129e7a8da578518729912c3bd19cb71541b147e41f426381", size = 16115, upload-time = "2025-08-20T16:30:54.971Z" }, + { url = "https://files.pythonhosted.org/packages/23/fb/9fe767d44520691e2b706769466852fbdeb44a82dc294c2766bce1049d22/ciso8601-2.3.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:a553f3fc03a2ed5ca6f5716de0b314fa166461df01b45d8b36043ccac3a5e79f", size = 24214, upload-time = "2025-08-20T16:30:56.359Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ac/984fd3948f372c46c436a2b48da43f4fb7bc6f156a6f4bc858adaab79d42/ciso8601-2.3.3-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:ff59c26083b7bef6df4f0d96e4b649b484806d3d7bcc2de14ad43147c3aafb04", size = 15929, upload-time = "2025-08-20T16:30:58.352Z" }, + { url = "https://files.pythonhosted.org/packages/de/3a/5572917d4e0bec2c1ef0eda8652f9dc8d1850d29d3eef9e5e82ffe5d6791/ciso8601-2.3.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:99a1fa5a730790431d0bfcd1f3a6387f60cddc6853d8dcc5c2e140cd4d67a928", size = 41578, upload-time = "2025-08-20T16:30:59.351Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cf/07321ce5cf099b98de0c02cd4bab4818610da69743003e94c8fb6e8a59cb/ciso8601-2.3.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c35265c1b0bd2ac30ed29b49818dd38b0d1dfda43086af605d8b91722727dec0", size = 42085, upload-time = "2025-08-20T16:31:00.338Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c7/3c521d6779ee433d9596eb3fcded79549bbe371843f25e62006c04f74dc9/ciso8601-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aa9df2f84ab25454f14df92b2dd4f9aae03dbfa581565a716b3e89b8e2110c03", size = 41313, upload-time = "2025-08-20T16:31:01.313Z" }, + { url = "https://files.pythonhosted.org/packages/f9/93/efd40db0d6b512be1cbe4e7e750882c2e88f580e17f35b3e9cc9c23004b5/ciso8601-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:32e06a35eb251cfc4bbe01a858c598da0a160e4ad7f42ff52477157ceaf48061", size = 41443, upload-time = "2025-08-20T16:31:02.357Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/515f9404faa39af8df5e2b899cafbca5dbe7cd2ffe5cc124ef393ffdaf1c/ciso8601-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:7657ba9730dc1340d73b9e61eca14f341c41dd308128c808b8b084d2b85bc03e", size = 17977, upload-time = "2025-08-20T16:31:03.429Z" }, +] + [[package]] name = "click" version = "8.4.1" @@ -708,6 +757,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "leb128" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/3c/ef6fe994b4b45d84187fea994f124173d587f4f8ab0641693ef269e80f56/leb128-1.0.9.tar.gz", hash = "sha256:8f8b0e2216ba8a318e2897360d9a34d88e2c968656fcb7c7bbb1aef31010f1c6", size = 26710, upload-time = "2026-01-09T08:29:39.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/f6/62cd379fe8527c6d685013ebed11c85fd4fced125bde9b3c80ebd5759850/leb128-1.0.9-py2.py3-none-any.whl", hash = "sha256:fef16ef20aca33dfdd2f4841d8004ec4acb7ed8545b63a7bc1183292c9a3e594", size = 3700, upload-time = "2026-01-09T08:29:37.446Z" }, +] + [[package]] name = "librt" version = "0.11.0" @@ -1464,6 +1522,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1551,6 +1618,7 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "alembic" }, + { name = "asynch" }, { name = "clickhouse-connect" }, { name = "fastapi" }, { name = "httpx" }, @@ -1580,6 +1648,7 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.10" }, { name = "alembic", specifier = ">=1.14" }, + { name = "asynch", specifier = ">=0.2.5" }, { name = "clickhouse-connect", specifier = ">=0.8" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.28" }, @@ -1703,6 +1772,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" @@ -2046,3 +2127,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, ] + +[[package]] +name = "zstd" +version = "1.5.7.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/62/b9c075ad664e7c4cbb3d8d2be7c246506abe1bc7f778eb58d260ef9538c8/zstd-1.5.7.3.tar.gz", hash = "sha256:403e5205f4ac04b92e6b0cda654be2f51de268228a0db0067bc087faacf2f495", size = 672559, upload-time = "2026-01-08T16:24:43.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/ed/5a3bf2e29dc56d4cc7619929bb51f0c758de6d02967cc73c5d8755a862c0/zstd-1.5.7.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01a39efb0eeab7cc45cb308618233b624b0840d5e16dcf85456b6cca0592f203", size = 268124, upload-time = "2026-01-08T16:29:57.091Z" }, + { url = "https://files.pythonhosted.org/packages/e2/1d/efc2074ac90af938e78f2ed4004639fe24f294d9086c5280f8d9a02b9897/zstd-1.5.7.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7a8e8838cf35fa3987bfe1958584cc22e1797efce8e155a63544b4144fc671f8", size = 230988, upload-time = "2026-01-08T16:29:55.604Z" }, + { url = "https://files.pythonhosted.org/packages/2a/52/178393b8d70e23fba67f42dfce4663e4e8a30867110168beb490a36d4639/zstd-1.5.7.3-cp312-cp312-manylinux_2_14_i686.whl", hash = "sha256:f3920ac1d1cc7e9f252f3e29f217fe3cd36f2191bb3dbcae826c29e189b7ad54", size = 300207, upload-time = "2026-01-10T11:26:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7a/8dcd86a2efb2ed3f9dae39545a05d3c7ed26c7678330786ce4a44cd8b099/zstd-1.5.7.3-cp312-cp312-manylinux_2_14_x86_64.whl", hash = "sha256:143f9062953fb5590cbd47c1040d357336742c79696bf90b6d5b835279a68304", size = 304154, upload-time = "2026-01-10T11:17:40.91Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ce/0c96905ab01ffe0e53a3cec8132123b82db26bd583a71608029bcc789ebc/zstd-1.5.7.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36d1fd8647e47e1f21b345e192f1a279e925678c23dad8236b547d04456cd699", size = 2162222, upload-time = "2026-01-08T18:02:22.762Z" }, + { url = "https://files.pythonhosted.org/packages/11/c4/db4807d6a68b4628c74fd379de7e3c67ec34f19a2a80ac246b3837cde6cb/zstd-1.5.7.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1538db419afa62773cf534fc7f3009ff59ecf55ecee4e889587ac2ef0010ed8", size = 2201732, upload-time = "2026-01-08T18:02:20.835Z" }, + { url = "https://files.pythonhosted.org/packages/c5/99/c19a3c0f5580ff9c33a74f06d98d6060ed1fa6bd09b55aed9be852ec191f/zstd-1.5.7.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c5efd16adb092e2a547a7d51cfdaf6fd5680528227684c5bafc7669ab4a55f41", size = 2096459, upload-time = "2026-01-08T18:02:25.336Z" }, + { url = "https://files.pythonhosted.org/packages/23/fd/02eac30419475dbe50212c119043a2d0698a0cbc756da85fd3fd9abddf42/zstd-1.5.7.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:39b3438e64637d80a5b1860526903b92020acb9bae9ceb5adffd9838c1441328", size = 2125442, upload-time = "2026-01-08T18:02:17.715Z" }, + { url = "https://files.pythonhosted.org/packages/bb/43/3a16ff0a8c913bb9825379db1bd533c75c57c2d2f31dd9111aa9b53711f4/zstd-1.5.7.3-cp312-cp312-win32.whl", hash = "sha256:cbf48c53461e224ffc2490cfe5120a1ff40d14c84d2b512c6d6d99fc91685cf3", size = 150367, upload-time = "2026-01-08T17:03:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/83/b85875d7428e63dfa9247e41d17fac611443c774f7892f8643bd4164a6b2/zstd-1.5.7.3-cp312-cp312-win_amd64.whl", hash = "sha256:943a189910f2fea997462e3e4d7fbf727a06d231ef801ebee557b1c87568981c", size = 167604, upload-time = "2026-01-08T17:03:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/37/42/cf291e26804de2f55500cdac93f5e9fa6267cf315def8aa402529bae3a87/zstd-1.5.7.3-cp312-cp312-win_arm64.whl", hash = "sha256:85c4d508f8109afa7c51c4960626c3325af2cf1e442c6c36ebfea15d04757e3f", size = 157241, upload-time = "2026-01-08T16:47:34.615Z" }, + { url = "https://files.pythonhosted.org/packages/21/7c/f2fe6e09b9d064873ebd384f2692b9fcad3d8e9412298dfb09a935aec77d/zstd-1.5.7.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2455e56f1d265dacbd450510b8c2f632a5d8d92c23282e7723fb04af37001a2", size = 268133, upload-time = "2026-01-08T17:31:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/14/8d/0a1e49844ed82c7ab0f66dce5e0dd822742fc7e9d04f147032db33740840/zstd-1.5.7.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3486dc4f1b4e52bb059f8eec1f31daa3e540062c0f522f221782cf132a8bc9a8", size = 231005, upload-time = "2026-01-08T17:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/3b/24/0ab682096da2411f83236a10c89423f26859a65431660e460e2d637b5628/zstd-1.5.7.3-cp313-cp313-manylinux_2_14_i686.whl", hash = "sha256:1cb47bf10ffcb6a782edacfe758da2c94879f7e89c6628feb3f1254daf8cc596", size = 300230, upload-time = "2026-01-08T16:30:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/3ff0d28ea8d6b9bd931af7477fad082b99633cb7901cdd657dcb7ecfee11/zstd-1.5.7.3-cp313-cp313-manylinux_2_14_x86_64.whl", hash = "sha256:07b1378d1230ddeea8773f99d7518a3060e6468c76edd502057cb795fe278d7e", size = 297097, upload-time = "2026-01-08T16:49:41.211Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ef/25c15570fb6b06a4a03bd054afa2d084df687ac10e336b703139acc77182/zstd-1.5.7.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ee34317f013e3405108f5baea53502159809cfc4510598d614257525500c70d", size = 2162274, upload-time = "2026-01-08T17:35:24.464Z" }, + { url = "https://files.pythonhosted.org/packages/f1/24/fd16ba9e9be877a2194f05462ae77dcb62c8f90c4ecc186d9ee71e9bdc9f/zstd-1.5.7.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c19127ca2c79855376a34a2d7a6969408094b25c1f44485b0373eba4be851b98", size = 2201877, upload-time = "2026-01-08T17:35:28.596Z" }, + { url = "https://files.pythonhosted.org/packages/85/ef/73c37a81ac36429bb1bbb69c8ac43f3a154cfab739dce6424d28f95301c3/zstd-1.5.7.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e79cae70dd08cb247391312463085c624c0302e8c860d13f87f4c76502d8202", size = 2096535, upload-time = "2026-01-08T17:35:30.001Z" }, + { url = "https://files.pythonhosted.org/packages/d3/1f/859a8049634e444feb6855347d5e558c1280d87b0bc6385f13cd3d95dbe6/zstd-1.5.7.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0e83e91e5daf89037c737f5529da0f80da80a78a6ad0b1d70a09860eb267dea4", size = 2125473, upload-time = "2026-01-08T17:35:26.596Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5c/e7b8aa8eea46f032891ecef187f1d469e20f9ddf0d10607e9523b3d306a1/zstd-1.5.7.3-cp313-cp313-win32.whl", hash = "sha256:2283f3bb910c028e1b9fe76b834016012ab021025a0ea197e27a1333f85e3031", size = 150370, upload-time = "2026-01-08T17:18:12.899Z" }, + { url = "https://files.pythonhosted.org/packages/50/2b/ba558ff87ba7f6c29e3a9b1c3b3e95338146aee7250b10807229d412a9c4/zstd-1.5.7.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ad5fe4c36bab5dfa5a4b8d050bd07c50c1e69f94d381bc65337ab14cd69e5b1", size = 167602, upload-time = "2026-01-08T17:18:13.858Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7d/4a5c9813fafad2949d42deee3857d7ecc8caf369bbf82a88b60519200083/zstd-1.5.7.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e878172b0eb69ac2edc6576eb862e00747c7c25e638fb354630a1ea7cfddf49", size = 157239, upload-time = "2026-01-08T16:42:28.885Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/5fcec6bbf8aab4aeb26e3cbe8cc9fa2f323dd143d066e313b134b8c28dc8/zstd-1.5.7.3-cp313-cp313t-manylinux_2_14_x86_64.whl", hash = "sha256:7e0a7e94d5b63b4cacf2396079ca9584d11f49f87cb4e5aa21f126a8f6b83446", size = 297302, upload-time = "2026-01-08T16:36:49.027Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d9/854b2643b677d22c914e352691cc46fea875867863fb453814fabaa9cc38/zstd-1.5.7.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5412c86c34cbaf6906433ef3f2c96c407f208782f06cd3e5f01f066788adb3b8", size = 268245, upload-time = "2026-01-08T16:43:39.347Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b7/97f8c206efd27df3dc7ebe2bf2d69193a67d5de024726b15c2b79ddace30/zstd-1.5.7.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f94246befb1e473211a298c96e5768f3c63eaad814ac14d160d79ae9858e1d03", size = 230992, upload-time = "2026-01-08T16:43:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d9/e2771b6144f3b2a26daaf6bb40c233ef887fce03627e7825e11a674ced65/zstd-1.5.7.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31050e17a1a546fb82c90eee8ee3c30d22b9d0594b5937e69d38b7a5084af2a2", size = 2162418, upload-time = "2026-01-08T18:09:00.35Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a5/871ac984f6b8bad512104747e55eb3b65bd89342715e030522e9cbe3b1c7/zstd-1.5.7.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ba8ec5dfd48c86d19f880713246f85d09ee06e8cd17141956258650878000d6", size = 2202154, upload-time = "2026-01-08T18:08:57.748Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5d/828157d6fe7f72d5c5473231a575e98e4e2c861d65a8acdd0e7200588ff6/zstd-1.5.7.3-cp314-cp314-manylinux_2_14_i686.whl", hash = "sha256:3005540ba406157f3e205c998709ab5f8e68b390c658c7c238eb8986092089d5", size = 300205, upload-time = "2026-01-08T16:29:58.595Z" }, + { url = "https://files.pythonhosted.org/packages/12/71/09617dcd1d3be40fc43f210f6d99c2011861db9c8e1a444a571e1022716e/zstd-1.5.7.3-cp314-cp314-manylinux_2_14_x86_64.whl", hash = "sha256:3934b54a3b7df039fcd4cf7b0f0a38c86ce44d26321255ffc3fac73d6cdcc59d", size = 304148, upload-time = "2026-01-08T16:41:47.079Z" }, + { url = "https://files.pythonhosted.org/packages/8d/c2/6abdab0a1b0b1326bd53fa187568630d78a01bd3734e035360869d80a52b/zstd-1.5.7.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e9230cd3e9153e2bed16f332558f8f3f7d869f4d15e8fa3f9c360bfa163a8b4a", size = 2096739, upload-time = "2026-01-08T18:09:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/66/7b/3867d0279bab7d0a69fdc06403e4d23f7e6a8f54530ef6626bcf09a19d62/zstd-1.5.7.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5bffba70af539f14f9df5367b1add9119f14d5e35b658aef7b765417ea461e0e", size = 2125551, upload-time = "2026-01-08T18:09:02.133Z" }, + { url = "https://files.pythonhosted.org/packages/87/f3/75566c4d29ab144ac4e1c915e4f58ce239cbe5ae4f8376fb56edf4114866/zstd-1.5.7.3-cp314-cp314-win32.whl", hash = "sha256:a006e70c88ab67bb56989e11d820adc7601a6a7ad5558b3c6c690b19a1dadc5b", size = 153519, upload-time = "2026-01-08T17:20:15.507Z" }, + { url = "https://files.pythonhosted.org/packages/ff/22/6c42f636f92e74cda4acfeab0e64dd4a3db3a07d5f5723ae885b8155c85a/zstd-1.5.7.3-cp314-cp314-win_amd64.whl", hash = "sha256:cb4957c330c7b94b0546c7b9529723b49e865608683b9503a251fe793da9d4db", size = 171454, upload-time = "2026-01-08T17:20:17.124Z" }, + { url = "https://files.pythonhosted.org/packages/79/71/ff79a965e040d90de1cabb842ef22ef4475ef37cae81be3646cbb9c7a9b4/zstd-1.5.7.3-cp314-cp314-win_arm64.whl", hash = "sha256:a785426081ab7cafe4522876ac771d701766deea9a6d8352e87744da00e6637f", size = 163084, upload-time = "2026-01-08T16:37:52.448Z" }, + { url = "https://files.pythonhosted.org/packages/5f/46/56453790f9b7859e5593b767361adb2662318c55c3e4af0fffa1d389a2a5/zstd-1.5.7.3-cp314-cp314t-manylinux_2_14_i686.whl", hash = "sha256:b52ef154793be0399befd742328ec6f5dff95154248d6d18dd65851cf22a1a5f", size = 300364, upload-time = "2026-01-10T11:12:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9e/501836085c9c6672effc0bc11d0383b439d414edd033958209bb93c68ac3/zstd-1.5.7.3-cp314-cp314t-manylinux_2_14_x86_64.whl", hash = "sha256:8024a8ba9156b1b2e64e69d147df5ddedeaed107f9da02a3428fd7baf3e5b920", size = 304338, upload-time = "2026-01-10T11:12:03.378Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ee/b71a41ae810b429b909b4d68c9bd6decc4bca1407cac8d157b9470a654e2/zstd-1.5.7.3-cp315-cp315-manylinux_2_14_i686.whl", hash = "sha256:31ac7fbacca4759aad4b6abc13bbc05e68788e9e85a968255f7624b3b8db31df", size = 300146, upload-time = "2026-01-08T16:56:10.452Z" }, + { url = "https://files.pythonhosted.org/packages/93/24/3925528355ededa372951f10e85bec9870e3f4d325b04cfb6d9707ba7dec/zstd-1.5.7.3-cp315-cp315-manylinux_2_14_x86_64.whl", hash = "sha256:d03b2927c5843ded4d1319836a33a9c21675d2f86f916a2f234a060d4c67d87c", size = 304202, upload-time = "2026-01-08T16:35:34.94Z" }, + { url = "https://files.pythonhosted.org/packages/8d/37/2ed92c824e2f8ecff1a5f699677277a4d46ce9d527800aadf5f867455ccd/zstd-1.5.7.3-cp315-cp315t-manylinux_2_14_i686.whl", hash = "sha256:5dfbf2564eb574fc1f45613ecf28036a82533c3dd70e7bb1c9854168c638da7a", size = 300404, upload-time = "2026-01-08T16:24:41.593Z" }, + { url = "https://files.pythonhosted.org/packages/92/77/a152dde083129c3eee60c3bc83963b8c747112afe375b3d6d4a96fa15427/zstd-1.5.7.3-cp315-cp315t-manylinux_2_14_x86_64.whl", hash = "sha256:7f2f5776b902f41daf7b63e75a9384b0d7c855f824f14dabefc67814b8fa5611", size = 304333, upload-time = "2026-01-08T16:25:49.196Z" }, +] diff --git a/apps/web/src/lib/bulk-upload.ts b/apps/web/src/lib/bulk-upload.ts index 466c521..452c686 100644 --- a/apps/web/src/lib/bulk-upload.ts +++ b/apps/web/src/lib/bulk-upload.ts @@ -90,30 +90,34 @@ export async function bulkUpload( let sm: string[] = []; let smSize = 0; - for (const file of files) { - const text = await file.text(); - p.filesRead += 1; - const kind = classify(text); - if (kind === "hh") { - hh.push(text); - hhSize += text.length; - if (hhSize >= CHUNK_BYTES) { - await ship(hh.join("\n")); - hh = []; - hhSize = 0; + // Read files with bounded concurrency (disk I/O parallelism) while bundling sequentially. + const READ_CONCURRENCY = 16; + for (let i = 0; i < files.length; i += READ_CONCURRENCY) { + const texts = await Promise.all(files.slice(i, i + READ_CONCURRENCY).map((f) => f.text())); + for (const text of texts) { + p.filesRead += 1; + const kind = classify(text); + if (kind === "hh") { + hh.push(text); + hhSize += text.length; + if (hhSize >= CHUNK_BYTES) { + await ship(hh.join("\n")); + hh = []; + hhSize = 0; + } + } else if (kind === "summary") { + sm.push(text); + smSize += text.length; + if (smSize >= CHUNK_BYTES) { + await ship(sm.join("\n")); + sm = []; + smSize = 0; + } + } else { + p.filesSkipped += 1; } - } else if (kind === "summary") { - sm.push(text); - smSize += text.length; - if (smSize >= CHUNK_BYTES) { - await ship(sm.join("\n")); - sm = []; - smSize = 0; - } - } else { - p.filesSkipped += 1; } - if (p.filesRead % 250 === 0) onProgress({ ...p }); + onProgress({ ...p }); } if (hh.length) await ship(hh.join("\n")); diff --git a/crates/hh-parser/Cargo.toml b/crates/hh-parser/Cargo.toml index 34da98d..526a9e7 100644 --- a/crates/hh-parser/Cargo.toml +++ b/crates/hh-parser/Cargo.toml @@ -10,5 +10,6 @@ crate-type = ["cdylib", "rlib"] [dependencies] pyo3 = { version = "0.22", features = ["extension-module", "abi3-py312"] } +uuid = { version = "1", features = ["v5"] } [dev-dependencies] diff --git a/crates/hh-parser/src/lib.rs b/crates/hh-parser/src/lib.rs index 25906f3..7bbca89 100644 --- a/crates/hh-parser/src/lib.rs +++ b/crates/hh-parser/src/lib.rs @@ -61,6 +61,7 @@ fn parse_summaries<'py>(py: Python<'py>, raw: &str) -> PyResult(py: Python<'py>, h: &Hand) -> PyResult> { let d = PyDict::new_bound(py); + d.set_item("hand_id", &h.hand_id)?; d.set_item("source_hand_id", &h.source_hand_id)?; d.set_item("tournament_id", &h.tournament_id)?; d.set_item("format", h.format)?; diff --git a/crates/hh-parser/src/model.rs b/crates/hh-parser/src/model.rs index 2d5bdda..00c8842 100644 --- a/crates/hh-parser/src/model.rs +++ b/crates/hh-parser/src/model.rs @@ -28,6 +28,7 @@ pub struct Player { #[derive(Debug, Clone)] pub struct Hand { + pub hand_id: String, // deterministic UUIDv5(namespace, "tournament_id:source_hand_id") pub source_hand_id: String, pub tournament_id: String, pub format: &'static str, // "3max" | "6max" @@ -45,6 +46,19 @@ pub struct Hand { pub actions: Vec, } +/// Namespace for deterministic hand ids — must match Python's `_HAND_NS` +/// (6f9b2a1e-0c3d-5e4f-8a7b-1d2c3e4f5a6b) so Rust and Python agree on `hand_id`. +const HAND_NS: uuid::Uuid = uuid::Uuid::from_bytes([ + 0x6f, 0x9b, 0x2a, 0x1e, 0x0c, 0x3d, 0x5e, 0x4f, 0x8a, 0x7b, 0x1d, 0x2c, 0x3e, 0x4f, 0x5a, 0x6b, +]); + +/// Deterministic hand id: UUIDv5 of "tournament_id:source_hand_id". Re-parsing the same hand +/// yields the same id, which drives idempotent dedup. +pub fn deterministic_hand_id(tournament_id: &str, source_hand_id: &str) -> String { + let name = format!("{tournament_id}:{source_hand_id}"); + uuid::Uuid::new_v5(&HAND_NS, name.as_bytes()).to_string() +} + /// Stable, dependency-free 64-bit hash (FNV-1a) for anonymizing opponent ids. /// Deterministic across runs/versions; the hero maps to 0. pub fn villain_hash(name: &str) -> u64 { diff --git a/crates/hh-parser/src/parser.rs b/crates/hh-parser/src/parser.rs index 4965281..83ecf43 100644 --- a/crates/hh-parser/src/parser.rs +++ b/crates/hh-parser/src/parser.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; -use crate::model::{assign_positions, quantize_bb, villain_hash, Action, Hand, Player}; +use crate::model::{ + assign_positions, deterministic_hand_id, quantize_bb, villain_hash, Action, Hand, Player, +}; /// Split a raw blob into individual hand blocks (each starts with `Poker Hand #`). pub fn split_hands(raw: &str) -> Vec<&str> { @@ -244,7 +246,10 @@ pub fn parse_hand(block: &str) -> Option { // Hero-centric effective stack: min(hero, max opponent), in BB*10. let effective_stack_bb = effective_stack(&players, big_blind); + let hand_id = deterministic_hand_id(&tournament_id, &source_hand_id); + Some(Hand { + hand_id, source_hand_id, tournament_id, format, diff --git a/docker-compose.yml b/docker-compose.yml index c408864..31fa251 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,8 @@ services: CLICKHOUSE_PASSWORD: "" CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" ports: - - "8123:8123" # HTTP (used by clickhouse-connect) + - "8123:8123" # HTTP (clickhouse-connect, reads) + - "9009:9000" # native TCP (asynch, fast inserts); host 9000 is taken by MinIO ulimits: nofile: { soft: 262144, hard: 262144 } volumes: @@ -116,9 +117,14 @@ services: - api environment: CLICKHOUSE_URL: http://clickhouse:8123 + CLICKHOUSE_NATIVE_PORT: "9000" DATABASE_URL: postgresql+psycopg://postgres:postgres@postgres:5432/spin2note REDIS_URL: redis://redis:6379/0 MINIO_ENDPOINT: minio:9000 + # Workers consume one Redis queue with atomic BLPOP, so they scale horizontally. + # Default 3 replicas; override with `docker compose up --scale worker=N`. + deploy: + replicas: 3 web: build: diff --git a/migrations/clickhouse/0003_dedup_projection.sql b/migrations/clickhouse/0003_dedup_projection.sql new file mode 100644 index 0000000..404e835 --- /dev/null +++ b/migrations/clickhouse/0003_dedup_projection.sql @@ -0,0 +1,15 @@ +-- Speed up input deduplication at scale: a projection ordered by (user_id, hand_id) turns the +-- membership query `... WHERE user_id = X AND hand_id IN (...)` into an index seek instead of a +-- scan of the user's partition (the base PK ends with hand_id behind format/effective_stack). +-- +-- `hands` is a ReplacingMergeTree, so CH 24.8 requires deduplicate_merge_projection_mode to be +-- set before a projection is allowed; 'rebuild' rebuilds the projection from deduplicated data +-- on merge. Statements are separated by ';' for the /db-migrate runner. + +ALTER TABLE hands MODIFY SETTING deduplicate_merge_projection_mode = 'rebuild'; + +ALTER TABLE hands + ADD PROJECTION IF NOT EXISTS p_user_hand + (SELECT user_id, hand_id ORDER BY (user_id, hand_id)); + +ALTER TABLE hands MATERIALIZE PROJECTION p_user_hand;