Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 27 additions & 20 deletions apps/api/src/spin2note_api/clickhouse/queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,40 @@

_FORMAT_FILTER = " AND tournament_format = {fmt:String}"

# Membership checks are batched: ClickHouse's HTTP layer rejects an over-long parameter value,
# so a 16k-id IN list must be split into chunks small enough to fit the form field.
_ID_BATCH = 1000

async def existing_hand_ids(
client: AsyncClient, user_id: UUID, ids: list[UUID]
) -> set[UUID]:

def _batched(items: list[Any], size: int = _ID_BATCH) -> list[list[Any]]:
return [items[i : i + size] for i in range(0, len(items), size)]


async def existing_hand_ids(client: AsyncClient, user_id: UUID, ids: list[UUID]) -> set[UUID]:
"""Return which of ``ids`` already exist for this user (for input deduplication)."""
if not ids:
return set()
res = await client.query(
"SELECT DISTINCT hand_id FROM hands "
"WHERE user_id = {u:UUID} AND hand_id IN {ids:Array(UUID)}",
parameters={"u": str(user_id), "ids": [str(i) for i in ids]},
)
return {row[0] if isinstance(row[0], UUID) else UUID(str(row[0])) for row in res.result_rows}
found: set[UUID] = set()
for batch in _batched(ids):
res = await client.query(
"SELECT DISTINCT hand_id FROM hands "
"WHERE user_id = {u:UUID} AND hand_id IN {ids:Array(UUID)}",
parameters={"u": str(user_id), "ids": [str(i) for i in batch]},
)
found |= {r[0] if isinstance(r[0], UUID) else UUID(str(r[0])) for r in res.result_rows}
return found


async def existing_tournament_ids(
client: AsyncClient, user_id: UUID, ids: Iterable[str]
) -> set[str]:
ids = list(ids)
if not ids:
return set()
res = await client.query(
"SELECT DISTINCT tournament_id FROM tournaments "
"WHERE user_id = {u:UUID} AND tournament_id IN {ids:Array(String)}",
parameters={"u": str(user_id), "ids": ids},
)
return {str(row[0]) for row in res.result_rows}
found: set[str] = set()
for batch in _batched(list(ids)):
res = await client.query(
"SELECT DISTINCT tournament_id FROM tournaments "
"WHERE user_id = {u:UUID} AND tournament_id IN {ids:Array(String)}",
parameters={"u": str(user_id), "ids": batch},
)
found |= {str(r[0]) for r in res.result_rows}
return found


async def overview(client: AsyncClient, user_id: UUID, fmt: str | None) -> dict[str, Any]:
Expand Down
35 changes: 35 additions & 0 deletions apps/api/tests/test_queries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Dedup membership queries must batch large id lists (ClickHouse HTTP rejects huge params)."""

from __future__ import annotations

from typing import Any
from uuid import uuid4

from spin2note_api.clickhouse import queries
from spin2note_api.clickhouse.queries import existing_hand_ids


class CountingClient:
def __init__(self) -> None:
self.calls = 0
self.max_batch = 0

async def query(self, sql: str, parameters: Any = None) -> Any:
self.calls += 1
self.max_batch = max(self.max_batch, len(parameters["ids"]))
return type("R", (), {"result_rows": []})()


async def test_existing_hand_ids_batches_large_lists() -> None:
ids = [uuid4() for _ in range(2500)]
client = CountingClient()
found = await existing_hand_ids(client, uuid4(), ids) # type: ignore[arg-type]
assert found == set()
assert client.calls == 3 # 1000 + 1000 + 500
assert client.max_batch <= queries._ID_BATCH


async def test_existing_hand_ids_empty() -> None:
client = CountingClient()
assert await existing_hand_ids(client, uuid4(), []) == set() # type: ignore[arg-type]
assert client.calls == 0
Loading