From 81923e95d19b60d7b0719d7457625a66d7867b37 Mon Sep 17 00:00:00 2001 From: grenkoff Date: Sun, 14 Jun 2026 21:17:45 +0500 Subject: [PATCH] Fix bulk-ingest data loss: serialize inserts on the single native connection --- .../src/spin2note_api/clickhouse/batcher.py | 8 ++- .../src/spin2note_api/clickhouse/native.py | 14 +++-- apps/api/tests/test_native.py | 54 +++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 apps/api/tests/test_native.py diff --git a/apps/api/src/spin2note_api/clickhouse/batcher.py b/apps/api/src/spin2note_api/clickhouse/batcher.py index 547100e..a0d5b60 100644 --- a/apps/api/src/spin2note_api/clickhouse/batcher.py +++ b/apps/api/src/spin2note_api/clickhouse/batcher.py @@ -13,11 +13,14 @@ from __future__ import annotations import asyncio +import logging from collections.abc import Awaitable, Callable, Sequence from typing import Any InsertFn = Callable[[str, Sequence[Any]], Awaitable[None]] +logger = logging.getLogger("spin2note.clickhouse.batcher") + class ClickHouseBatcher: def __init__( @@ -99,4 +102,7 @@ async def flush_all(self) -> None: async def _periodic_flush(self) -> None: while self._running: await asyncio.sleep(self._max_interval) - await self.flush_all() + try: + await self.flush_all() + except Exception: # noqa: BLE001 - never let the flusher task die silently + logger.exception("periodic flush failed; will retry on next tick") diff --git a/apps/api/src/spin2note_api/clickhouse/native.py b/apps/api/src/spin2note_api/clickhouse/native.py index 88204a1..ca4cdf0 100644 --- a/apps/api/src/spin2note_api/clickhouse/native.py +++ b/apps/api/src/spin2note_api/clickhouse/native.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio from collections.abc import Awaitable, Callable, Sequence from typing import Any @@ -31,11 +32,18 @@ async def make_native_conn(settings: Settings) -> Any: def make_native_insert_fn( conn: Any, columns: dict[str, list[str]] ) -> Callable[[str, Sequence[Any]], Awaitable[None]]: - """Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn.""" + """Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn. + + A single asynch connection cannot run concurrent queries (it raises "some records have not + been fetched"); the batcher's periodic flush can race a ``submit_block`` insert, so we + serialize all inserts on this connection with a lock. + """ + lock = asyncio.Lock() async def insert_fn(table: str, rows: Sequence[Any]) -> None: cols = ", ".join(columns[table]) - async with conn.cursor() as cur: - await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows)) + async with lock: + async with conn.cursor() as cur: + await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows)) return insert_fn diff --git a/apps/api/tests/test_native.py b/apps/api/tests/test_native.py new file mode 100644 index 0000000..4687faa --- /dev/null +++ b/apps/api/tests/test_native.py @@ -0,0 +1,54 @@ +"""The native insert_fn must serialize concurrent calls on one connection. + +A single asynch connection cannot run concurrent queries; the batcher's periodic flush can race +a submit_block insert. Regression for the data loss that caused (#missing hands). +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from spin2note_api.clickhouse.native import make_native_insert_fn + + +class _Cursor: + def __init__(self, conn: _Conn) -> None: + self._conn = conn + + async def __aenter__(self) -> _Cursor: + return self + + async def __aexit__(self, *exc: object) -> bool: + return False + + async def executemany(self, query: str, rows: list) -> None: + assert not self._conn.busy, "concurrent query on a single connection!" + self._conn.busy = True + await asyncio.sleep(0.005) # force overlap if not serialized + self._conn.busy = False + self._conn.total += len(rows) + + +class _Conn: + def __init__(self) -> None: + self.busy = False + self.total = 0 + + def cursor(self) -> _Cursor: + return _Cursor(self) + + +async def test_insert_fn_serializes_concurrent_calls() -> None: + conn = _Conn() + insert = make_native_insert_fn(conn, {"hands": ["a", "b"]}) + # Fire many inserts concurrently — the lock must prevent overlapping queries. + await asyncio.gather(*[insert("hands", [(1, 2)] * 10) for _ in range(8)]) + assert conn.total == 80 + + +def test_unknown_table_raises() -> None: + insert = make_native_insert_fn(_Conn(), {"hands": ["a"]}) + with pytest.raises(KeyError): + asyncio.run(insert("nope", [(1,)]))