Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion apps/api/src/spin2note_api/clickhouse/batcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
from __future__ import annotations

import asyncio
import logging
from collections.abc import Awaitable, Callable, Sequence
from typing import Any

InsertFn = Callable[[str, Sequence[Any]], Awaitable[None]]

logger = logging.getLogger("spin2note.clickhouse.batcher")


class ClickHouseBatcher:
def __init__(
Expand Down Expand Up @@ -99,4 +102,7 @@ async def flush_all(self) -> None:
async def _periodic_flush(self) -> None:
while self._running:
await asyncio.sleep(self._max_interval)
await self.flush_all()
try:
await self.flush_all()
except Exception: # noqa: BLE001 - never let the flusher task die silently
logger.exception("periodic flush failed; will retry on next tick")
14 changes: 11 additions & 3 deletions apps/api/src/spin2note_api/clickhouse/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from __future__ import annotations

import asyncio
from collections.abc import Awaitable, Callable, Sequence
from typing import Any

Expand All @@ -31,11 +32,18 @@ async def make_native_conn(settings: Settings) -> Any:
def make_native_insert_fn(
conn: Any, columns: dict[str, list[str]]
) -> Callable[[str, Sequence[Any]], Awaitable[None]]:
"""Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn."""
"""Build an ``insert_fn(table, rows)`` for ClickHouseBatcher backed by the native conn.

A single asynch connection cannot run concurrent queries (it raises "some records have not
been fetched"); the batcher's periodic flush can race a ``submit_block`` insert, so we
serialize all inserts on this connection with a lock.
"""
lock = asyncio.Lock()

async def insert_fn(table: str, rows: Sequence[Any]) -> None:
cols = ", ".join(columns[table])
async with conn.cursor() as cur:
await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows))
async with lock:
async with conn.cursor() as cur:
await cur.executemany(f"INSERT INTO {table} ({cols}) VALUES", list(rows))

return insert_fn
54 changes: 54 additions & 0 deletions apps/api/tests/test_native.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""The native insert_fn must serialize concurrent calls on one connection.

A single asynch connection cannot run concurrent queries; the batcher's periodic flush can race
a submit_block insert. Regression for the data loss that caused (#missing hands).
"""

from __future__ import annotations

import asyncio

import pytest

from spin2note_api.clickhouse.native import make_native_insert_fn


class _Cursor:
def __init__(self, conn: _Conn) -> None:
self._conn = conn

async def __aenter__(self) -> _Cursor:
return self

async def __aexit__(self, *exc: object) -> bool:
return False

async def executemany(self, query: str, rows: list) -> None:
assert not self._conn.busy, "concurrent query on a single connection!"
self._conn.busy = True
await asyncio.sleep(0.005) # force overlap if not serialized
self._conn.busy = False
self._conn.total += len(rows)


class _Conn:
def __init__(self) -> None:
self.busy = False
self.total = 0

def cursor(self) -> _Cursor:
return _Cursor(self)


async def test_insert_fn_serializes_concurrent_calls() -> None:
conn = _Conn()
insert = make_native_insert_fn(conn, {"hands": ["a", "b"]})
# Fire many inserts concurrently — the lock must prevent overlapping queries.
await asyncio.gather(*[insert("hands", [(1, 2)] * 10) for _ in range(8)])
assert conn.total == 80


def test_unknown_table_raises() -> None:
insert = make_native_insert_fn(_Conn(), {"hands": ["a"]})
with pytest.raises(KeyError):
asyncio.run(insert("nope", [(1,)]))
Loading