Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/api/src/spin2note_api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ class Settings(BaseSettings):

app_name: str = "spin2note-api"
environment: str = "development"
# Logging: text in development, JSON in any other environment (prod-friendly aggregation).
log_level: str = "INFO"

# ClickHouse (analytics store)
clickhouse_url: str = "http://localhost:8123" # HTTP, used for reads (clickhouse-connect)
Expand Down
4 changes: 4 additions & 0 deletions apps/api/src/spin2note_api/http/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer

from ..config import Settings, get_settings
from ..logging_config import bind_log_context

_bearer = HTTPBearer(auto_error=True)
_optional_bearer = HTTPBearer(auto_error=False)
Expand Down Expand Up @@ -51,6 +52,7 @@ async def require_user(
)
except jwt.PyJWTError as exc:
raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid token") from exc
bind_log_context(user_id=str(claims.get("sub", "")))
return claims


Expand All @@ -63,9 +65,11 @@ async def current_user_id(
Lets bulk imports / local dev upload without auth while honouring a real user when signed in.
"""
if credentials is None:
bind_log_context(user_id=settings.default_user_id)
return settings.default_user_id
try:
claims = await require_user(credentials, settings)
except HTTPException:
bind_log_context(user_id=settings.default_user_id)
return settings.default_user_id
return str(claims.get("sub", settings.default_user_id))
92 changes: 92 additions & 0 deletions apps/api/src/spin2note_api/logging_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
"""Central logging setup shared by the API and the worker.

- One `configure_logging()` entrypoint (stdlib only — no extra deps).
- Text format in development, **JSON in any other environment** (Railway/prod aggregation).
- Level from `LOG_LEVEL` (settings).
- A contextvar-backed filter injects correlation fields (request_id, user_id, import_id, …) into
every record, so a request or import can be traced across the API and the worker.
"""

from __future__ import annotations

import json
import logging
from contextvars import ContextVar
from typing import Any

from .config import Settings

# Correlation fields surfaced in every log line when set on the current task's context.
CONTEXT_KEYS = ("request_id", "user_id", "import_id", "session_id", "object_key")

_log_context: ContextVar[dict[str, Any]] = ContextVar("_log_context", default={}) # noqa: B039

_NOISY_LOGGERS = (
"clickhouse_connect", "asynch", "urllib3", "minio", "httpcore", "httpx", "aiohttp",
)


def bind_log_context(**fields: Any) -> None:
"""Merge correlation fields into the current context (e.g. per request / per job)."""
_log_context.set({**_log_context.get(), **fields})


def reset_log_context() -> None:
_log_context.set({})


class _ContextFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
ctx = _log_context.get()
for key in CONTEXT_KEYS:
setattr(record, key, ctx.get(key))
return True


class _JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
data: dict[str, Any] = {
"ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
"level": record.levelname,
"logger": record.name,
"msg": record.getMessage(),
}
for key in CONTEXT_KEYS:
value = getattr(record, key, None)
if value is not None:
data[key] = value
if record.exc_info:
data["exc"] = self.formatException(record.exc_info)
return json.dumps(data, default=str)


class _TextFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
line = (
f"{self.formatTime(record, '%H:%M:%S')} {record.levelname:<7} "
f"{record.name} {record.getMessage()}"
)
ctx = " ".join(
f"{k}={getattr(record, k)}"
for k in CONTEXT_KEYS
if getattr(record, k, None) is not None
)
if ctx:
line += f" [{ctx}]"
if record.exc_info:
line += "\n" + self.formatException(record.exc_info)
return line


def configure_logging(settings: Settings) -> None:
"""Idempotently configure root logging for the current process."""
handler = logging.StreamHandler()
handler.addFilter(_ContextFilter())
handler.setFormatter(
_TextFormatter() if settings.environment == "development" else _JsonFormatter()
)
root = logging.getLogger()
root.handlers = [handler]
root.setLevel(settings.log_level.upper())
for name in _NOISY_LOGGERS:
logging.getLogger(name).setLevel(logging.WARNING)
29 changes: 28 additions & 1 deletion apps/api/src/spin2note_api/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,25 @@

from __future__ import annotations

from fastapi import FastAPI
import logging
import time

from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware

from . import __version__
from .config import get_settings
from .domain.ids import new_id
from .http import hands, health, imports, stats
from .ingest import router as ingest_router
from .logging_config import bind_log_context, configure_logging, reset_log_context

logger = logging.getLogger("spin2note.http")


def create_app() -> FastAPI:
settings = get_settings()
configure_logging(settings)
app = FastAPI(
title="Spin&Gold Analytics API",
version=__version__,
Expand All @@ -29,6 +37,25 @@ def create_app() -> FastAPI:
allow_methods=["*"],
allow_headers=["*"],
)

@app.middleware("http")
async def log_requests(request: Request, call_next): # type: ignore[no-untyped-def]
reset_log_context()
request_id = request.headers.get("x-request-id") or str(new_id())
bind_log_context(request_id=request_id)
start = time.perf_counter()
try:
response = await call_next(request)
except Exception:
logger.exception("unhandled error: %s %s", request.method, request.url.path)
raise
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(
"%s %s -> %d (%.0fms)",
request.method, request.url.path, response.status_code, elapsed_ms,
)
response.headers["x-request-id"] = request_id
return response
app.include_router(health.router)
app.include_router(ingest_router.router)
app.include_router(stats.router)
Expand Down
5 changes: 4 additions & 1 deletion apps/api/src/spin2note_api/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from .config import get_settings
from .db.imports import ImportCounts, finish_import
from .ingest.storage import RawStorage
from .logging_config import bind_log_context, configure_logging, reset_log_context
from .parser import (
ParserUnavailable,
build_chunk_rows,
Expand Down Expand Up @@ -93,6 +94,8 @@ async def _handle(
user_id = UUID(job.get("user_id") or settings.default_user_id)
key = job["object_key"]
import_id = UUID(job["import_id"]) if job.get("import_id") else None
reset_log_context()
bind_log_context(user_id=str(user_id), object_key=key, import_id=str(import_id or ""))
try:
raw = _decode_object(key, storage.get(key))
counts = await handle_raw(raw, user_id, batcher, client)
Expand Down Expand Up @@ -138,7 +141,7 @@ async def run() -> None:


def main() -> None:
logging.basicConfig(level=logging.INFO)
configure_logging(get_settings())
asyncio.run(run())


Expand Down
60 changes: 60 additions & 0 deletions apps/api/tests/test_logging.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""Central logging: context binding + JSON/text formatting + configuration."""

from __future__ import annotations

import json
import logging

from spin2note_api.config import Settings
from spin2note_api.logging_config import (
_JsonFormatter,
_TextFormatter,
bind_log_context,
configure_logging,
reset_log_context,
)


def _record(msg: str = "hello") -> logging.LogRecord:
return logging.LogRecord("spin2note.test", logging.INFO, __file__, 1, msg, None, None)


def test_json_formatter_includes_bound_context() -> None:
reset_log_context()
bind_log_context(request_id="r1", user_id="u1")
rec = _record("ingested")

# The context filter normally sets these; emulate it for the record.
from spin2note_api.logging_config import _ContextFilter

_ContextFilter().filter(rec)
out = json.loads(_JsonFormatter().format(rec))
assert out["msg"] == "ingested"
assert out["level"] == "INFO"
assert out["request_id"] == "r1"
assert out["user_id"] == "u1"
assert "session_id" not in out # unset keys are omitted
reset_log_context()


def test_text_formatter_appends_present_context() -> None:
reset_log_context()
bind_log_context(object_key="2026/x.txt")
rec = _record("parsed")
from spin2note_api.logging_config import _ContextFilter

_ContextFilter().filter(rec)
line = _TextFormatter().format(rec)
assert "parsed" in line
assert "object_key=2026/x.txt" in line
reset_log_context()


def test_configure_logging_sets_level_and_json_outside_dev() -> None:
configure_logging(Settings(environment="production", log_level="WARNING"))
root = logging.getLogger()
assert root.level == logging.WARNING
assert isinstance(root.handlers[0].formatter, _JsonFormatter)
# restore a sane dev config for the rest of the suite
configure_logging(Settings(environment="development", log_level="INFO"))
assert isinstance(logging.getLogger().handlers[0].formatter, _TextFormatter)
Loading