diff --git a/apps/api/src/spin2note_api/config.py b/apps/api/src/spin2note_api/config.py index 9274a1a..eaa3220 100644 --- a/apps/api/src/spin2note_api/config.py +++ b/apps/api/src/spin2note_api/config.py @@ -17,6 +17,8 @@ class Settings(BaseSettings): app_name: str = "spin2note-api" environment: str = "development" + # Logging: text in development, JSON in any other environment (prod-friendly aggregation). + log_level: str = "INFO" # ClickHouse (analytics store) clickhouse_url: str = "http://localhost:8123" # HTTP, used for reads (clickhouse-connect) diff --git a/apps/api/src/spin2note_api/http/auth.py b/apps/api/src/spin2note_api/http/auth.py index 8e02fe2..94a0008 100644 --- a/apps/api/src/spin2note_api/http/auth.py +++ b/apps/api/src/spin2note_api/http/auth.py @@ -14,6 +14,7 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from ..config import Settings, get_settings +from ..logging_config import bind_log_context _bearer = HTTPBearer(auto_error=True) _optional_bearer = HTTPBearer(auto_error=False) @@ -51,6 +52,7 @@ async def require_user( ) except jwt.PyJWTError as exc: raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid token") from exc + bind_log_context(user_id=str(claims.get("sub", ""))) return claims @@ -63,9 +65,11 @@ async def current_user_id( Lets bulk imports / local dev upload without auth while honouring a real user when signed in. """ if credentials is None: + bind_log_context(user_id=settings.default_user_id) return settings.default_user_id try: claims = await require_user(credentials, settings) except HTTPException: + bind_log_context(user_id=settings.default_user_id) return settings.default_user_id return str(claims.get("sub", settings.default_user_id)) diff --git a/apps/api/src/spin2note_api/logging_config.py b/apps/api/src/spin2note_api/logging_config.py new file mode 100644 index 0000000..8eb6aaf --- /dev/null +++ b/apps/api/src/spin2note_api/logging_config.py @@ -0,0 +1,92 @@ +"""Central logging setup shared by the API and the worker. + +- One `configure_logging()` entrypoint (stdlib only — no extra deps). +- Text format in development, **JSON in any other environment** (Railway/prod aggregation). +- Level from `LOG_LEVEL` (settings). +- A contextvar-backed filter injects correlation fields (request_id, user_id, import_id, …) into + every record, so a request or import can be traced across the API and the worker. +""" + +from __future__ import annotations + +import json +import logging +from contextvars import ContextVar +from typing import Any + +from .config import Settings + +# Correlation fields surfaced in every log line when set on the current task's context. +CONTEXT_KEYS = ("request_id", "user_id", "import_id", "session_id", "object_key") + +_log_context: ContextVar[dict[str, Any]] = ContextVar("_log_context", default={}) # noqa: B039 + +_NOISY_LOGGERS = ( + "clickhouse_connect", "asynch", "urllib3", "minio", "httpcore", "httpx", "aiohttp", +) + + +def bind_log_context(**fields: Any) -> None: + """Merge correlation fields into the current context (e.g. per request / per job).""" + _log_context.set({**_log_context.get(), **fields}) + + +def reset_log_context() -> None: + _log_context.set({}) + + +class _ContextFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + ctx = _log_context.get() + for key in CONTEXT_KEYS: + setattr(record, key, ctx.get(key)) + return True + + +class _JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + data: dict[str, Any] = { + "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"), + "level": record.levelname, + "logger": record.name, + "msg": record.getMessage(), + } + for key in CONTEXT_KEYS: + value = getattr(record, key, None) + if value is not None: + data[key] = value + if record.exc_info: + data["exc"] = self.formatException(record.exc_info) + return json.dumps(data, default=str) + + +class _TextFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + line = ( + f"{self.formatTime(record, '%H:%M:%S')} {record.levelname:<7} " + f"{record.name} {record.getMessage()}" + ) + ctx = " ".join( + f"{k}={getattr(record, k)}" + for k in CONTEXT_KEYS + if getattr(record, k, None) is not None + ) + if ctx: + line += f" [{ctx}]" + if record.exc_info: + line += "\n" + self.formatException(record.exc_info) + return line + + +def configure_logging(settings: Settings) -> None: + """Idempotently configure root logging for the current process.""" + handler = logging.StreamHandler() + handler.addFilter(_ContextFilter()) + handler.setFormatter( + _TextFormatter() if settings.environment == "development" else _JsonFormatter() + ) + root = logging.getLogger() + root.handlers = [handler] + root.setLevel(settings.log_level.upper()) + for name in _NOISY_LOGGERS: + logging.getLogger(name).setLevel(logging.WARNING) diff --git a/apps/api/src/spin2note_api/main.py b/apps/api/src/spin2note_api/main.py index a4a5348..abd2a12 100644 --- a/apps/api/src/spin2note_api/main.py +++ b/apps/api/src/spin2note_api/main.py @@ -6,17 +6,25 @@ from __future__ import annotations -from fastapi import FastAPI +import logging +import time + +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from . import __version__ from .config import get_settings +from .domain.ids import new_id from .http import hands, health, imports, stats from .ingest import router as ingest_router +from .logging_config import bind_log_context, configure_logging, reset_log_context + +logger = logging.getLogger("spin2note.http") def create_app() -> FastAPI: settings = get_settings() + configure_logging(settings) app = FastAPI( title="Spin&Gold Analytics API", version=__version__, @@ -29,6 +37,25 @@ def create_app() -> FastAPI: allow_methods=["*"], allow_headers=["*"], ) + + @app.middleware("http") + async def log_requests(request: Request, call_next): # type: ignore[no-untyped-def] + reset_log_context() + request_id = request.headers.get("x-request-id") or str(new_id()) + bind_log_context(request_id=request_id) + start = time.perf_counter() + try: + response = await call_next(request) + except Exception: + logger.exception("unhandled error: %s %s", request.method, request.url.path) + raise + elapsed_ms = (time.perf_counter() - start) * 1000 + logger.info( + "%s %s -> %d (%.0fms)", + request.method, request.url.path, response.status_code, elapsed_ms, + ) + response.headers["x-request-id"] = request_id + return response app.include_router(health.router) app.include_router(ingest_router.router) app.include_router(stats.router) diff --git a/apps/api/src/spin2note_api/worker.py b/apps/api/src/spin2note_api/worker.py index 32c0a38..56ff977 100644 --- a/apps/api/src/spin2note_api/worker.py +++ b/apps/api/src/spin2note_api/worker.py @@ -26,6 +26,7 @@ from .config import get_settings from .db.imports import ImportCounts, finish_import from .ingest.storage import RawStorage +from .logging_config import bind_log_context, configure_logging, reset_log_context from .parser import ( ParserUnavailable, build_chunk_rows, @@ -93,6 +94,8 @@ async def _handle( user_id = UUID(job.get("user_id") or settings.default_user_id) key = job["object_key"] import_id = UUID(job["import_id"]) if job.get("import_id") else None + reset_log_context() + bind_log_context(user_id=str(user_id), object_key=key, import_id=str(import_id or "")) try: raw = _decode_object(key, storage.get(key)) counts = await handle_raw(raw, user_id, batcher, client) @@ -138,7 +141,7 @@ async def run() -> None: def main() -> None: - logging.basicConfig(level=logging.INFO) + configure_logging(get_settings()) asyncio.run(run()) diff --git a/apps/api/tests/test_logging.py b/apps/api/tests/test_logging.py new file mode 100644 index 0000000..3a6a94f --- /dev/null +++ b/apps/api/tests/test_logging.py @@ -0,0 +1,60 @@ +"""Central logging: context binding + JSON/text formatting + configuration.""" + +from __future__ import annotations + +import json +import logging + +from spin2note_api.config import Settings +from spin2note_api.logging_config import ( + _JsonFormatter, + _TextFormatter, + bind_log_context, + configure_logging, + reset_log_context, +) + + +def _record(msg: str = "hello") -> logging.LogRecord: + return logging.LogRecord("spin2note.test", logging.INFO, __file__, 1, msg, None, None) + + +def test_json_formatter_includes_bound_context() -> None: + reset_log_context() + bind_log_context(request_id="r1", user_id="u1") + rec = _record("ingested") + + # The context filter normally sets these; emulate it for the record. + from spin2note_api.logging_config import _ContextFilter + + _ContextFilter().filter(rec) + out = json.loads(_JsonFormatter().format(rec)) + assert out["msg"] == "ingested" + assert out["level"] == "INFO" + assert out["request_id"] == "r1" + assert out["user_id"] == "u1" + assert "session_id" not in out # unset keys are omitted + reset_log_context() + + +def test_text_formatter_appends_present_context() -> None: + reset_log_context() + bind_log_context(object_key="2026/x.txt") + rec = _record("parsed") + from spin2note_api.logging_config import _ContextFilter + + _ContextFilter().filter(rec) + line = _TextFormatter().format(rec) + assert "parsed" in line + assert "object_key=2026/x.txt" in line + reset_log_context() + + +def test_configure_logging_sets_level_and_json_outside_dev() -> None: + configure_logging(Settings(environment="production", log_level="WARNING")) + root = logging.getLogger() + assert root.level == logging.WARNING + assert isinstance(root.handlers[0].formatter, _JsonFormatter) + # restore a sane dev config for the rest of the suite + configure_logging(Settings(environment="development", log_level="INFO")) + assert isinstance(logging.getLogger().handlers[0].formatter, _TextFormatter)