add observability in preparation for prod

README.md

@@ -73,10 +73,13 @@ pre-commit run --all-files
 
 # Database management
 
-The deployed service uses a Postgres database on AWS RDS.
-In order to generate migrations for the database locally,
+The deployed service uses a Postgres database on AWS RDS (provisioned automatically by the
+infrastructure stack). In order to generate migrations for the database locally,
 we use a Postgres docker container to generate migrations against.
 
+At runtime the task receives database connection details through environment variables
+(`DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD`) sourced from AWS Secrets Manager.
+
 After making any changes to the database models, run the
 `generate_migrations.py` script to create migrations:
 

main.py

@@ -1,3 +1,4 @@
+import os
 from contextlib import asynccontextmanager
 
 from dotenv import dotenv_values
@@ -50,6 +51,13 @@ async def lifespan(app: FastAPI):
     allow_headers=["*"],
 )
 
+metrics_enabled = os.getenv("ENABLE_PROMETHEUS_METRICS", "").lower() in {"1", "true", "yes"}
+
+if metrics_enabled:
+    from prometheus_fastapi_instrumentator import Instrumentator
+
+    Instrumentator().instrument(app).expose(app, include_in_schema=False)
+
 
 @app.get("/")
 def public_route():

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "authlib>=1.6.1",
     "sqladmin>=0.21.0",
     "itsdangerous>=2.2.0",
+    "prometheus-fastapi-instrumentator>=6.0.0",
     "apscheduler[redis,sqlalchemy]~=3.11",
     "loguru>=0.7.3",
 ]

tests/test_metrics.py

@@ -0,0 +1,36 @@
+import importlib
+
+from fastapi.testclient import TestClient
+
+import main as main_module
+from auth.management import get_management_token
+from config import get_settings
+from galaxy.config import get_galaxy_settings
+
+
+def test_metrics_endpoint_disabled(test_client):
+    response = test_client.get("/metrics")
+    assert response.status_code == 404
+
+
+def test_metrics_endpoint_enabled(monkeypatch, mock_settings, mock_galaxy_settings):
+    monkeypatch.setenv("ENABLE_PROMETHEUS_METRICS", "1")
+    instrumented_main = importlib.reload(main_module)
+    app = instrumented_main.app
+
+    app.dependency_overrides[get_settings] = lambda: mock_settings
+    app.dependency_overrides[get_galaxy_settings] = lambda: mock_galaxy_settings
+    app.dependency_overrides[get_management_token] = lambda: "mock_token"
+
+    monkeypatch.setattr("db.admin.DatabaseAdmin.setup", lambda *args, **kwargs: None)
+
+    try:
+        with TestClient(app) as client:
+            response = client.get("/metrics")
+        assert response.status_code == 200
+        assert response.headers["content-type"].startswith("text/plain")
+        assert "http_requests_total" in response.text
+    finally:
+        app.dependency_overrides.clear()
+        monkeypatch.delenv("ENABLE_PROMETHEUS_METRICS", raising=False)
+        importlib.reload(main_module)