diff --git a/AGENTS.md b/AGENTS.md index 71ccd1a1a..41955b8e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,7 @@ All core endpoints are registered under **4 path prefixes**: - `/v0/` — Legacy short - `/v1/` — OpenAI SDK / LiteLLM compatibility -**Core endpoints:** `chat/completions`, `completions`, `embeddings`, `reranking`, `models`, `models/{id}`, `health`, `pull`, `load`, `unload`, `delete`, `params`, `install`, `uninstall`, `audio/transcriptions`, `audio/speech`, `images/generations`, `images/edits`, `images/variations`, `responses`, `stats`, `system-info`, `system-stats`, `log-level`, `logs/stream` +**Core endpoints:** `chat/completions`, `completions`, `embeddings`, `reranking`, `models`, `models/{id}`, `health`, `pull`, `load`, `unload`, `delete`, `params`, `install`, `uninstall`, `audio/transcriptions`, `audio/speech`, `images/generations`, `images/edits`, `images/variations`, `responses`, `stats`, `system-info`, `system-stats`, `log-level`, `logs/stream`, `request-log/recent`, `request-log/search`, `request-log/stats`, `request-log/clear` **Ollama-compatible endpoints** (under `/api/` without version prefix): `chat`, `generate`, `tags`, `show`, `delete`, `pull`, `embed`, `embeddings`, `ps`, `version` @@ -69,7 +69,7 @@ Optional API key auth via `LEMONADE_API_KEY` env var (regular API endpoints) or ### Key Dependencies -**C++ (FetchContent):** cpp-httplib, nlohmann/json, CLI11, libcurl, zstd, libwebsockets, brotli (macOS). Platform SSL: Schannel (Windows), SecureTransport (macOS), OpenSSL (Linux). +**C++ (FetchContent):** cpp-httplib, nlohmann/json, CLI11, libcurl, zstd, libwebsockets, brotli (macOS). Platform SSL: Schannel (Windows), SecureTransport (macOS), OpenSSL (Linux). Optional **libpq** (PostgreSQL client) when `LEMONADE_REQUEST_LOG=ON` for HTTP request logging — see `docs/guide/configuration/request-log.md`. **Desktop app:** Tauri v2 (Rust), React 19, TypeScript 5.3, Webpack 5, markdown-it, highlight.js, katex. Rust crates: `tauri`, `tauri-plugin-{opener,clipboard-manager,single-instance,deep-link}`, `tokio`, `reqwest`, `serde`. @@ -133,6 +133,9 @@ python test/server_whisper.py # Image generation tests (slow) python test/server_sd.py + +# Request log review API tests (optional PostgreSQL) +python test/server_request_log.py ``` Test utilities in `test/utils/` with `server_base.py` as the base class. Test dependencies include `requests`, `httpx`, `openai`, `huggingface_hub`, `psutil`, `numpy`, `websockets`, and `ollama`. @@ -170,6 +173,7 @@ Test utilities in `test/utils/` with `server_base.py` as the base class. Test de | `src/cpp/server/anthropic_api.cpp` | Anthropic API compatibility | | `src/cpp/server/ollama_api.cpp` | Ollama API compatibility | | `src/cpp/server/mcp_server.cpp` | MCP gateway (POST /mcp) | +| `src/cpp/server/request_log_service.cpp` | PostgreSQL HTTP request logging (optional libpq) | | `src/cpp/include/lemon/websocket_server.h` | WebSocket Realtime API server | | `src/cpp/include/lemon/model_types.h` | Model type and device type enums | | `src/cpp/include/lemon/config_file.h` | config.json load/save/migrate | diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e125642f..1b1b7466c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -625,6 +625,9 @@ set(SOURCES_CORE src/cpp/server/vad.cpp src/cpp/server/realtime_session.cpp src/cpp/server/websocket_server.cpp + src/cpp/server/request_log_parser.cpp + src/cpp/server/request_log_handlers.cpp + src/cpp/server/request_log_service.cpp ) # Add platform-specific source files to core @@ -650,6 +653,18 @@ endif() # ============================================================ # Server core OBJECT library (shared by lemond and Lemonade.exe) # ============================================================ +option(LEMONADE_REQUEST_LOG "Build PostgreSQL request logging support" ON) +if(LEMONADE_REQUEST_LOG) + find_package(PostgreSQL QUIET) + if(PostgreSQL_FOUND) + message(STATUS "PostgreSQL request logging enabled (libpq found)") + else() + message(STATUS "PostgreSQL libpq not found; request logging will compile without persistence") + endif() +else() + message(STATUS "PostgreSQL request logging disabled (LEMONADE_REQUEST_LOG=OFF)") +endif() + add_library(lemonade-server-core OBJECT ${SOURCES_CORE}) # ============================================================ @@ -726,6 +741,12 @@ else() target_include_directories(lemonade-server-core PUBLIC ${libwebsockets_BINARY_DIR}/include ${libwebsockets_SOURCE_DIR}/include) endif() +if(LEMONADE_REQUEST_LOG AND PostgreSQL_FOUND) + target_compile_definitions(lemonade-server-core PUBLIC LEMONADE_HAVE_REQUEST_LOG) + target_link_libraries(lemonade-server-core PUBLIC PostgreSQL::PostgreSQL) + target_include_directories(lemonade-server-core PUBLIC ${PostgreSQL_INCLUDE_DIRS}) +endif() + # Enable ARC (Automatic Reference Counting) for macOS Objective-C++ files if(APPLE) target_compile_options(lemonade-server-core PUBLIC -fobjc-arc) @@ -1720,3 +1741,24 @@ if(EXISTS "${_GGUF_CAPS_TEST_SRC}") include(CTest) add_test(NAME GgufCapabilitiesTest COMMAND test_gguf_capabilities) endif() + +set(_REQUEST_LOG_PARSER_TEST_SRC + "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_request_log_parser.cpp" +) +set(_REQUEST_LOG_PARSER_LIB_SRC + "${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/server/request_log_parser.cpp" +) +if(EXISTS "${_REQUEST_LOG_PARSER_TEST_SRC}" AND EXISTS "${_REQUEST_LOG_PARSER_LIB_SRC}") + add_executable(test_request_log_parser + test/cpp/test_request_log_parser.cpp + src/cpp/server/request_log_parser.cpp + ) + target_include_directories(test_request_log_parser PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp/include + ${CMAKE_CURRENT_BINARY_DIR}/include + ) + target_link_libraries(test_request_log_parser PRIVATE nlohmann_json::nlohmann_json) + + include(CTest) + add_test(NAME RequestLogParserTest COMMAND test_request_log_parser) +endif() diff --git a/data/lemond.service.in b/data/lemond.service.in index 7bd816ba5..2aa90786a 100644 --- a/data/lemond.service.in +++ b/data/lemond.service.in @@ -17,6 +17,10 @@ WorkingDirectory=%S/lemonade # HF_TOKEN — HuggingFace authentication token # LEMONADE_API_KEY — require API-key auth on all routes # LEMONADE_ADMIN_API_KEY — API-key specific to internal routes +# LEMONADE_REQUEST_LOG_ENABLED — enable PostgreSQL HTTP request logging +# LEMONADE_REQUEST_LOG_DATABASE_URL — PostgreSQL URL for request logs +# LEMONADE_REQUEST_LOG_RETENTION_DAYS — retention days (-1 forever, 0 purge all hourly) +# LEMONADE_LOG_PROMPTS — store prompt/message content in request logs (default false) EnvironmentFile=-/etc/lemonade/conf.d/*.conf ExecStart=@CMAKE_INSTALL_FULL_BINDIR@/lemond ExecReload=/bin/kill -HUP $MAINPID diff --git a/data/secrets.conf b/data/secrets.conf deleted file mode 100644 index 4bb39691f..000000000 --- a/data/secrets.conf +++ /dev/null @@ -1,3 +0,0 @@ -# Installed as /etc/lemonade/conf.d/zz-secrets.conf so it loads after other -# drop-ins and keeps secrets separate from the base config file. -#LEMONADE_API_KEY= diff --git a/docs/guide/configuration/README.md b/docs/guide/configuration/README.md index 93977148c..84f45af22 100644 --- a/docs/guide/configuration/README.md +++ b/docs/guide/configuration/README.md @@ -313,6 +313,10 @@ The `LEMONADE_ADMIN_API_KEY` environment variable provides elevated access to bo **Client Behavior:** Clients (CLI, tray app) automatically prefer `LEMONADE_ADMIN_API_KEY` if set, otherwise fall back to `LEMONADE_API_KEY`. +### Request logging (PostgreSQL) + +Optional HTTP request logging to PostgreSQL is configured via environment variables (not `config.json`). See [Request Logging](./request-log.md) for setup, retention, privacy behavior, review API endpoints, and Docker/database bootstrap. + ## Remote Server Connection To make Lemonade Server accessible from other machines on your network, set the host to `0.0.0.0`: diff --git a/docs/guide/configuration/request-log.md b/docs/guide/configuration/request-log.md new file mode 100644 index 000000000..3b32dec50 --- /dev/null +++ b/docs/guide/configuration/request-log.md @@ -0,0 +1,207 @@ +# Request Logging (PostgreSQL) + +Lemonade can persist HTTP request metadata to PostgreSQL for client auditing and troubleshooting. This is useful for identifying which clients send inference requests, especially calls that set `keep_alive` or pin models in VRAM. + +## Prerequisites + +- `lemond` built with libpq support (default when `libpq` development headers are installed) +- PostgreSQL 16+ (local install or Docker) + +Linux packages: + +```bash +# Debian/Ubuntu +sudo apt install libpq-dev + +# Fedora/RHEL +sudo dnf install postgresql-devel +``` + +macOS: + +```bash +brew install libpq +export PKG_CONFIG_PATH="$(brew --prefix libpq)/lib/pkgconfig" +``` + +## Quick start with Docker + +From the repository root: + +```bash +./examples/start-request-log-db.sh +``` + +This starts PostgreSQL with a random host port (Docker assigns an available port) and prints the connection URL. + +Suggested env drop-in for systemd (`/etc/lemonade/conf.d/request-log.conf`): + +```ini +LEMONADE_REQUEST_LOG_ENABLED=true +LEMONADE_REQUEST_LOG_DATABASE_URL=postgresql://lemonade:change-me@127.0.0.1:/lemonade_logs +LEMONADE_REQUEST_LOG_RETENTION_DAYS=30 +LEMONADE_LOG_PROMPTS=false +``` + +Restart `lemond` after adding the env file. + +## Environment variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `LEMONADE_REQUEST_LOG_ENABLED` | `false` | Master switch for database request logging | +| `LEMONADE_REQUEST_LOG_DATABASE_URL` | (empty) | PostgreSQL URL, e.g. `postgresql://user:pass@host:5432/lemonade_logs` | +| `LEMONADE_REQUEST_LOG_RETENTION_DAYS` | `30` | Retention policy (see below) | +| `LEMONADE_LOG_PROMPTS` | `false` | When `true`, store prompt/message content in `redacted_body` | + +If logging is enabled but PostgreSQL is unreachable, Lemonade logs a warning and continues serving requests. + +## Retention behavior + +| Value | Behavior | +|-------|----------| +| `-1` | Keep rows forever (purge disabled) | +| `0` | Write rows normally, but purge **all** rows on each hourly purge cycle | +| `N > 0` | Delete rows older than `N` days during hourly purge | + +## Privacy and redaction + +- Authorization headers are never stored. +- JSON body fields matching sensitive key names (`api_key`, `token`, `password`, `secret`, etc.) are replaced with `[REDACTED]`. +- When `LEMONADE_LOG_PROMPTS=false` (default), only character counts are stored for `prompt` and `messages`. +- Full prompt/message content is stored only when `LEMONADE_LOG_PROMPTS=true`. + +## Review API + +All endpoints are registered under the standard quad-prefix paths (`/api/v0/`, `/api/v1/`, `/v0/`, `/v1/`). + +| Endpoint | Description | +|----------|-------------| +| `GET /api/v1/request-log/recent?limit=100` | Newest entries (max 1000) | +| `GET /api/v1/request-log/search?model=&client_ip=&path=&since=&keep_alive=` | Filtered search | +| `GET /api/v1/request-log/stats?since=24h` | Aggregates for the time window | + +When `LEMONADE_API_KEY` is set, these endpoints require Bearer authentication (admin key also accepted). + +### Example queries + +Recent entries: + +```bash +curl -s 'http://127.0.0.1:13305/api/v1/request-log/recent?limit=20' \ + -H "Authorization: Bearer $LEMONADE_API_KEY" | jq . +``` + +Find Ollama unload requests (`keep_alive=0`): + +```bash +curl -s 'http://127.0.0.1:13305/api/v1/request-log/search?keep_alive=0&limit=50' \ + -H "Authorization: Bearer $LEMONADE_API_KEY" | jq . +``` + +Stats for the last hour: + +```bash +curl -s 'http://127.0.0.1:13305/api/v1/request-log/stats?since=1h' \ + -H "Authorization: Bearer $LEMONADE_API_KEY" | jq . +``` + +Generate a sample request to verify logging: + +```bash +curl -s -X POST http://127.0.0.1:13305/api/chat \ + -H 'Content-Type: application/json' \ + -d '{"model":"llama3.2","messages":[],"keep_alive":0}' +``` + +## SQL examples + +Connect to the database: + +Use the connection URL printed by `./examples/start-request-log-db.sh`, for example: + +```bash +psql postgresql://lemonade:change-me@127.0.0.1:/lemonade_logs +``` + +Clients sending `keep_alive`: + +```sql +SELECT created_at, client_ip, path, model, keep_alive +FROM request_logs +WHERE keep_alive IS NOT NULL +ORDER BY created_at DESC +LIMIT 50; +``` + +Load requests with pinned models: + +```sql +SELECT created_at, client_ip, model, redacted_body->'_meta'->>'pinned' AS pinned +FROM request_logs +WHERE path LIKE '%/load' + AND redacted_body->'_meta'->>'pinned' = 'true' +ORDER BY created_at DESC; +``` + +## Build + +```bash +./setup.sh +cmake --build --preset default +``` + +The binary is `build/lemond`. + +To disable request-log support at compile time: + +```bash +cmake -DLEMONADE_REQUEST_LOG=OFF --preset default +``` + +## Safe systemd upgrade + +```bash +./examples/start-request-log-db.sh + +sudo tee /etc/lemonade/conf.d/request-log.conf <<'EOF' +LEMONADE_REQUEST_LOG_ENABLED=true +LEMONADE_REQUEST_LOG_DATABASE_URL=postgresql://lemonade:change-me@127.0.0.1:/lemonade_logs +LEMONADE_REQUEST_LOG_RETENTION_DAYS=30 +LEMONADE_LOG_PROMPTS=false +EOF + +Use the `` value printed by `./examples/start-request-log-db.sh`. + +sudo systemctl stop lemond.service +sudo cp build/lemond /usr/bin/lemond +sudo systemctl daemon-reload +sudo systemctl start lemond.service +sudo systemctl status lemond.service +``` + +## Patch workflow for upstream rebases + +Keep changes on a feature branch and export a patch: + +```bash +git checkout -b feature/request-log-db +# ... commit changes ... +git format-patch main --stdout > request-log.patch +``` + +Apply on a fresh branch: + +```bash +git checkout -b feature/request-log-db main +git apply request-log.patch +``` + +## Limitations + +- Streaming/chunked responses may report `response_body_bytes=0` because the body is not buffered. +- WebSocket traffic (`/realtime`, `/logs/stream`) is not logged by this subsystem. + +## Manual schema init + +The server creates the schema automatically on startup. You can also apply [`sql/request_logs_init.sql`](../../sql/request_logs_init.sql) manually. diff --git a/examples/docker-compose.request-log.yml b/examples/docker-compose.request-log.yml new file mode 100644 index 000000000..0000db994 --- /dev/null +++ b/examples/docker-compose.request-log.yml @@ -0,0 +1,14 @@ +services: + lemonade-request-log-db: + image: postgres:16 + environment: + POSTGRES_DB: lemonade_logs + POSTGRES_USER: lemonade + POSTGRES_PASSWORD: change-me + volumes: + - lemonade_request_logs:/var/lib/postgresql/data + ports: + - "5432" + +volumes: + lemonade_request_logs: diff --git a/examples/start-request-log-db.sh b/examples/start-request-log-db.sh new file mode 100755 index 000000000..885ab356d --- /dev/null +++ b/examples/start-request-log-db.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="${REPO_ROOT}/examples/docker-compose.request-log.yml" +SERVICE_NAME="lemonade-request-log-db" +DB_HOST="127.0.0.1" +DB_USER="lemonade" +DB_PASSWORD="change-me" +DB_NAME="lemonade_logs" + +if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker is required but not installed." >&2 + exit 1 +fi + +if docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE=(docker-compose) +else + echo "Error: docker compose or docker-compose is required." >&2 + exit 1 +fi + +echo "Starting PostgreSQL request log database..." +"${COMPOSE[@]}" -f "${COMPOSE_FILE}" up -d + +echo "Waiting for PostgreSQL to accept connections..." +ready=0 +for _ in $(seq 1 60); do + if "${COMPOSE[@]}" -f "${COMPOSE_FILE}" exec -T "${SERVICE_NAME}" \ + pg_isready -U "${DB_USER}" -d "${DB_NAME}" >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 +done + +if [[ "${ready}" -ne 1 ]]; then + echo "Error: PostgreSQL did not become ready in time." >&2 + exit 1 +fi + +port_line="$("${COMPOSE[@]}" -f "${COMPOSE_FILE}" port "${SERVICE_NAME}" 5432)" +DB_PORT="${port_line##*:}" +if [[ -z "${DB_PORT}" || "${DB_PORT}" == "${port_line}" ]]; then + echo "Error: Could not determine published host port for ${SERVICE_NAME}." >&2 + exit 1 +fi + +DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" + +cat < { const [leftPanelView, setLeftPanelView] = useState('models'); const [externalContentUrl, setExternalContentUrl] = useState(null); const [isLogsVisible, setIsLogsVisible] = useState(DEFAULT_LAYOUT_SETTINGS.isLogsVisible); + const [centerPanelTab, setCenterPanelTab] = useState(DEFAULT_LAYOUT_SETTINGS.centerPanelTab); const [isDownloadManagerVisible, setIsDownloadManagerVisible] = useState(false); const [modelManagerWidth, setModelManagerWidth] = useState(DEFAULT_LAYOUT_SETTINGS.modelManagerWidth); const [chatWidth, setChatWidth] = useState(DEFAULT_LAYOUT_SETTINGS.chatWidth); @@ -92,6 +95,10 @@ const AppContent: React.FC = () => { setLeftPanelView(savedView); } setIsLogsVisible(settings.layout.isLogsVisible ?? DEFAULT_LAYOUT_SETTINGS.isLogsVisible); + const savedCenterTab = settings.layout.centerPanelTab; + if (savedCenterTab === 'server-logs' || savedCenterTab === 'request-logs') { + setCenterPanelTab(savedCenterTab); + } setModelManagerWidth(settings.layout.modelManagerWidth ?? DEFAULT_LAYOUT_SETTINGS.modelManagerWidth); setChatWidth(settings.layout.chatWidth ?? DEFAULT_LAYOUT_SETTINGS.chatWidth); setLogsHeight(settings.layout.logsHeight ?? DEFAULT_LAYOUT_SETTINGS.logsHeight); @@ -105,6 +112,10 @@ const AppContent: React.FC = () => { if (urlParams.get('view') === 'logs') { setIsLogsVisible(true); } + if (urlParams.get('view') === 'request-logs') { + setIsLogsVisible(true); + setCenterPanelTab('request-logs'); + } setLayoutLoaded(true); } }; @@ -126,6 +137,7 @@ const AppContent: React.FC = () => { isModelManagerVisible, leftPanelView, isLogsVisible, + centerPanelTab, modelManagerWidth, chatWidth, logsHeight, @@ -135,7 +147,7 @@ const AppContent: React.FC = () => { } catch (error) { console.error('Failed to save layout settings:', error); } - }, [layoutLoaded, theme, isChatVisible, isModelManagerVisible, leftPanelView, isLogsVisible, modelManagerWidth, chatWidth, logsHeight]); + }, [layoutLoaded, theme, isChatVisible, isModelManagerVisible, leftPanelView, isLogsVisible, centerPanelTab, modelManagerWidth, chatWidth, logsHeight]); // Debounced save effect useEffect(() => { @@ -219,6 +231,10 @@ const AppContent: React.FC = () => { if (data.view === 'logs') { setIsLogsVisible(true); } + if (data.view === 'request-logs') { + setIsLogsVisible(true); + setCenterPanelTab('request-logs'); + } }); if (typeof unsub === 'function') unsubscribe = unsub; window?.api?.signalReady?.(); @@ -524,10 +540,12 @@ const AppContent: React.FC = () => { )} {isLogsVisible && (
- + + {centerPanelTab === 'server-logs' ? ( + + ) : ( + + )}
)} {isChatVisible && ( diff --git a/src/app/src/renderer/CenterPanelTabs.tsx b/src/app/src/renderer/CenterPanelTabs.tsx new file mode 100644 index 000000000..3d077c6f1 --- /dev/null +++ b/src/app/src/renderer/CenterPanelTabs.tsx @@ -0,0 +1,31 @@ +import React from 'react'; + +export type CenterPanelTab = 'server-logs' | 'request-logs'; + +interface CenterPanelTabsProps { + activeTab: CenterPanelTab; + onTabChange: (tab: CenterPanelTab) => void; +} + +const CenterPanelTabs: React.FC = ({ activeTab, onTabChange }) => { + return ( +
+ + +
+ ); +}; + +export default CenterPanelTabs; diff --git a/src/app/src/renderer/LogsWindow.tsx b/src/app/src/renderer/LogsWindow.tsx index a50816974..a8f4cd40d 100644 --- a/src/app/src/renderer/LogsWindow.tsx +++ b/src/app/src/renderer/LogsWindow.tsx @@ -297,7 +297,7 @@ const LogsWindow: React.FC = ({ isVisible, height }) => { return (
-

Server Logs

+

System Logs

{serverSupportsLogLevel && (