docker-compose.yml

# Multi-backend self-hosted chat API stack.
#
# The `api` gateway always runs. Bring up exactly ONE inference backend via a
# compose profile. Defaults assume BACKEND_KIND=vllm — change .env to match
# whichever backend you start.
#
# Usage:
#   docker compose --profile demo      up -d --build   # CPU-only, laptop-friendly (Ollama + tiny model)
#   docker compose --profile vllm      up -d --build
#   docker compose --profile ollama    up -d --build
#   docker compose --profile llamacpp  up -d --build
#   docker compose --profile tgi       up -d --build
#   docker compose --profile sglang    up -d --build
#   docker compose --profile localai   up -d --build
#   docker compose --profile none      up -d --build   # gateway only, point BACKEND_BASE_URL at an external runtime

x-gpu-service: &gpu-service
  restart: unless-stopped
  ipc: host
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            count: all
            capabilities: ["gpu"]

services:
  api:
    build:
      context: ./api
    image: selfhosted-chat-api:latest
    container_name: selfhosted-chat-api
    restart: unless-stopped
    env_file:
      - .env
    read_only: true
    tmpfs:
      - /tmp
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    ports:
      - "${API_HOST:-127.0.0.1}:${API_PORT:-8000}:8000"
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8000/livez || exit 1"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s

  vllm:
    <<: *gpu-service
    profiles: ["vllm"]
    image: vllm/vllm-openai:latest
    container_name: vllm
    env_file:
      - .env
    environment:
      HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACE_HUB_TOKEN:-}
      HUGGINGFACE_HUB_TOKEN: ${HUGGINGFACE_HUB_TOKEN:-}
    expose:
      - "8001"
    ports:
      - "127.0.0.1:8001:8001"
    volumes:
      - ./data/hf-cache:/root/.cache/huggingface
      - ./data/vllm-cache:/root/.cache/vllm
    command:
      - --model
      - ${MODEL_NAME:-Qwen/Qwen2.5-7B-Instruct}
      - --host
      - 0.0.0.0
      - --port
      - "8001"
      - --dtype
      - ${VLLM_DTYPE:-half}
      - --max-model-len
      - ${VLLM_MAX_MODEL_LEN:-16384}
      - --gpu-memory-utilization
      - ${VLLM_GPU_MEMORY_UTILIZATION:-0.92}

  ollama:
    <<: *gpu-service
    profiles: ["ollama"]
    image: ollama/ollama:latest
    container_name: ollama
    expose:
      - "11434"
    ports:
      - "127.0.0.1:11434:11434"
    volumes:
      - ./data/ollama:/root/.ollama
    environment:
      OLLAMA_KEEP_ALIVE: "24h"

  # CPU-only demo. Works on any laptop with Docker — no GPU required.
  # docker compose --profile demo up -d --build
  # then: docker exec -it ollama-demo ollama pull qwen2.5:0.5b-instruct
  ollama-demo:
    profiles: ["demo"]
    image: ollama/ollama:latest
    container_name: ollama-demo
    restart: unless-stopped
    expose:
      - "11434"
    ports:
      - "127.0.0.1:11434:11434"
    volumes:
      - ./data/ollama:/root/.ollama
    environment:
      OLLAMA_KEEP_ALIVE: "24h"

  llamacpp:
    <<: *gpu-service
    profiles: ["llamacpp"]
    image: ghcr.io/ggerganov/llama.cpp:server-cuda
    container_name: llamacpp
    expose:
      - "8001"
    ports:
      - "127.0.0.1:8001:8001"
    volumes:
      - ./data/models:/models
    command:
      - -m
      - /models/${LLAMACPP_MODEL_FILE:-model.gguf}
      - --host
      - 0.0.0.0
      - --port
      - "8001"
      - --n-gpu-layers
      - ${LLAMACPP_NGL:-999}
      - --ctx-size
      - ${LLAMACPP_CTX:-8192}
      - --parallel
      - ${LLAMACPP_PARALLEL:-2}
      - --api-key
      - ${BACKEND_API_KEY:-unused}

  tgi:
    <<: *gpu-service
    profiles: ["tgi"]
    image: ghcr.io/huggingface/text-generation-inference:latest
    container_name: tgi
    environment:
      HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACE_HUB_TOKEN:-}
    expose:
      - "8001"
    ports:
      - "127.0.0.1:8001:8001"
    volumes:
      - ./data/hf-cache:/data
    command:
      - --model-id
      - ${MODEL_NAME:-Qwen/Qwen2.5-7B-Instruct}
      - --port
      - "8001"
      - --hostname
      - 0.0.0.0
      - --max-input-length
      - ${TGI_MAX_INPUT:-8192}
      - --max-total-tokens
      - ${TGI_MAX_TOTAL:-16384}

  sglang:
    <<: *gpu-service
    profiles: ["sglang"]
    image: lmsysorg/sglang:latest
    container_name: sglang
    environment:
      HF_TOKEN: ${HUGGINGFACE_HUB_TOKEN:-}
    expose:
      - "8001"
    ports:
      - "127.0.0.1:8001:8001"
    volumes:
      - ./data/hf-cache:/root/.cache/huggingface
    command:
      - python3
      - -m
      - sglang.launch_server
      - --model-path
      - ${MODEL_NAME:-Qwen/Qwen2.5-7B-Instruct}
      - --host
      - 0.0.0.0
      - --port
      - "8001"
      - --context-length
      - ${SGLANG_CTX:-16384}

  localai:
    profiles: ["localai"]
    image: localai/localai:latest-aio-gpu-nvidia-cuda-12
    container_name: localai
    restart: unless-stopped
    ipc: host
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: ["gpu"]
    expose:
      - "8001"
    ports:
      - "127.0.0.1:8001:8080"
    volumes:
      - ./data/localai-models:/build/models

networks:
  default:
    name: selfhosted-chat-api