compose.redis.yml

# Rusternetes cluster with Redis storage via rhino.
#
# Rhino is an etcd-compatible gRPC server backed by Redis.
# All components use their existing --etcd-servers flag pointed at rhino.
#
# Usage:
#   podman compose -f compose.redis.yml build
#   podman compose -f compose.redis.yml up -d
#   bash scripts/bootstrap-cluster.sh
#
# Requires the rhino repo adjacent to this one:
#   dev/
#     rusternetes/   (this repo)
#     rhino/         (https://github.com/calfonso/rhino)

services:
  # Redis - backing store for rhino
  redis:
    image: docker.io/library/redis:7-alpine
    container_name: rusternetes-redis
    ports:
      - "6379:6379"
    volumes:
      - redis-data:/data
    networks:
      - rusternetes-net
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 5
      start_period: 2s

  # Rhino - etcd-compatible gRPC server backed by Redis
  rhino:
    build:
      context: ./rhino
      dockerfile: ../rhino.Dockerfile
    container_name: rusternetes-rhino
    ports:
      - "2379:2379"
    depends_on:
      redis:
        condition: service_healthy
    networks:
      - rusternetes-net
    command:
      - "--listen-address"
      - "0.0.0.0:2379"
      - "--endpoint"
      - "redis://redis:6379"
    healthcheck:
      test: ["CMD-SHELL", "true"]
      interval: 5s
      timeout: 3s
      retries: 3
      start_period: 2s

  # API Server - central management component
  api-server:
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: api-server
    container_name: rusternetes-api-server
    # NET_ADMIN lets the entrypoint install a route to the pod CIDR
    # (10.244.0.0/16) via the containerd container, so the pod/service proxy
    # subresources can reach pod IPs that live in containerd's CNI netns.
    cap_add:
      - NET_ADMIN
    ports:
      - "6443:6443"
    depends_on:
      rhino:
        condition: service_healthy
    networks:
      rusternetes-net:
        aliases:
          - api-server
    volumes:
      - ./.rusternetes/certs:/etc/kubernetes/pki:ro
      # CRI socket from the containerd service: the api-server reads container
      # stats (metrics) and runs exec/logs over CRI, not a Docker socket.
      - containerd-run:/run/containerd
      # Pod container logs. containerd writes each container's stdout/stderr
      # under ${KUBELET_VOLUMES_PATH}/pod-logs/...; GET pods/<p>/log is served by
      # the api-server reading those files off the filesystem at the SAME
      # absolute path the CRI ContainerStatus.log_path reports (see
      # api-server/handlers/pod_subresources.rs::stream_container_logs).
      - ${KUBELET_VOLUMES_PATH}:${KUBELET_VOLUMES_PATH}:ro
    environment:
      - RUST_LOG=info
      - CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
    command:
      - "--bind-address"
      - "0.0.0.0:6443"
      - "--etcd-servers"
      - "http://rhino:2379"
      - "--tls"
      - "--tls-cert-file"
      - "/etc/kubernetes/pki/api-server.crt"
      - "--tls-key-file"
      - "/etc/kubernetes/pki/api-server.key"
      - "--skip-auth"
      - "--console-dir"
      - "/app/console"
      - "--log-level"
      - "info"

  # Scheduler — build-only. Runs as a static pod on node-1
  # (manifests/control-plane/kube-scheduler.yaml) via the node-1 kubelet, not as
  # its own compose service. Present only so `compose --profile build build`
  # produces rusternetes-scheduler:latest. Never starts under `up`.
  scheduler:
    profiles: ["build"]
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: scheduler
    image: rusternetes-scheduler:latest

  # Controller Manager — build-only. Runs in-cluster as a static pod on node-1
  # (manifests/control-plane/kube-controller-manager.yaml, API mode via
  # ApiStorage); `compose --profile build build` produces
  # rusternetes-controller-manager:latest for the static pod. It never starts
  # under `up` (gated by the `build` profile).
  controller-manager:
    profiles: ["build"]
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: controller-manager
    image: rusternetes-controller-manager:latest

  # Shared CRI runtime: containerd (CRI plugin) + Youki. The kubelets talk to
  # this over CRI v1 instead of a Docker/Podman socket; pods run as containers
  # inside this service. The CRI socket is shared with the kubelets via the
  # `containerd-run` volume, and KUBELET_VOLUMES_PATH is bind-mounted at the same
  # host path so the pod volumes the kubelet provisions resolve here too.
  containerd:
    image: rusternetes-containerd:latest
    build:
      context: .
      dockerfile: containerd.Dockerfile
    container_name: rusternetes-containerd
    restart: unless-stopped
    privileged: true
    networks:
      - rusternetes-net
    volumes:
      - containerd-run:/run/containerd
      # /var/lib/containerd (the overlayfs snapshotter root) must live on a real
      # filesystem, not the container's own overlay rootfs — nested overlay
      # mounts fail with "invalid argument". A named volume is host-fs-backed.
      - containerd-data:/var/lib/containerd
      - ${KUBELET_VOLUMES_PATH}:${KUBELET_VOLUMES_PATH}:rw,rshared
      - ${CERTS_PATH:-./.rusternetes/certs}:${CERTS_PATH:-/etc/rusternetes/certs}:ro

  # Kubelet - node agent that manages containers
  kubelet:
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: kubelet
    container_name: rusternetes-kubelet
    privileged: true
    depends_on:
      - rhino
      - api-server
      - containerd
    networks:
      - rusternetes-net
    volumes:
      # CRI socket shared from the containerd service.
      - containerd-run:/run/containerd
      # rshared so Memory-medium emptyDir tmpfs mounts propagate to the host
      # mount namespace and survive container restarts (#445).
      - ${KUBELET_VOLUMES_PATH}:${KUBELET_VOLUMES_PATH}:rw,rshared
      - ./.rusternetes/certs:/root/.rusternetes/certs:ro
      # Templated control-plane static pod manifests (kube-scheduler) +
      # host-absolute certs mount for the scheduler static pod's hostPath. See
      # the matching block in compose.sqlite.yml for the full rationale.
      - ./.rusternetes/manifests:/etc/rusternetes/manifests:ro
      - ${CERTS_PATH:-./.rusternetes/certs}:${CERTS_PATH:-/etc/rusternetes/certs}:ro
    environment:
      - RUST_LOG=info
      - CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
      - KUBERNETES_SERVICE_HOST_OVERRIDE=api-server
      - KUBELET_VOLUMES_PATH=${KUBELET_VOLUMES_PATH}
    command: ["--node-name", "node-1", "--etcd-servers", "http://rhino:2379", "--cluster-dns", "10.96.0.10", "--metrics-port", "10250", "--sync-interval", "3", "--pod-manifest-path", "/etc/rusternetes/manifests"]

  # Kubelet 2 - second node agent for multi-node conformance tests
  kubelet2:
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: kubelet
    container_name: rusternetes-kubelet2
    privileged: true
    depends_on:
      - rhino
      - api-server
      - containerd
    networks:
      - rusternetes-net
    volumes:
      # CRI socket shared from the containerd service.
      - containerd-run:/run/containerd
      # rshared so Memory-medium emptyDir tmpfs mounts propagate to the host
      # mount namespace and survive container restarts (#445).
      - ${KUBELET_VOLUMES_PATH}:${KUBELET_VOLUMES_PATH}:rw,rshared
      - ./.rusternetes/certs:/root/.rusternetes/certs:ro
    environment:
      - RUST_LOG=info
      - CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
      - KUBERNETES_SERVICE_HOST_OVERRIDE=api-server
      - KUBELET_VOLUMES_PATH=${KUBELET_VOLUMES_PATH}
    command: ["--node-name", "node-2", "--etcd-servers", "http://rhino:2379", "--cluster-dns", "10.96.0.10", "--metrics-port", "10250", "--sync-interval", "3"]

  # Kube-proxy runs INSIDE the containerd service's network namespace
  # (network_mode: service:containerd) so its iptables Service rules live where
  # the pods are: pod IPs come from CNI (cni0, 10.244.0.0/16) inside containerd,
  # which the host netns cannot reach. It reaches rhino over the compose network
  # (containerd is attached to rusternetes-net) at rhino:2379.
  kube-proxy:
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      additional_contexts:
        rhino: ./rhino
      target: kube-proxy
    container_name: rusternetes-kube-proxy
    privileged: true
    network_mode: "service:containerd"
    user: "0:0"
    depends_on:
      - containerd
      - api-server
    cap_add:
      - NET_ADMIN
      - NET_RAW
      - SYS_ADMIN
    volumes:
      # Mount the host kernel module tree so kube-proxy can `modprobe
      # br_netfilter` at startup (the container is privileged). Without
      # br_netfilter the bridge-nf-call-iptables sysctl cannot be enabled, so
      # L2-bridged traffic to a node IP:nodePort bypasses the host-netns
      # NodePort DNAT rules and NodePort Services are unreachable from pods
      # (ClusterIP still works — it is L3-routed through the host).
      # See crates/kube-proxy/src/iptables.rs (modprobe + sysctl on startup).
      - /lib/modules:/lib/modules:ro
    environment:
      - RUST_LOG=info
    # Storage mode: read cluster state straight from rhino. Sharing containerd's
    # netns → rhino reached at its compose alias rhino:2379.
    command: ["--node-name", "node-1", "--etcd-servers", "http://rhino:2379"]

  # DNS image, build-only. The dns binary runs as a kube-system Deployment
  # (bootstrap-dns.yaml, applied by bootstrap-cluster.sh) reading cluster
  # state from the api-server — not as a compose container.
  # Build with: docker compose -f compose.redis.yml --profile build build dns
  dns:
    profiles: ["build"]
    image: rusternetes-dns:latest
    build:
      context: .
      dockerfile: services.Dockerfile
      args:
        RUSTERNETES_GIT_SHA: "${RUSTERNETES_GIT_SHA-}"
      target: dns
      additional_contexts:
        rhino: ./rhino

networks:
  rusternetes-net:
    driver: bridge
    name: rusternetes-network

volumes:
  redis-data:
    name: rusternetes-redis-data
  # CRI socket shared from the containerd service to the kubelets + api-server.
  containerd-run:
    name: rusternetes-containerd-run
  # containerd's snapshotter root on a real (host-fs-backed) volume so the
  # overlayfs snapshotter works inside the Docker-hosted containerd.
  containerd-data:
    name: rusternetes-containerd-data