docker-compose.yml

services:

 # -------------------------------------------------------
 # PRODUCER (CAMER STREAM INGESTOR)
 # -------------------------------------------------------
  kafka-producer: 
    build: 
      context: .
      dockerfile: kafka-producers/Dockerfile.producer
    container_name: kafka_producer
    network_mode: host
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      # KAFKA_BROKER: ${KAFKA_BROKER} 
      KAFKA_BROKER: localhost:29092
      KAFKA_TOPIC: ${KAFKA_TOPIC_1}
      URL: ${URL}
      METRICS_PORT: 8000
    ports:
      - "8000:8000"
    restart: unless-stopped


  # -------------------------------------------------------
  # CORE INFRASTRUCTURE
  # -------------------------------------------------------
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    container_name: kafka_zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - "2181:2181"
    healthcheck:
      test: ["CMD", "echo", "ruok", "|", "nc", "localhost", "2181"]
      interval: 5s
      timeout: 10s
      retries: 5
    volumes: 
      - zookeeper_data:/data
      - zookeeper_log:/datalog

  kafka:
    image: confluentinc/cp-kafka:7.5.0
    hostname: kafka
    container_name: kafka
    ports:
      - "9092:9092"
      - "29092:29092"
    depends_on:
      zookeeper:
        condition: service_healthy
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      # KAFKA_TOPIC_NAME: ${KAFKA_TOPIC}
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_NUM_PARTITIONS: 3  # Increased from 1 for parallel processing
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
    healthcheck:
      test: ["CMD", "bash", "-c", "</dev/tcp/localhost/9092 >/dev/null 2>&1"]
      interval: 10s
      timeout: 10s
      retries: 10
    volumes: 
      - kafka_data:/var/lib/kafka/data  
  
  kafdrop:
    image: obsidiandynamics/kafdrop:3.30.0
    container_name: cv_kafdrop
    depends_on:
      - kafka
    environment:
      KAFKA_BROKERCONNECT: kafka:9092
    ports:
      - "19000:9000"
    restart: unless-stopped
  


  kafka-preprocessor:
    build: 
      context: .
      dockerfile: kafka-consumer/Dockerfile.consumer
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      KAFKA_INPUT_TOPIC: ${KAFKA_TOPIC_1}
      KAFKA_OUTPUT_TOPIC: ${KAFKA_TOPIC_2}
      KAFKA_DLQ_TOPIC: dlq-preprocessing-errors
      CONSUMER_GROUP_ID: ${CONSUMER_GROUP_ID_1}
      YOLO_INPUT_SIZE_WIDTH: 416  # Optimal for CPU: 416x416 balances speed and accuracy
      YOLO_INPUT_SIZE_HEIGHT: 416
      OUTPUT_JPEG_QUALITY: 90  # Higher quality for better detection (85-95 recommended)
      ENABLE_CLAHE: "true"  # Enable adaptive contrast for varying lighting conditions
      ENABLE_MOTION_DETECTION: "true"  # Skip static frames to reduce processing load
      MOTION_THRESHOLD: "5.0"  # Percentage of pixels changed to consider motion (1-20 typical)
      MIN_FRAME_INTERVAL: "0.5"  # Minimum seconds between frames (prevent indefinite skipping)
      METRICS_PORT: 8001
    ports:
      - "8001:8001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8001/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    restart: unless-stopped


 # -------------------------------------------------------
 # VIEWERS
 # -------------------------------------------------------
  kafka-viewer: 
    build: 
      context: .
      dockerfile: view-process/Dockerfile.viewer
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      KAFKA_TOPIC: ${KAFKA_TOPIC_2}
      CONSUMER_GROUP_ID: ${CONSUMER_GROUP_ID_2}
      METRICS_PORT: 8004
    ports:
      - "5000:5000"
      - "8004:8004"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped

  detector-viewer:
    build:
      context: .
      dockerfile: view-detection/Dockerfile.viewer-d
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: kafka:9092
      KAFKA_TOPIC: ${KAFKA_TOPIC_4}
      CONSUMER_GROUP_ID: ${CONSUMER_GROUP_ID_4}
      METRICS_PORT: 8005
    ports:
      - "7000:7000"
      - "8005:8005"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:7000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped

  yolo-inference:
    build: 
      context: .
      dockerfile: model/Dockerfile.model
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKER: ${KAFKA_BROKER}
      KAFKA_INPUT_TOPIC: ${KAFKA_TOPIC_2}
      KAFKA_VISUAL_TOPIC: ${KAFKA_TOPIC_4}
      KAFKA_DATA_TOPIC: ${KAFKA_TOPIC_3}
      KAFKA_DLQ_TOPIC: dlq-inference-errors
      CONSUMER_GROUP_ID: ${CONSUMER_GROUP_ID_3}
      MODEL_WEIGHTS_PATH: ${MODEL_WEIGHTS_PATH}
      DEVICE: "auto"  # Auto-detect GPU, fallback to CPU
      USE_TRACKING: "false"  # Detection only - much faster
      INPUT_SIZE: "416"      # Match preprocessor output: 416x416 for CPU optimization
      CAMERA_ID: ${CAMERA_ID}  # Camera identifier for multi-camera deployments
      METRICS_PORT: 8002
    ports:
      - "8002:8002"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped
#     # GPU support (uncomment when NVIDIA Container Toolkit is installed)
#     # deploy:
#     #   resources:
#     #     reservations:
#     #       devices:
#     #         - driver: nvidia
#     #           count: 1
#     #           capabilities: [gpu]


  
#   # -------------------------------------------------------
#   # DATA STORAGE AND MANAGEMENT 
#   minio-writer:
#     build:
#       context: .
#       dockerfile: minio/Dockerfile.minio-writer
#     depends_on:
#       kafka:
#         condition: service_healthy
#       minio:
#         condition: service_healthy
#     environment:
#       KAFKA_BROKER: kafka:9092
#       KAFKA_TOPIC: ${KAFKA_TOPIC_3}
#       KAFKA_VISUAL_TOPIC: yolo-visual-output
#       KAFKA_DLQ_TOPIC: dlq-storage-errors
#       CONSUMER_GROUP_ID: ${CONSUMER_GROUP_ID_5}
#       FRAMES_CONSUMER_GROUP: frames-saver-group
#       MINIO_ENDPOINT: minio:9000
#       MINIO_ROOT_USER: minioadmin
#       MINIO_ROOT_PASSWORD: minioadmin
#       BUCKET_NAME: detections-data
#       FRAMES_BUCKET_NAME: annotated-frames
#       BATCH_SIZE: 50 # Write a file after 50 detections...
#       BATCH_TIMEOUT_SEC: 30 # ...or every 30 seconds
#       SAVE_ANNOTATED_FRAMES: "true"  # Save annotated frames to MinIO for review
#       METRICS_PORT: 8003
#     ports:
#       - "8003:8003"
#     healthcheck:
#       test: ["CMD", "curl", "-f", "http://localhost:8003/health"]
#       interval: 30s
#       timeout: 10s
#       retries: 3
#     restart: unless-stopped


#   minio:
#     image: minio/minio
#     container_name: minio
#     command: server /data --console-address ":9001"
#     ports:
#       - "9000:9000" # MinIO API
#       - "9001:9001" # MinIO Console UI
#     volumes:
#       - minio_data:/data
#     environment:
#       MINIO_ROOT_USER: minioadmin
#       MINIO_ROOT_PASSWORD: minioadmin
#     healthcheck:
#       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
#       interval: 30s
#       timeout: 20s
#       retries: 3
   


  # -------------------------------------------------------
  # OBSERVABILITY AND MONITORING
  # -------------------------------------------------------
  prometheus:
    image: prom/prometheus:v2.48.0
    container_name: cv_prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
      - '--web.console.templates=/usr/share/prometheus/consoles'
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    restart: unless-stopped

  grafana:
    image: grafana/grafana:10.2.2
    container_name: cv_grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    depends_on:
      - prometheus
    restart: unless-stopped


  # -------------------------------------------------------
  # MANAGEMENT INTERFACES
  # -------------------------------------------------------
  # portainer:
  #   image: portainer/portainer-ce:latest
  #   container_name: cv_portainer
  #   restart: always
  #   ports:
  #     - "9443:9443"
  #   volumes:
  #     - /var/run/docker.sock:/var/run/docker.sock
  #     - portainer_data:/data



  # trino-coordinator:
  #   image: trinodb/trino:402
  #   container_name: trino-coordinator
  #   ports:
  #     - "8088:8080" 
  #   volumes:
  #     - ./trino/etc:/etc/trino


  
volumes: 
  zookeeper_data:
  zookeeper_log:
  kafka_data:
  portainer_data:
  minio_data:
  prometheus_data:
  grafana_data: