Makefile

# Backtrader Optimization System Makefile
.PHONY: help install run stop restart clean logs status redis-cli db db-stop db-restart down-all nuke run-nodb celery celery-stop celery-status flower test-dashboard test-perf test-quick

# Use a local empty Docker config to bypass credential helpers when pulling base images
DOCKER_CONFIG ?= $(PWD)/.docker-empty

# Default target
.DEFAULT_GOAL := help

# Colors for output
GREEN := \033[0;32m
YELLOW := \033[1;33m
RED := \033[0;31m
NC := \033[0m # No Color

help: ## Show this help message
	@echo "$(GREEN)Backtrader Optimization System$(NC)"
	@echo "================================"
	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "$(YELLOW)%-15s$(NC) %s\n", $$1, $$2}' $(MAKEFILE_LIST)

install: ## Install dependencies
	@echo "$(GREEN)Installing dependencies...$(NC)"
	@pip install -r deployment/requirements/requirements.txt
	@pip install -r deployment/requirements/requirements-redis.txt
	@pip install -r deployment/requirements/requirements-celery.txt
	@echo "$(GREEN)✅ Dependencies installed (including Celery and enhanced optimization)$(NC)"

run: ## Start full stack including AI Agents, Vector DB, API, Redis, Frontend, DB+Adminer, Celery workers, and Monitoring
	@echo "$(GREEN)🚀 Starting full stack with AI Agents, Vector DB, Celery workers and monitoring...$(NC)"
	@mkdir -p $(DOCKER_CONFIG) && [ -f $(DOCKER_CONFIG)/config.json ] || printf '{}' > $(DOCKER_CONFIG)/config.json
	@mkdir -p monitoring/loki monitoring/promtail monitoring/grafana/provisioning/datasources monitoring/grafana/dashboards
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml -f deployment/docker/docker-compose.monitoring.yml up -d --build db db_admin vectordb agents api redis frontend celery_worker_cpu celery_worker_optimization celery_worker_optimization_2 celery_worker_optimization_3 celery_worker_optimization_4 celery_worker_validation flower loki promtail grafana seq
	@echo "$(GREEN)✨ Core Services:$(NC)"
	@echo "$(GREEN)  API: http://localhost:8000/docs$(NC)"
	@echo "$(GREEN)  Frontend: http://localhost:5173$(NC)"
	@echo "$(GREEN)  AI Agents: http://localhost:8001/docs$(NC)"
	@echo "$(GREEN)  Vector DB UI: http://localhost:6334$(NC)"
	@echo "$(GREEN)📊 Monitoring:$(NC)"
	@echo "$(GREEN)  Flower (Celery): http://localhost:5555$(NC)"
	@echo "$(GREEN)  Adminer (DB): http://localhost:8081$(NC)"
	@echo "$(GREEN)  Grafana: http://localhost:3000 (admin/backtrader123)$(NC)"
	@echo "$(GREEN)  Seq (Logs): http://localhost:5341$(NC)"
	@echo "$(YELLOW)🤖 AI Agents: Ready (add API key to .env)$(NC)"
	@echo "$(YELLOW)⚙️  Workers: 2 CPU, 4 optimization$(NC)"

run-nodb: ## Alias for run (DB excluded); kept for clarity
	@$(MAKE) run

stop: ## Stop all services except DB (API, Agents, Redis, Frontend, Celery, Monitoring)
	@echo "$(YELLOW)🛑 Stopping all services except DB...$(NC)"
	-@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml -f deployment/docker/docker-compose.monitoring.yml stop api agents vectordb redis frontend celery_worker_cpu celery_worker_optimization celery_worker_optimization_2 celery_worker_optimization_3 celery_worker_optimization_4 celery_worker_validation flower loki promtail grafana seq 2>/dev/null || true
	@echo "$(GREEN)✅ Stopped all services (DB still running)$(NC)"

restart: ## Restart API, Redis, and Frontend (DB unchanged)
	@echo "$(YELLOW)🔁 Restarting api, redis, frontend...$(NC)"
	-@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml restart api redis frontend 2>/dev/null || true
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml up -d --build api redis frontend
	@echo "$(GREEN)✅ Restarted api, redis, frontend$(NC)"

db: ## Start or ensure DB (and Adminer) are running and persistent
	@echo "$(YELLOW)🗄  Ensuring DB services are up...$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml up -d db db_admin
	@echo "$(GREEN)✅ DB: localhost:5432 • Adminer: http://localhost:8081$(NC)"

db-stop: ## Stop DB and Adminer (data persists)
	@echo "$(YELLOW)🛑 Stopping DB and Adminer (volumes persist)...$(NC)"
	-@docker compose -f deployment/docker/docker-compose.yml stop db db_admin 2>/dev/null || true
	@echo "$(GREEN)✅ DB services stopped (data preserved)$(NC)"

db-restart: ## Restart DB and Adminer (data persists)
	@echo "$(YELLOW)🔁 Restarting DB and Adminer...$(NC)"
	-@docker compose -f deployment/docker/docker-compose.yml restart db db_admin 2>/dev/null || true
	@docker compose -f deployment/docker/docker-compose.yml up -d db db_admin
	@echo "$(GREEN)✅ DB services restarted$(NC)"

redis-start: ## Start only Redis
	@echo "$(YELLOW)🔧 Starting Redis...$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml up -d redis
	@sleep 3
	@echo "$(GREEN)✅ Redis started$(NC)"

redis-stop: ## Stop only Redis
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml stop redis

api: ## Start only FastAPI
	@echo "$(YELLOW)🌐 Starting FastAPI (compose)...$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml up -d api

status: ## Check service status
	@echo "$(GREEN)Service Status:$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml ps

logs: ## Tail logs for all running services
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml logs --tail=50 -f

redis-cli: ## Connect to Redis CLI
	@docker compose -f deployment/docker/docker-compose.yml exec redis redis-cli

down-all: ## Stop and remove all services including AI (volumes preserved)
	@echo "$(YELLOW)🛑 Bringing down entire stack (keeping volumes)...$(NC)"
	-@docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml -f deployment/docker/docker-compose.monitoring.yml down --remove-orphans 2>/dev/null || true
	@echo "$(GREEN)✅ Stack down (volumes kept, including vector DB data)$(NC)"

nuke: ## HARD RESET: stop and remove all containers and volumes (DB & Vector DB wiped!)
	@echo "$(RED)💥 HARD RESET: Removing containers and volumes (DB & Vector DB will be wiped)$(NC)"
	@echo "$(RED)⚠️  WARNING: This will delete all indexed data in the vector database!$(NC)"
	@read -p "Are you sure? Type 'yes' to continue: " confirm && [ "$$confirm" = "yes" ] || (echo "Cancelled" && exit 1)
	-@docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml -f deployment/docker/docker-compose.monitoring.yml down -v --remove-orphans 2>/dev/null || true
	-@docker system prune -af 2>/dev/null || true
	-@docker volume prune -f 2>/dev/null || true
	-@docker builder prune -af 2>/dev/null || true
	@echo "$(GREEN)✅ Full cleanup complete$(NC)"

clean: down-all ## Backward-compatible alias for non-destructive cleanup

# Shortcuts
up: run
down: stop
ps: status

# Frontend
frontend-install: ## Install frontend dependencies
	@cd frontend && corepack enable || true
	@cd frontend && npm install

frontend-dev: ## Start frontend dev server
	@cd frontend && npm run dev

frontend-run: ## Start frontend dev server in background (http://localhost:5173)
	-@pkill -f "vite" 2>/dev/null || true
	-@fuser -k 5173/tcp 2>/dev/null || true
	@cd frontend && (nohup npm run dev -- --host 0.0.0.0 --port 5173 > ../frontend-dev.log 2>&1 &)
	@echo "$(GREEN)Frontend started at http://localhost:5173 (logs: frontend-dev.log)$(NC)"

# Celery Commands
celery: ## Start Celery workers (CPU and optimization)
	@echo "$(GREEN)🚀 Starting Celery workers...$(NC)"
	@mkdir -p $(DOCKER_CONFIG) && [ -f $(DOCKER_CONFIG)/config.json ] || printf '{}' > $(DOCKER_CONFIG)/config.json
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml up -d celery_worker_cpu celery_worker_optimization celery_worker_optimization_2 celery_worker_optimization_3 celery_worker_optimization_4
	@echo "$(GREEN)✅ Celery workers started$(NC)"

celery-gpu: ## Start GPU worker for ML tasks (requires NVIDIA Docker)
	@echo "$(GREEN)🚀 Starting GPU worker...$(NC)"
	@docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml up -d celery_worker_gpu
	@echo "$(GREEN)✅ GPU worker started$(NC)"

celery-stop: ## Stop all Celery workers
	@echo "$(YELLOW)🛑 Stopping Celery workers...$(NC)"
	-@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml stop celery_worker_cpu celery_worker_optimization celery_worker_optimization_2 celery_worker_optimization_3 celery_worker_optimization_4 celery_worker_gpu flower 2>/dev/null || true
	@echo "$(GREEN)✅ Celery workers stopped$(NC)"

celery-status: ## Check Celery worker status
	@echo "$(GREEN)Celery Worker Status:$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml ps | grep celery

celery-scale: ## Scale CPU workers (usage: make celery-scale N=4)
	@echo "$(GREEN)🔄 Scaling CPU workers to $(N)...$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml up -d --scale celery_worker_cpu=$(N)
	@echo "$(GREEN)✅ Scaled to $(N) CPU workers$(NC)"

flower: ## Open Flower monitoring dashboard
	@echo "$(GREEN)🌸 Opening Flower dashboard...$(NC)"
	@open http://localhost:5555 2>/dev/null || xdg-open http://localhost:5555 2>/dev/null || echo "Open http://localhost:5555 in your browser"

celery-logs: ## Tail Celery worker logs
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.celery.yml logs --tail=50 -f celery_worker_cpu celery_worker_optimization celery_worker_optimization_2 celery_worker_optimization_3 celery_worker_optimization_4

# Monitoring Commands
grafana: ## Open Grafana dashboard
	@echo "$(GREEN)📊 Opening Grafana dashboard...$(NC)"
	@open http://localhost:3000 2>/dev/null || xdg-open http://localhost:3000 2>/dev/null || echo "Open http://localhost:3000 in your browser (admin/backtrader123)"

seq: ## Open Seq structured logging UI
	@echo "$(GREEN)📝 Opening Seq dashboard...$(NC)"
	@open http://localhost:5341 2>/dev/null || xdg-open http://localhost:5341 2>/dev/null || echo "Open http://localhost:5341 in your browser"

monitoring-status: ## Check monitoring stack status
	@echo "$(GREEN)Monitoring Stack Status:$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.monitoring.yml ps | grep -E "loki|promtail|grafana|seq"

monitoring-logs: ## Tail monitoring stack logs
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml -f deployment/docker/docker-compose.monitoring.yml logs --tail=50 -f loki promtail grafana seq

test-dashboard: ## Run comprehensive dashboard performance tests
	@echo "$(GREEN)🧪 Running Dashboard Performance Tests...$(NC)"
	@./test_dashboard_performance.sh

test-perf: ## Quick performance test (original vs optimized)
	@echo "$(GREEN)⚡ Quick Performance Test$(NC)"
	@echo "Testing original endpoint (might be slow)..."
	@time curl -s "http://localhost:8000/persist/results?limit=50" > /dev/null || echo "Original endpoint failed"
	@echo ""
	@echo "Testing optimized endpoint (should be fast)..."
	@time curl -s "http://localhost:8000/persist/results/batch?limit=50" > /dev/null || echo "Optimized endpoint not available"
	@echo ""
	@echo "Testing cache hit (should be very fast)..."
	@time curl -s "http://localhost:8000/persist/results/batch?limit=50" > /dev/null || echo "Cache test failed"

test-quick: ## Ultra-quick dashboard test (just check if endpoints work)
	@echo "$(GREEN)✓ Testing Dashboard Endpoints$(NC)"
	@echo -n "Health check: "
	@curl -s "http://localhost:8000/health" | grep -q "ok" && echo "$(GREEN)✓$(NC)" || echo "$(RED)✗$(NC)"
	@echo -n "Original endpoint: "
	@curl -s "http://localhost:8000/persist/results?limit=1" | grep -q "results" && echo "$(GREEN)✓$(NC)" || echo "$(RED)✗$(NC)"
	@echo -n "Optimized batch endpoint: "
	@curl -s "http://localhost:8000/persist/results/batch?limit=1" | grep -q "results" && echo "$(GREEN)✓$(NC)" || echo "$(RED)✗$(NC)"
	@echo -n "Cache stats endpoint: "
	@curl -s "http://localhost:8000/persist/cache/stats" | grep -q "enabled" && echo "$(GREEN)✓$(NC)" || echo "$(YELLOW)⚠ (Redis not configured)$(NC)"
	@echo -n "Database connection: "
	@docker exec backtrader-db-1 psql -U bt -d backtrader -c "SELECT 1" > /dev/null 2>&1 && echo "$(GREEN)✓$(NC)" || echo "$(RED)✗$(NC)"

# AI Agents Commands
agents: ## Start AI Agents and Vector DB services
	@echo "$(GREEN)🤖 Starting AI Agents and Vector Database...$(NC)"
	@DOCKER_CONFIG=$(DOCKER_CONFIG) docker compose -f deployment/docker/docker-compose.yml up -d --build vectordb agents
	@echo "$(GREEN)✅ AI Agents: http://localhost:8001$(NC)"
	@echo "$(GREEN)✅ Vector DB: http://localhost:6334$(NC)"
	@echo "$(YELLOW)Note: Add your API key to .env file$(NC)"

agents-stop: ## Stop AI Agents and Vector DB
	@echo "$(YELLOW)🛑 Stopping AI Agents and Vector DB...$(NC)"
	-@docker compose -f deployment/docker/docker-compose.yml stop agents vectordb 2>/dev/null || true
	@echo "$(GREEN)✅ AI services stopped$(NC)"

agents-logs: ## Tail AI Agents logs
	@docker compose -f deployment/docker/docker-compose.yml logs --tail=50 -f agents

agents-index: ## Manually trigger codebase indexing
	@echo "$(GREEN)🔍 Indexing codebase in vector database...$(NC)"
	@curl -X POST http://localhost:8001/index/codebase
	@echo "$(GREEN)✅ Indexing complete$(NC)"

agents-status: ## Check AI Agents and Vector DB status
	@echo "$(GREEN)AI Services Status:$(NC)"
	@echo -n "AI Agents API: "
	@curl -s "http://localhost:8001/health" | grep -q "healthy" && echo "$(GREEN)✓ Running$(NC)" || echo "$(RED)✗ Not running$(NC)"
	@echo -n "Vector Database: "
	@curl -s "http://localhost:6333/health" | grep -q "ok" && echo "$(GREEN)✓ Running$(NC)" || echo "$(RED)✗ Not running$(NC)"
	@echo -n "Index status: "
	@curl -s "http://localhost:8001/index/status" | grep -q '"indexed":true' && echo "$(GREEN)✓ Indexed$(NC)" || echo "$(YELLOW)⚠ Not indexed (run: make agents-index)$(NC)"

agents-test: ## Test AI Agents functionality
	@echo "$(GREEN)🧪 Testing AI Agents...$(NC)"
	@./scripts/test_agents.sh 2>/dev/null || (echo "$(YELLOW)Test script not found. Creating...$(NC)" && \
		echo '#!/bin/bash\ncurl -s http://localhost:8001/health | grep -q "healthy" && echo "✓ AI Agents working" || echo "✗ AI Agents not running"' > scripts/test_agents.sh && \
		chmod +x scripts/test_agents.sh && ./scripts/test_agents.sh)