feat: add evaluation framework with DepthAPI client, benchmark dataset generation, and analysis scripts

Author: sanjeevafk

.gitignore

@@ -30,6 +30,7 @@ test-results/
 .codex
 .env.local
 .env.cloud
+myenv/
 
 # Local/dev artifacts
 .venv-ingest/

evaluation/benchmark.py

@@ -0,0 +1,129 @@
+import asyncio
+import typer
+import json
+from pathlib import Path
+from tqdm.asyncio import tqdm
+from typing import List
+
+from generate_benchmark import generate_benchmark_dataset
+from depthapi_client import DepthAPIClient
+from langchain_baseline import LangChainBaseline
+from run_judge import CustomLLMJudge
+from run_deepeval import evaluate_deepeval
+from run_ragas import evaluate_ragas
+from analyze_results import analyze_and_report
+
+app = typer.Typer()
+
+SEMAPHORE = asyncio.Semaphore(1)
+
+async def evaluate_single(item, system_name, client, judge, evals_to_run):
+    """Run evaluation for a single item."""
+    async with SEMAPHORE:
+        await asyncio.sleep(1.0)
+        query = item["query"]
+        prompt_spec = item.get("prompt_spec")
+        
+        if system_name == "depthapi":
+            res = await client.query(query, prompt_spec)
+        else:
+            res = await client.query(query)
+            
+        answer = res.get("answer", "")
+        context = res.get("context", res.get("sources", [])) # Handle different keys
+        
+        result_entry = {
+            "id": item["id"],
+            "system": system_name,
+            "query": query,
+            "prompt_spec": prompt_spec,
+            "metadata": item.get("metadata", {}),
+            "answer": answer
+        }
+        
+        if "judge" in evals_to_run:
+            judge_res = await judge.evaluate(query, answer, context, prompt_spec)
+            result_entry["judge"] = judge_res
+            
+        if "deepeval" in evals_to_run:
+            deepeval_res = evaluate_deepeval(query, answer, context)
+            result_entry["deepeval"] = deepeval_res
+            
+        if "ragas" in evals_to_run:
+            ragas_res = evaluate_ragas(query, answer, context)
+            result_entry["ragas"] = ragas_res
+            
+        return result_entry
+
+async def run_benchmark(size: int, evals: List[str], compare_baseline: bool):
+    """Run the complete benchmark."""
+    print(f"Generating dataset of size {size}...")
+    dataset = generate_benchmark_dataset(size)
+    
+    # Save dataset
+    Path("evaluation").mkdir(exist_ok=True)
+    with open("evaluation/benchmark_dataset.json", "w") as f:
+        json.dump(dataset, f, indent=2)
+        
+    judge = CustomLLMJudge() if "judge" in evals else None
+    
+    results = []
+    
+    async with DepthAPIClient() as depth_client:
+        print("Evaluating DepthAPI...")
+        tasks = [evaluate_single(item, "depthapi", depth_client, judge, evals) for item in dataset]
+        depth_results = await tqdm.gather(*tasks)
+        results.extend(depth_results)
+        
+    if compare_baseline:
+        print("Evaluating LangChain Baseline...")
+        # Create some dummy docs for baseline initialization
+        dummy_docs = [{"content": f"Dummy context for {item['query']}", "metadata": {}} for item in dataset]
+        baseline_client = LangChainBaseline(dummy_docs)
+        tasks = [evaluate_single(item, "langchain_baseline", baseline_client, judge, evals) for item in dataset]
+        baseline_results = await tqdm.gather(*tasks)
+        results.extend(baseline_results)
+        
+    print("Analyzing results...")
+    analyze_and_report(results, "evaluation/results/reports")
+    
+    # Save raw to evaluation/results/raw
+    Path("evaluation/results/raw").mkdir(parents=True, exist_ok=True)
+    with open("evaluation/results/raw/all_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+        
+    print("Benchmark complete!")
+
+@app.command()
+def main(
+    size: int = typer.Option(120, help="Number of test cases to generate"),
+    evals: List[str] = typer.Option(["deepeval", "ragas", "judge"], help="Evaluations to run"),
+    compare_baseline: bool = typer.Option(True, "--compare-baseline/--no-baseline", help="Run against LangChain baseline")
+):
+    """Run the DepthAPI evaluation benchmark."""
+    asyncio.run(run_benchmark(size, evals, compare_baseline))
+
+def fix_sys_argv():
+    """Workaround for Typer to support `--evals a b c` syntax."""
+    import sys
+    new_argv = []
+    i = 0
+    while i < len(sys.argv):
+        if sys.argv[i] == "--evals":
+            new_argv.append("--evals")
+            i += 1
+            if i < len(sys.argv) and not sys.argv[i].startswith("-"):
+                new_argv.append(sys.argv[i])
+                i += 1
+            while i < len(sys.argv) and not sys.argv[i].startswith("-"):
+                new_argv.append("--evals")
+                new_argv.append(sys.argv[i])
+                i += 1
+            continue
+        new_argv.append(sys.argv[i])
+        i += 1
+    sys.argv = new_argv
+
+if __name__ == "__main__":
+    fix_sys_argv()
+    app()

evaluation/depthapi_client.py

@@ -0,0 +1,104 @@
+import asyncio
+import httpx
+from typing import Dict, Any, Optional
+
+class DepthAPIClient:
+    """Async client for interacting with DepthAPI."""
+    
+    def __init__(self, base_url: str = "http://127.0.0.1:8000"):
+        self.base_url = base_url.rstrip("/")
+        import os
+        dev_key = os.getenv("DEV_API_KEYS", "sk-depth-test-key-12345").split(",")[0].strip()
+        self.client = httpx.AsyncClient(
+            timeout=60.0,
+            headers={"Authorization": f"Bearer {dev_key}"}
+        )
+        # Support a mock mode to avoid needing a running DepthAPI server.
+        # Set MOCK_DEPTHAPI=1 in the environment to enable.
+        self._mock = os.getenv("MOCK_DEPTHAPI", "0") == "1"
+
+    async def close(self):
+        await self.client.aclose()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+    async def query(self, query: str, prompt_spec: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """
+        Query DepthAPI with an optional PromptSpec.
+        """
+        mapped_spec = {
+            "depth": "accessible",
+            "task": "explain",
+            "reasoning": "direct",
+            "style": "normal",
+            "capabilities": []
+        }
+
+        if prompt_spec:
+            depth_map = {
+                "surface": "simple",
+                "detailed": "accessible",
+                "expert": "expert",
+                "academic": "technical"
+            }
+            if "depth" in prompt_spec:
+                mapped_spec["depth"] = depth_map.get(prompt_spec["depth"], "accessible")
+
+            tone_map = {
+                "objective": "direct",
+                "educational": "socratic",
+                "critical": "debate",
+                "concise": "guided"
+            }
+            if "tone" in prompt_spec:
+                mapped_spec["reasoning"] = tone_map.get(prompt_spec["tone"], "direct")
+
+            format_map = {
+                "markdown": "normal",
+                "bullet_points": "concise",
+                "essay": "academic",
+                "code_heavy": "normal"
+            }
+            if "format" in prompt_spec:
+                mapped_spec["style"] = format_map.get(prompt_spec["format"], "normal")
+
+            if prompt_spec.get("include_citations"):
+                mapped_spec["capabilities"].append("requires_citations")
+
+        payload = {
+            "topic": query,
+            "prompt_spec": mapped_spec,
+            "mode": "chat"
+        }
+
+        # Mocked response path
+        if self._mock:
+            # produce a deterministic short answer and fake context
+            answer = f"(MOCK) Explanation for: {query}"
+            context = [f"Mock context paragraph about {query}"]
+            return {"answer": answer, "context": context}
+
+        try:
+            response = await self.client.post(
+                f"{self.base_url}/api/query",
+                json=payload
+            )
+            response.raise_for_status()
+            res_json = response.json()
+            explanations = res_json.get("explanations", {})
+            answer = next(iter(explanations.values()), "") if explanations else ""
+            return {
+                "answer": answer,
+                "context": []
+            }
+        except httpx.HTTPError as e:
+            detail = ""
+            try:
+                detail = f": {response.json()}"
+            except Exception:
+                pass
+            return {"error": f"{str(e)}{detail}", "answer": "Error fetching from API", "context": []}

evaluation/generate_benchmark.py

@@ -0,0 +1,53 @@
+import json
+import random
+import uuid
+from typing import List, Dict, Any
+
+TECHNICAL_QUERIES = [
+    "How does asyncio event loop work in Python?",
+    "Explain the difference between threading and multiprocessing in Python.",
+    "What are the best practices for structuring a large FastAPI application?",
+    "Describe the Raft consensus algorithm and its leader election process.",
+    "How does PostgreSQL implement MVCC?",
+    "What is the time complexity of looking up a value in a Python dict and why?",
+    "Explain how Redis achieves persistence.",
+    "What are the trade-offs between GraphQL and REST?",
+    "How does garbage collection work in CPython?",
+    "Describe the architecture of Kubernetes control plane.",
+]
+
+def generate_benchmark_dataset(size: int = 120) -> List[Dict[str, Any]]:
+    """Generate a diverse benchmark dataset with PromptSpec combinations."""
+    dataset = []
+    
+    # Stratified sampling parameters
+    depths = ["surface", "detailed", "expert", "academic"]
+    tones = ["objective", "educational", "critical", "concise"]
+    formats = ["markdown", "bullet_points", "essay", "code_heavy"]
+    
+    for i in range(size):
+        query = random.choice(TECHNICAL_QUERIES) + f" (Variant {i})"
+        prompt_spec = {
+            "depth": random.choice(depths),
+            "tone": random.choice(tones),
+            "format": random.choice(formats),
+            "include_citations": random.choice([True, False])
+        }
+        
+        dataset.append({
+            "id": str(uuid.uuid4()),
+            "query": query,
+            "prompt_spec": prompt_spec,
+            "metadata": {
+                "category": "technical",
+                "difficulty": random.choice(["easy", "medium", "hard"])
+            }
+        })
+        
+    return dataset
+
+if __name__ == "__main__":
+    dataset = generate_benchmark_dataset(120)
+    with open("benchmark_dataset.json", "w") as f:
+        json.dump(dataset, f, indent=2)
+    print(f"Generated {len(dataset)} test cases in benchmark_dataset.json")