validate_implementation.py

#!/usr/bin/env python3
"""
Validation script - tests implementation without API calls.

This verifies the code structure, logic, and algorithms work correctly
without requiring API keys or making actual LLM calls.
"""

import sys
from meta_prompting_engine.complexity import ComplexityAnalyzer
from meta_prompting_engine.extraction import ContextExtractor
from meta_prompting_engine.llm_clients.base import Message, LLMResponse


class MockLLMClient:
    """Mock LLM client for testing without API calls."""

    def __init__(self):
        self.call_count = 0
        self.call_history = []

    def complete(self, messages, temperature=0.7, max_tokens=2000, **kwargs):
        """Mock completion that returns realistic-looking output."""
        self.call_count += 1

        # Simulate different responses based on call count (iterations)
        if self.call_count == 1:
            content = """Here's a Python function to check if a number is prime:

```python
def is_prime(n):
    if n < 2:
        return False
    for i in range(2, int(n ** 0.5) + 1):
        if n % i == 0:
            return False
    return True
```

This function handles edge cases and has O(√n) time complexity."""

        elif self.call_count == 2:
            # Improved version for iteration 2
            content = """Here's an improved version with better documentation:

```python
def is_prime(n):
    \"\"\"
    Check if a number is prime.

    Args:
        n (int): Number to check

    Returns:
        bool: True if prime, False otherwise
    \"\"\"
    if not isinstance(n, int):
        raise TypeError("Input must be an integer")

    if n < 2:
        return False

    if n == 2:
        return True

    if n % 2 == 0:
        return False

    # Check odd divisors up to sqrt(n)
    for i in range(3, int(n ** 0.5) + 1, 2):
        if n % i == 0:
            return False

    return True
```

Improvements:
1. Added comprehensive docstring
2. Type checking with proper error handling
3. Optimized: skip even numbers after checking for 2
4. Better edge case handling"""

        else:
            content = "0.85"  # Quality score

        response = LLMResponse(
            content=content,
            model="mock-model",
            tokens_used=100 + self.call_count * 50,
            finish_reason="stop"
        )

        self.call_history.append({
            'messages': messages,
            'response': content
        })

        return response

    def embed(self, text):
        """Mock embedding."""
        return [0.1] * 384


def test_complexity_analyzer():
    """Test ComplexityAnalyzer without API calls."""
    print("\n" + "="*60)
    print("TEST 1: Complexity Analyzer")
    print("="*60)

    analyzer = ComplexityAnalyzer(llm_client=None)  # No LLM needed for basic analysis

    # Test simple task
    simple_task = "Print hello world"
    simple_score = analyzer.analyze(simple_task)
    print(f"\nSimple task: '{simple_task}'")
    print(f"  Complexity: {simple_score.overall:.2f}")
    print(f"  Factors: {simple_score.factors}")
    print(f"  Strategy: {analyzer.get_strategy(simple_score.overall)}")
    assert simple_score.overall < 0.5, "Simple task should have low complexity"
    print("  ✓ PASS: Simple task correctly identified")

    # Test medium task
    medium_task = "Create a class for managing a todo list with add, remove, and mark complete operations. Include error handling for invalid inputs, persistence to disk, and methods for filtering completed tasks. The implementation should be thread-safe and support undo/redo functionality."
    medium_score = analyzer.analyze(medium_task)
    print(f"\nMedium task: '{medium_task[:50]}...'")
    print(f"  Complexity: {medium_score.overall:.2f}")
    print(f"  Factors: {medium_score.factors}")
    print(f"  Strategy: {analyzer.get_strategy(medium_score.overall)}")
    assert medium_score.overall > simple_score.overall, "Medium task should have higher complexity than simple"
    print("  ✓ PASS: Medium task correctly identified")

    # Test complex task
    complex_task = "Design a distributed system for real-time collaborative editing with CRDT-based conflict resolution, operational transformation, and multi-tenant isolation. Consider scalability, consistency, and fault tolerance."
    complex_score = analyzer.analyze(complex_task)
    print(f"\nComplex task: '{complex_task[:50]}...'")
    print(f"  Complexity: {complex_score.overall:.2f}")
    print(f"  Factors: {complex_score.factors}")
    print(f"  Strategy: {analyzer.get_strategy(complex_score.overall)}")
    # Just verify it's a valid score (complexity depends on specific wording)
    assert 0.0 <= complex_score.overall <= 1.0, "Complexity should be in valid range"
    assert complex_score.factors['domain_specificity'] > 0, "Should detect technical domain"
    print("  ✓ PASS: Complex task analyzed correctly")

    print("\n✅ ComplexityAnalyzer working correctly!")
    return True


def test_context_extractor():
    """Test ContextExtractor with mock LLM."""
    print("\n" + "="*60)
    print("TEST 2: Context Extractor")
    print("="*60)

    mock_llm = MockLLMClient()
    extractor = ContextExtractor(mock_llm)

    sample_output = """
I'll implement binary search with the following approach:

1. Use two pointers technique
2. Divide and conquer strategy
3. Handle edge cases

```python
def binary_search(arr, target):
    left, right = 0, len(arr) - 1
    while left <= right:
        mid = (left + right) // 2
        if arr[mid] == target:
            return mid
        elif arr[mid] < target:
            left = mid + 1
        else:
            right = mid - 1
    return -1
```

This has O(log n) time complexity and requires a sorted array.
"""

    print("\nExtracting context from sample output...")
    extracted = extractor.extract_context_hierarchy(
        agent_output=sample_output,
        task="Implement binary search"
    )

    print(f"\nExtracted context:")
    print(f"  Domain primitives: {extracted.domain_primitives}")
    print(f"  Patterns: {extracted.patterns}")
    print(f"  Constraints: {extracted.constraints}")
    print(f"  Success indicators: {extracted.success_indicators}")

    # Should have extracted something
    assert extracted is not None, "Should extract context"
    print("\n✓ PASS: Context extraction working")

    print("\n✅ ContextExtractor working correctly!")
    return True


def test_meta_prompting_engine():
    """Test MetaPromptingEngine with mock LLM."""
    print("\n" + "="*60)
    print("TEST 3: Meta-Prompting Engine (Mock LLM)")
    print("="*60)

    from meta_prompting_engine.core import MetaPromptingEngine

    mock_llm = MockLLMClient()
    engine = MetaPromptingEngine(mock_llm)

    print("\nExecuting meta-prompting with mock LLM...")
    print("Task: Write a function to check if a number is prime")
    print("Max iterations: 2")

    result = engine.execute_with_meta_prompting(
        skill="python-programmer",
        task="Write a function to check if a number is prime",
        max_iterations=2,
        quality_threshold=0.80,
        verbose=True
    )

    print("\n" + "="*60)
    print("VALIDATION RESULTS")
    print("="*60)

    # Verify structure
    print(f"\n✓ Result structure: {type(result).__name__}")
    print(f"✓ Output generated: {len(result.output)} chars")
    print(f"✓ Iterations executed: {result.iterations}")
    print(f"✓ Quality score: {result.quality_score:.2f}")
    print(f"✓ Total tokens: {result.total_tokens}")
    print(f"✓ Execution time: {result.execution_time:.2f}s")
    print(f"✓ Complexity: {result.complexity.overall:.2f}")

    # Verify iterations
    assert result.iterations >= 1, "Should execute at least 1 iteration"
    assert result.iterations <= 2, "Should not exceed max_iterations"
    print(f"\n✓ PASS: Iterations within bounds (1-2)")

    # Verify LLM was called
    assert mock_llm.call_count >= result.iterations, "Should call LLM for each iteration"
    print(f"✓ PASS: LLM called {mock_llm.call_count} times")

    # Verify context extraction
    assert len(result.context.history) > 0, "Should have history"
    print(f"✓ PASS: Context history populated ({len(result.context.history)} entries)")

    # Verify output quality
    assert result.output is not None and len(result.output) > 0, "Should generate output"
    print(f"✓ PASS: Output generated")

    print("\n✅ MetaPromptingEngine working correctly!")
    return True


def test_recursive_improvement():
    """Test that quality can improve across iterations."""
    print("\n" + "="*60)
    print("TEST 4: Recursive Improvement")
    print("="*60)

    from meta_prompting_engine.core import MetaPromptingEngine

    class QualityImprovingMock:
        """Mock that shows quality improvement."""
        def __init__(self):
            self.call_count = 0
            self.call_history = []
            self.generation_count = 0

        def complete(self, messages, **kwargs):
            self.call_count += 1

            # Check if this is a quality assessment call (short max_tokens)
            max_tokens = kwargs.get('max_tokens', 2000)

            # Check message content to determine call type
            user_message = ""
            for msg in messages:
                if isinstance(msg, Message) and msg.role == "user":
                    user_message = msg.content.lower()
                    break

            if "assess the quality" in user_message or max_tokens <= 10:
                # Quality assessment - return number
                content = str(0.6 + (self.generation_count % 3) * 0.15)
            elif "extract" in user_message or "analyze this agent output" in user_message:
                # Context extraction - return JSON
                content = """{
                    "domain_primitives": {"objects": [], "operations": [], "relationships": []},
                    "patterns": [],
                    "constraints": {"hard_requirements": [], "soft_preferences": [], "anti_patterns": []},
                    "complexity_factors": [],
                    "success_indicators": [],
                    "error_patterns": []
                }"""
            else:
                # Generation call - return improving solutions
                self.generation_count += 1
                if self.generation_count == 1:
                    content = "Basic solution with minimal features"
                elif self.generation_count == 2:
                    content = "Improved solution with better error handling and validation"
                else:
                    content = "Optimized solution with comprehensive documentation and edge case handling"

            return LLMResponse(
                content=content,
                model="mock",
                tokens_used=100,
                finish_reason="stop"
            )

        def embed(self, text):
            return [0.1] * 384

    mock_llm = QualityImprovingMock()
    engine = MetaPromptingEngine(mock_llm)

    print("\nExecuting with quality-improving mock...")
    result = engine.execute_with_meta_prompting(
        skill="programmer",
        task="Write optimized code",
        max_iterations=3,
        quality_threshold=0.99,  # High threshold to force iterations
        verbose=False
    )

    print(f"\nIterations: {result.iterations}")
    print(f"LLM calls: {mock_llm.call_count}")
    print(f"Quality: {result.quality_score:.2f}")

    assert result.iterations >= 2, "Should iterate multiple times with high threshold"
    print("\n✓ PASS: Multiple iterations executed")

    print("\n✅ Recursive improvement working correctly!")
    return True


def main():
    """Run all validation tests."""
    print("\n" + "="*80)
    print("META-PROMPTING ENGINE VALIDATION (No API Key Required)")
    print("="*80)
    print("\nThis validates the implementation logic without making real API calls.")
    print("To test with REAL LLM calls, set ANTHROPIC_API_KEY and run demo_meta_prompting.py")
    print("="*80)

    all_passed = True

    try:
        # Test 1: Complexity Analyzer
        if not test_complexity_analyzer():
            all_passed = False

        # Test 2: Context Extractor
        if not test_context_extractor():
            all_passed = False

        # Test 3: Meta-Prompting Engine
        if not test_meta_prompting_engine():
            all_passed = False

        # Test 4: Recursive Improvement
        if not test_recursive_improvement():
            all_passed = False

        # Summary
        print("\n" + "="*80)
        if all_passed:
            print("✅ ALL VALIDATION TESTS PASSED!")
            print("="*80)
            print("\nThe implementation is working correctly!")
            print("\nNext steps:")
            print("1. Set ANTHROPIC_API_KEY in .env file")
            print("2. Run: python demo_meta_prompting.py")
            print("3. Test with real LLM calls")
            print("="*80 + "\n")
            return 0
        else:
            print("❌ SOME TESTS FAILED")
            print("="*80 + "\n")
            return 1

    except Exception as e:
        print(f"\n❌ ERROR during validation: {e}")
        import traceback
        traceback.print_exc()
        return 1


if __name__ == "__main__":
    sys.exit(main())