biomind/test_memory_integrated_mmlu_100.py at master · 269652/biomind · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
"""
Test Memory-Integrated BIOMIND on 20 MMLU Abstract Algebra Questions
"""

from integrated_biomind_evaluation import IntegratedBIOMINDEvaluator
from real_dataset_loader import RealDatasetLoader
import json
import time

def test_memory_integrated_mmlu_20():
    print("[BRAIN] Testing Memory-Integrated BIOMIND on 20 MMLU Abstract Algebra Questions")
    print("=" * 80)

    # Initialize the actual memory-integrated evaluator
    evaluator = IntegratedBIOMINDEvaluator([
        'qwen_math_expert',
        'qwen_general_reasoner',
        'tiny_llama_planner',
        'tiny_llama_critic'
    ], use_real_models=True)

    # Load 20 abstract algebra questions from real MMLU dataset
    loader = RealDatasetLoader()
    test_questions = loader.load_mmlu_subject("abstract_algebra", max_samples=20)

    if not test_questions:
        print("[FAIL] Failed to load MMLU abstract algebra questions")
        return

    print(f"[DATA] Loaded {len(test_questions)} abstract algebra questions")

    results = []
    total_correct = 0
    total_time = 0

    print("\n[TEST] Evaluating 20 questions with memory integration...")
    print("-" * 70)

    for i, q in enumerate(test_questions):
        print(f"\n[MDN] Question {i+1}/20")
        print(f"   {q['question']}")

        # Convert answer index to letter (0=A, 1=B, 2=C, 3=D)
        expected_answer = chr(65 + q['answer'])  # 0 -> 'A', 1 -> 'B', etc.

        start_time = time.time()
        result = evaluator.evaluate_with_reflection(
            q['question'], q['choices'], expected_answer, q['subject']
        )
        end_time = time.time()

        processing_time = (end_time - start_time) * 1000
        total_time += processing_time

        is_correct = result['is_correct']
        total_correct += int(is_correct)

        print(f"   Expected: {expected_answer} | Predicted: {result['predicted_answer']}")
        print(f"   Correct: {'[OK]' if is_correct else '[FAIL]'}")
        print(f"   Specialist: {result['selected_specialist']}")
        print(f"   Routing: {result['routing_method']}")
        print(f"   Time: {processing_time:.1f}ms")

        results.append(result)

    accuracy = total_correct / len(test_questions) * 100
    avg_time = total_time / len(test_questions)

    print("\n[TARGET] FINAL RESULTS")
    print("=" * 40)
    print(f"[OK] Correct: {total_correct}/{len(test_questions)}")
    print(f"[CHART] Accuracy: {accuracy:.1f}%")
    print(f"?  Avg Time: {avg_time:.1f}ms per question")
    print("[BRAIN] Memory Integration: ACTIVE [OK]")
    print("[TARGET] Realistic AI behavior: No hardcoded knowledge")

    # Save detailed results
    timestamp = int(time.time())
    results_file = f'memory_integrated_mmlu_20_abstract_algebra_{timestamp}.json'

    with open(results_file, 'w') as f:
        json.dump({
            'test_type': 'memory_integrated_biomind_mmlu_100_abstract_algebra',
            'questions_tested': len(test_questions),
            'accuracy': accuracy,
            'average_time_ms': avg_time,
            'memory_integration': True,
            'results': results
        }, f, indent=2)

    print(f"? Detailed results saved to: {results_file}")
    print("\n? Memory-integrated MMLU 20-question evaluation complete!")

if __name__ == "__main__":
    test_memory_integrated_mmlu_20()