Add RFC compliance metrics with grading system

- Add ComplianceMetrics class with compliance score calculation
- Add compliance grade system (A-F) with configurable thresholds
- Add severity level classification for test failures
- Add severity-weighted compliance scoring
- Add RFC section compliance breakdown by RFC 4271 sections
- Add generate_compliance_report() method
- Update test report to show compliance summary, grades, and RFC compliance
- Add comprehensive tests for all compliance metrics functionality
- All 118 tests pass

Author: daedalus

src/bgp_test_framework/runner.py

@@ -99,6 +99,162 @@ def get_log(self) -> List[Dict[str, Any]]:
         return self.entries
 
 
+class ComplianceMetrics:
+    COMPLIANCE_GRADE_THRESHOLDS = {
+        "A": 95.0,
+        "B": 85.0,
+        "C": 70.0,
+        "D": 50.0,
+        "F": 0.0,
+    }
+
+    SEVERITY_LEVELS = {
+        "CRITICAL": 10,
+        "HIGH": 7,
+        "MEDIUM": 4,
+        "LOW": 2,
+        "INFO": 1,
+    }
+
+    @staticmethod
+    def calculate_compliance_score(total: int, passed: int) -> float:
+        if total == 0:
+            return 0.0
+        return (passed / total) * 100.0
+
+    @classmethod
+    def get_compliance_grade(cls, score: float) -> str:
+        if score >= cls.COMPLIANCE_GRADE_THRESHOLDS["A"]:
+            return "A"
+        elif score >= cls.COMPLIANCE_GRADE_THRESHOLDS["B"]:
+            return "B"
+        elif score >= cls.COMPLIANCE_GRADE_THRESHOLDS["C"]:
+            return "C"
+        elif score >= cls.COMPLIANCE_GRADE_THRESHOLDS["D"]:
+            return "D"
+        return "F"
+
+    @staticmethod
+    def get_severity_level(test_id: str) -> str:
+        if test_id.startswith("MH-"):
+            return "CRITICAL"
+        elif test_id.startswith("OM-"):
+            return "HIGH"
+        elif test_id.startswith("UM-"):
+            return "HIGH"
+        elif test_id.startswith("AT-"):
+            return "MEDIUM"
+        elif test_id.startswith("FSM-"):
+            return "HIGH"
+        elif test_id.startswith("TM-"):
+            return "MEDIUM"
+        elif test_id.startswith("SEC-"):
+            return "CRITICAL"
+        elif test_id.startswith("RA-"):
+            return "MEDIUM"
+        elif test_id.startswith("DEC-"):
+            return "LOW"
+        return "INFO"
+
+    @staticmethod
+    def calculate_severity_score(failed_tests: List[TestResult]) -> Dict[str, int]:
+        severity_counts: Dict[str, int] = {
+            level: 0 for level in ComplianceMetrics.SEVERITY_LEVELS
+        }
+        for result in failed_tests:
+            severity = ComplianceMetrics.get_severity_level(result.test_id)
+            severity_counts[severity] += 1
+        return severity_counts
+
+    @staticmethod
+    def calculate_severity_weighted_score(
+        total: int, passed: int, failed_tests: List[TestResult]
+    ) -> float:
+        if total == 0:
+            return 0.0
+
+        max_score = sum(
+            ComplianceMetrics.SEVERITY_LEVELS[
+                ComplianceMetrics.get_severity_level(r.test_id)
+            ]
+            for r in failed_tests
+        ) + (ComplianceMetrics.SEVERITY_LEVELS["INFO"] * passed if passed > 0 else 0)
+
+        failed_score = sum(
+            ComplianceMetrics.SEVERITY_LEVELS[
+                ComplianceMetrics.get_severity_level(r.test_id)
+            ]
+            for r in failed_tests
+        )
+
+        if max_score == 0:
+            return 100.0
+
+        return ((max_score - failed_score) / max_score) * 100.0
+
+    @staticmethod
+    def get_rfc_section_compliance(
+        results: List[TestResult],
+    ) -> Dict[str, Dict[str, Any]]:
+        rfc_sections = {
+            "RFC 4271 Section 4.1": "message_header",
+            "RFC 4271 Section 4.2": "open_message",
+            "RFC 4271 Section 4.3": "update_message",
+            "RFC 4271 Section 5": "attribute",
+            "RFC 4271 Section 8": "fsm",
+            "RFC 4271 Section 4.4": "timing",
+            "RFC 4271 Section 6": "security",
+            "RFC 4271 Section 9.2": "route_aggregation",
+            "RFC 4271 Section 9.1": "decision_process",
+        }
+
+        compliance: Dict[str, Dict[str, Any]] = {}
+        for section, category in rfc_sections.items():
+            section_results = [r for r in results if r.category.value == category]
+            total = len(section_results)
+            passed_count = sum(1 for r in section_results if r.passed)
+            score = ComplianceMetrics.calculate_compliance_score(total, passed_count)
+
+            compliance[section] = {
+                "total": total,
+                "passed": passed_count,
+                "failed": total - passed_count,
+                "score": score,
+                "grade": ComplianceMetrics.get_compliance_grade(score),
+            }
+
+        return compliance
+
+    @staticmethod
+    def generate_compliance_report(results: List[TestResult]) -> Dict[str, Any]:
+        total = len(results)
+        passed = sum(1 for r in results if r.passed)
+        failed = total - passed
+        score = ComplianceMetrics.calculate_compliance_score(total, passed)
+        failed_tests = [r for r in results if not r.passed]
+
+        return {
+            "compliance_score": round(score, 2),
+            "compliance_grade": ComplianceMetrics.get_compliance_grade(score),
+            "total_tests": total,
+            "tests_passed": passed,
+            "tests_failed": failed,
+            "pass_rate": f"{score:.1f}%",
+            "severity_distribution": ComplianceMetrics.calculate_severity_score(
+                failed_tests
+            ),
+            "weighted_score": round(
+                ComplianceMetrics.calculate_severity_weighted_score(
+                    total, passed, failed_tests
+                ),
+                2,
+            ),
+            "rfc_section_compliance": ComplianceMetrics.get_rfc_section_compliance(
+                results
+            ),
+        }
+
+
 class TestRunner:
     def __init__(self, config: TestConfiguration):
         self.config = config
@@ -722,6 +878,8 @@ def get_summary(self) -> Dict[str, Any]:
             else:
                 by_category[cat]["failed"] += 1
 
+        compliance_report = ComplianceMetrics.generate_compliance_report(self.results)
+
         return {
             "total": total,
             "passed": passed,
@@ -730,25 +888,51 @@ def get_summary(self) -> Dict[str, Any]:
             "by_category": by_category,
             "target": f"{self.config.target_host}:{self.config.target_port}",
             "source_as": self.config.source_as,
+            "compliance": compliance_report,
         }
 
     def generate_report(self) -> str:
         summary = self.get_summary()
+        compliance = summary.get("compliance", {})
         lines = [
             "=" * 80,
-            "BGPv4 Adversarial Test Report",
+            "BGPv4 RFC Compliance Test Report",
             "=" * 80,
             f"Target: {summary['target']}",
             f"Source AS: {summary['source_as']}",
             "-" * 80,
+            "Compliance Summary:",
+            f"  Compliance Score: {compliance.get('compliance_score', 0):.2f}%",
+            f"  Compliance Grade: {compliance.get('compliance_grade', 'N/A')}",
+            f"  Weighted Score: {compliance.get('weighted_score', 0):.2f}%",
+            "-" * 80,
             f"Total Tests: {summary['total']}",
             f"Passed: {summary['passed']}",
             f"Failed: {summary['failed']}",
             f"Pass Rate: {summary['pass_rate']}",
-            "-" * 80,
-            "Results by Category:",
         ]
 
+        severity = compliance.get("severity_distribution", {})
+        if severity:
+            lines.append("-" * 80)
+            lines.append("Severity Distribution of Failures:")
+            for level in ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"]:
+                count = severity.get(level, 0)
+                if count > 0:
+                    lines.append(f"  {level}: {count}")
+
+        lines.append("-" * 80)
+        lines.append("RFC Section Compliance:")
+        rfc_compliance = compliance.get("rfc_section_compliance", {})
+        for section, stats in rfc_compliance.items():
+            if stats["total"] > 0:
+                lines.append(
+                    f"  {section}: {stats['score']:.1f}% (Grade: {stats['grade']}) "
+                    f"[{stats['passed']}/{stats['total']}]"
+                )
+
+        lines.append("-" * 80)
+        lines.append("Results by Category:")
         for cat, stats in summary["by_category"].items():
             lines.append(f"  {cat}: {stats['passed']}/{stats['total']} passed")
 

tests/functional/test_runner.py

@@ -4,7 +4,12 @@
 
 from unittest.mock import Mock, patch
 
-from bgp_test_framework.runner import TestRunner, TestConfiguration, BGPLogger
+from bgp_test_framework.runner import (
+    TestRunner,
+    TestConfiguration,
+    BGPLogger,
+    ComplianceMetrics,
+)
 from bgp_test_framework.tests import TestCategory, TestResult
 
 
@@ -230,7 +235,9 @@ def test_generate_report(self):
             ),
         ]
         report = runner.generate_report()
-        assert "BGPv4 Adversarial Test Report" in report
+        assert "BGPv4 RFC Compliance Test Report" in report
+        assert "Compliance Score:" in report
+        assert "Compliance Grade:" in report
         assert "MH-001" in report
         assert "PASS" in report
 
@@ -302,3 +309,134 @@ def test_test_filtering_by_id(self):
         filtered = [(tid, cat) for tid, cat in all_tests if tid in target_ids]
         assert len(filtered) == 2
         assert all(tid in target_ids for tid, _ in filtered)
+
+
+class TestComplianceMetrics:
+    def test_calculate_compliance_score(self):
+        assert ComplianceMetrics.calculate_compliance_score(100, 95) == 95.0
+        assert ComplianceMetrics.calculate_compliance_score(10, 10) == 100.0
+        assert ComplianceMetrics.calculate_compliance_score(10, 0) == 0.0
+        assert ComplianceMetrics.calculate_compliance_score(0, 0) == 0.0
+
+    def test_get_compliance_grade(self):
+        assert ComplianceMetrics.get_compliance_grade(100) == "A"
+        assert ComplianceMetrics.get_compliance_grade(95) == "A"
+        assert ComplianceMetrics.get_compliance_grade(94) == "B"
+        assert ComplianceMetrics.get_compliance_grade(85) == "B"
+        assert ComplianceMetrics.get_compliance_grade(84) == "C"
+        assert ComplianceMetrics.get_compliance_grade(70) == "C"
+        assert ComplianceMetrics.get_compliance_grade(69) == "D"
+        assert ComplianceMetrics.get_compliance_grade(50) == "D"
+        assert ComplianceMetrics.get_compliance_grade(49) == "F"
+        assert ComplianceMetrics.get_compliance_grade(0) == "F"
+
+    def test_get_severity_level(self):
+        assert ComplianceMetrics.get_severity_level("MH-001") == "CRITICAL"
+        assert ComplianceMetrics.get_severity_level("OM-001") == "HIGH"
+        assert ComplianceMetrics.get_severity_level("UM-001") == "HIGH"
+        assert ComplianceMetrics.get_severity_level("AT-001") == "MEDIUM"
+        assert ComplianceMetrics.get_severity_level("FSM-001") == "HIGH"
+        assert ComplianceMetrics.get_severity_level("TM-001") == "MEDIUM"
+        assert ComplianceMetrics.get_severity_level("SEC-001") == "CRITICAL"
+        assert ComplianceMetrics.get_severity_level("RA-001") == "MEDIUM"
+        assert ComplianceMetrics.get_severity_level("DEC-001") == "LOW"
+        assert ComplianceMetrics.get_severity_level("OTHER-001") == "INFO"
+
+    def test_calculate_severity_score(self):
+        failed = [
+            TestResult(
+                "MH-001",
+                "Test 1",
+                TestCategory.MESSAGE_HEADER,
+                False,
+                "Expected",
+                "Actual",
+            ),
+            TestResult(
+                "SEC-001",
+                "Test 2",
+                TestCategory.SECURITY,
+                False,
+                "Expected",
+                "Actual",
+            ),
+        ]
+        severity = ComplianceMetrics.calculate_severity_score(failed)
+        assert severity["CRITICAL"] == 2
+        assert severity["HIGH"] == 0
+        assert severity["MEDIUM"] == 0
+
+    def test_calculate_severity_weighted_score(self):
+        failed = [
+            TestResult(
+                "MH-001",
+                "Test 1",
+                TestCategory.MESSAGE_HEADER,
+                False,
+                "Expected",
+                "Actual",
+            ),
+        ]
+        score = ComplianceMetrics.calculate_severity_weighted_score(10, 9, failed)
+        assert score > 0 and score <= 100
+
+    def test_get_rfc_section_compliance(self):
+        results = [
+            TestResult(
+                "MH-001",
+                "Test 1",
+                TestCategory.MESSAGE_HEADER,
+                True,
+                "Expected",
+                "Actual",
+            ),
+            TestResult(
+                "MH-002",
+                "Test 2",
+                TestCategory.MESSAGE_HEADER,
+                False,
+                "Expected",
+                "Actual",
+            ),
+        ]
+        compliance = ComplianceMetrics.get_rfc_section_compliance(results)
+        assert "RFC 4271 Section 4.1" in compliance
+        assert compliance["RFC 4271 Section 4.1"]["total"] == 2
+        assert compliance["RFC 4271 Section 4.1"]["passed"] == 1
+        assert compliance["RFC 4271 Section 4.1"]["failed"] == 1
+
+    def test_generate_compliance_report(self):
+        results = [
+            TestResult(
+                "MH-001",
+                "Test 1",
+                TestCategory.MESSAGE_HEADER,
+                True,
+                "Expected",
+                "Actual",
+            ),
+            TestResult(
+                "MH-002",
+                "Test 2",
+                TestCategory.MESSAGE_HEADER,
+                False,
+                "Expected",
+                "Actual",
+            ),
+            TestResult(
+                "OM-001",
+                "Test 3",
+                TestCategory.OPEN_MESSAGE,
+                True,
+                "Expected",
+                "Actual",
+            ),
+        ]
+        report = ComplianceMetrics.generate_compliance_report(results)
+        assert report["total_tests"] == 3
+        assert report["tests_passed"] == 2
+        assert report["tests_failed"] == 1
+        assert report["compliance_score"] > 0
+        assert "compliance_grade" in report
+        assert "severity_distribution" in report
+        assert "rfc_section_compliance" in report