fix: skip uncorrectable RAPL counter wraps

codecarbon/core/cpu.py

@@ -489,7 +489,7 @@ def _scan_entry_subdirs(self, entry_path: str) -> list:
         """Scan subdirectories of a RAPL entry for domain directories."""
         subdirs = []
         try:
-            for sub in os.listdir(entry_path):
+            for sub in sorted(os.listdir(entry_path)):
                 sub_path = os.path.join(entry_path, sub)
                 if ":" in sub and os.path.isdir(sub_path):
                     if os.path.exists(os.path.join(sub_path, "energy_uj")):
@@ -508,7 +508,7 @@ def _scan_base_entries(self, base: str) -> list:
         """Scan a base directory for intel-rapl entries."""
         domain_dirs = []
         try:
-            for entry in os.listdir(base):
+            for entry in sorted(os.listdir(base)):
                 if not entry.startswith("intel-rapl"):
                     continue
                 entry_path = os.path.join(base, entry)
@@ -537,7 +537,7 @@ def _fallback_collect_domains(self, domain_dirs: list) -> list:
         if domain_dirs:
             return domain_dirs
         try:
-            for item in os.listdir(self._lin_rapl_dir):
+            for item in sorted(os.listdir(self._lin_rapl_dir)):
                 if ":" in item:
                     path = os.path.join(self._lin_rapl_dir, item)
                     if os.path.isdir(path) and os.path.exists(
@@ -893,6 +893,7 @@ def start(self) -> None:
             (rapl_file.path, float(rapl_file.last_energy))
             for rapl_file in self._rapl_files
             if "dram" not in rapl_file.name.lower()
+            and rapl_file.last_energy is not None
         ]
         self._mirrored_candidates = find_mirrored_counters(
             counters, SEQUENTIAL_READ_TOLERANCE_KWH

codecarbon/core/rapl.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass, field
+from typing import Optional
 
 from codecarbon.core.units import Energy, Power, Time
 from codecarbon.external.logger import logger
@@ -64,10 +65,11 @@ class RAPLFile:
     energy_delta: Energy = field(default_factory=lambda: Energy(0))
     # Power based on reading
     power: Power = field(default_factory=lambda: Power(0))
-    # Last energy reading in kWh
-    last_energy: Energy = field(default_factory=lambda: Energy(0))
+    # Last energy reading in kWh, None if it could not be read
+    last_energy: Optional[Energy] = field(default_factory=lambda: Energy(0))
     # Max value energy can hold before it wraps
     max_energy_reading: Energy = field(default_factory=lambda: Energy(0))
+    _warned_uncorrectable_wrap: bool = field(default=False, init=False)
 
     def __post_init__(self):
         self.last_energy = self._get_value()
@@ -92,9 +94,9 @@ def __post_init__(self):
                 )
             self.max_energy_reading = Energy.from_ujoules(0)
 
-    def _get_value(self) -> Energy:
+    def _get_value(self) -> Optional[Energy]:
         """
-        Reads the value in the file at the path
+        Reads the value in the file at the path, or None if it cannot be read.
         """
         try:
             with open(self.path, "r") as f:
@@ -110,21 +112,40 @@ def _get_value(self) -> Energy:
                 )
             else:
                 logger.debug("Unable to read RAPL value from %s: %s", self.path, e)
-            return Energy.from_ujoules(0)
+            return None
 
     def start(self) -> None:
         self.last_energy = self._get_value()
 
+    def _skip_sample(self, new_last_energy: Optional[Energy]) -> None:
+        self.energy_delta = Energy(0)
+        self.power = Power(0)
+        self.last_energy = new_last_energy
+
     def delta(self, duration: Time) -> None:
         """
         Compute the energy used since last call.
         """
         new_last_energy = energy = self._get_value()
+        if energy is None or self.last_energy is None:
+            # no baseline to compute a delta from: re-baseline silently
+            self._skip_sample(new_last_energy)
+            return
         if self.last_energy > energy:
             logger.debug(
                 f"In RAPLFile : Current energy value ({energy}) is lower than previous value ({self.last_energy}). Assuming wrap-around! Source file : {self.path}"
             )
             energy = energy + self.max_energy_reading
+        if self.last_energy > energy:
+            # still backwards after correction: skip rather than report a negative delta
+            if not self._warned_uncorrectable_wrap:
+                self._warned_uncorrectable_wrap = True
+                logger.warning(
+                    "In RAPLFile : counter went backwards and cannot be corrected for %s; skipping this sample (warned once).",
+                    self.path,
+                )
+            self._skip_sample(new_last_energy)
+            return
         self.power = self.power.from_energies_and_delay(
             energy, self.last_energy, duration
         )

tests/test_rapl_mmio_scanning.py

@@ -76,6 +76,20 @@ def test_rapl_start_keeps_dram_when_it_matches_a_package_counter(tmp_path, monke
     assert "dram" in details
 
 
+def test_rapl_start_skips_unreadable_counters(tmp_path, monkeypatch):
+    monkeypatch.setattr(sys, "platform", "linux")
+    energy_files = _make_rapl_tree(
+        tmp_path, [("package-0", 1000000), ("package-1", 1000000)]
+    )
+
+    rapl = IntelRAPL(rapl_dir=str(tmp_path))
+    energy_files[1].write_text("unreadable")
+    rapl.start()
+
+    assert rapl._rapl_files[1].last_energy is None
+    assert len(rapl._mirrored_candidates) == 0
+
+
 def test_rapl_start_reevaluates_mirror_candidates(tmp_path, monkeypatch):
     """A new start() re-runs the detection instead of compounding drops."""
     monkeypatch.setattr(sys, "platform", "linux")

tests/test_rapl_permissions.py

@@ -110,3 +110,130 @@ def emit(self, record):
             os.chmod(energy1, stat.S_IMODE(mode_before) or 0o644)
         except Exception:
             pass
+
+
+def test_rapl_wraparound_without_max_skips_sample(tmp_path):
+    """A wrap-around with an unknown max range must not yield a negative delta."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Time
+    from codecarbon.external.logger import logger as codecarbon_logger
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+
+    log_records = []
+
+    class TestHandler(logging.Handler):
+        def emit(self, record):
+            log_records.append(record)
+
+    test_handler = TestHandler()
+    test_handler.setLevel(logging.WARNING)
+    codecarbon_logger.addHandler(test_handler)
+
+    try:
+        rapl_file = RAPLFile(
+            name="package-0",
+            path=str(energy_file),
+            max_path=str(tmp_path / "does_not_exist"),
+        )
+        rapl_file.start()
+        energy_file.write_text("10000")
+        rapl_file.delta(Time.from_seconds(10))
+
+        assert rapl_file.energy_delta.kWh == 0
+        assert rapl_file.power.W == 0
+        assert any(
+            "counter went backwards" in r.getMessage() for r in log_records
+        ), f"Expected warning, got: {[r.getMessage() for r in log_records]}"
+    finally:
+        codecarbon_logger.removeHandler(test_handler)
+
+
+def test_rapl_wraparound_with_max_is_corrected(tmp_path):
+    """A wrap-around with a known max range is still corrected."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Energy, Time
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+    max_file = tmp_path / "max_energy_range_uj"
+    max_file.write_text("4294967295")
+
+    rapl_file = RAPLFile(
+        name="package-0", path=str(energy_file), max_path=str(max_file)
+    )
+    rapl_file.start()
+    energy_file.write_text("10000")
+    rapl_file.delta(Time.from_seconds(10))
+
+    expected = Energy.from_ujoules(10000 + 4294967295 - 4000000000).kWh
+    assert rapl_file.energy_delta.kWh == pytest.approx(expected)
+
+
+def test_rapl_read_error_does_not_inject_energy(tmp_path):
+    """A transient read error must not be mistaken for a wrap-around."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Energy, Time
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+    max_file = tmp_path / "max_energy_range_uj"
+    max_file.write_text("4294967295")
+
+    rapl_file = RAPLFile(
+        name="package-0", path=str(energy_file), max_path=str(max_file)
+    )
+    rapl_file.start()
+
+    # Unreadable content : previously this returned 0 and looked like a wrap.
+    energy_file.write_text("")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == 0
+    assert rapl_file.power.W == 0
+
+    # The next sample re-baselines instead of counting the gap twice.
+    energy_file.write_text("4000010000")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == 0
+
+    energy_file.write_text("4000020000")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == pytest.approx(Energy.from_ujoules(10000).kWh)
+
+
+def test_rapl_uncorrectable_wrap_warns_once(tmp_path):
+    """The uncorrectable-wrap warning must not repeat on every cycle."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Time
+    from codecarbon.external.logger import logger as codecarbon_logger
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+
+    log_records = []
+
+    class TestHandler(logging.Handler):
+        def emit(self, record):
+            log_records.append(record)
+
+    test_handler = TestHandler()
+    test_handler.setLevel(logging.WARNING)
+    codecarbon_logger.addHandler(test_handler)
+    try:
+        rapl_file = RAPLFile(
+            name="package-0",
+            path=str(energy_file),
+            max_path=str(tmp_path / "does_not_exist"),
+        )
+        rapl_file.start()
+        for value in ("10000", "9000", "8000"):
+            energy_file.write_text(value)
+            rapl_file.delta(Time.from_seconds(10))
+
+        backwards = [
+            r for r in log_records if "counter went backwards" in r.getMessage()
+        ]
+        assert len(backwards) == 1, [r.getMessage() for r in log_records]
+    finally:
+        codecarbon_logger.removeHandler(test_handler)