From f59ac223fdb0ba15e51521faf0e875585c363394 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Thu, 20 Aug 2026 08:13:16 +0200 Subject: [PATCH 1/3] fix: skip uncorrectable RAPL counter wraps When `max_energy_range_uj` is unreadable, `max_energy_reading` is 0, so the wrap-around branch still ran but corrected nothing, emitting a large negative energy delta straight into the CPU and total energy. Drop the sample instead, matching the EMI backend's behaviour. The same guard also covers driver resets and the transient read-error fallback. A failed read previously returned `Energy(0)`, which looked like a wrap-around and, with a readable `max_energy_range_uj`, injected a spurious positive delta of up to one counter range. `_get_value` now returns None, and the sample is skipped and re-baselined. The uncorrectable-wrap warning is emitted once per file instead of on every measurement cycle. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/rapl.py | 31 ++++++-- tests/test_rapl_permissions.py | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/codecarbon/core/rapl.py b/codecarbon/core/rapl.py index 5c742cf2b..fd9f4ebc3 100644 --- a/codecarbon/core/rapl.py +++ b/codecarbon/core/rapl.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import Optional from codecarbon.core.units import Energy, Power, Time from codecarbon.external.logger import logger @@ -16,10 +17,11 @@ class RAPLFile: energy_delta: Energy = field(default_factory=lambda: Energy(0)) # Power based on reading power: Power = field(default_factory=lambda: Power(0)) - # Last energy reading in kWh - last_energy: Energy = field(default_factory=lambda: Energy(0)) + # Last energy reading in kWh, None if it could not be read + last_energy: Optional[Energy] = field(default_factory=lambda: Energy(0)) # Max value energy can hold before it wraps max_energy_reading: Energy = field(default_factory=lambda: Energy(0)) + _warned_uncorrectable_wrap: bool = field(default=False, init=False) def __post_init__(self): self.last_energy = self._get_value() @@ -44,9 +46,9 @@ def __post_init__(self): ) self.max_energy_reading = Energy.from_ujoules(0) - def _get_value(self) -> Energy: + def _get_value(self) -> Optional[Energy]: """ - Reads the value in the file at the path + Reads the value in the file at the path, or None if it cannot be read. """ try: with open(self.path, "r") as f: @@ -62,21 +64,40 @@ def _get_value(self) -> Energy: ) else: logger.debug("Unable to read RAPL value from %s: %s", self.path, e) - return Energy.from_ujoules(0) + return None def start(self) -> None: self.last_energy = self._get_value() + def _skip_sample(self, new_last_energy: Optional[Energy]) -> None: + self.energy_delta = Energy(0) + self.power = Power(0) + self.last_energy = new_last_energy + def delta(self, duration: Time) -> None: """ Compute the energy used since last call. """ new_last_energy = energy = self._get_value() + if energy is None or self.last_energy is None: + # no baseline to compute a delta from: re-baseline silently + self._skip_sample(new_last_energy) + return if self.last_energy > energy: logger.debug( f"In RAPLFile : Current energy value ({energy}) is lower than previous value ({self.last_energy}). Assuming wrap-around! Source file : {self.path}" ) energy = energy + self.max_energy_reading + if self.last_energy > energy: + # still backwards after correction: skip rather than report a negative delta + if not self._warned_uncorrectable_wrap: + self._warned_uncorrectable_wrap = True + logger.warning( + "In RAPLFile : counter went backwards and cannot be corrected for %s; skipping this sample (warned once).", + self.path, + ) + self._skip_sample(new_last_energy) + return self.power = self.power.from_energies_and_delay( energy, self.last_energy, duration ) diff --git a/tests/test_rapl_permissions.py b/tests/test_rapl_permissions.py index 4c522435d..f83bdf4e7 100644 --- a/tests/test_rapl_permissions.py +++ b/tests/test_rapl_permissions.py @@ -110,3 +110,130 @@ def emit(self, record): os.chmod(energy1, stat.S_IMODE(mode_before) or 0o644) except Exception: pass + + +def test_rapl_wraparound_without_max_skips_sample(tmp_path): + """A wrap-around with an unknown max range must not yield a negative delta.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + assert any( + "counter went backwards" in r.getMessage() for r in log_records + ), f"Expected warning, got: {[r.getMessage() for r in log_records]}" + finally: + codecarbon_logger.removeHandler(test_handler) + + +def test_rapl_wraparound_with_max_is_corrected(tmp_path): + """A wrap-around with a known max range is still corrected.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + expected = Energy.from_ujoules(10000 + 4294967295 - 4000000000).kWh + assert rapl_file.energy_delta.kWh == pytest.approx(expected) + + +def test_rapl_read_error_does_not_inject_energy(tmp_path): + """A transient read error must not be mistaken for a wrap-around.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + + # Unreadable content : previously this returned 0 and looked like a wrap. + energy_file.write_text("") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + + # The next sample re-baselines instead of counting the gap twice. + energy_file.write_text("4000010000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + + energy_file.write_text("4000020000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == pytest.approx(Energy.from_ujoules(10000).kWh) + + +def test_rapl_uncorrectable_wrap_warns_once(tmp_path): + """The uncorrectable-wrap warning must not repeat on every cycle.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + for value in ("10000", "9000", "8000"): + energy_file.write_text(value) + rapl_file.delta(Time.from_seconds(10)) + + backwards = [ + r for r in log_records if "counter went backwards" in r.getMessage() + ] + assert len(backwards) == 1, [r.getMessage() for r in log_records] + finally: + codecarbon_logger.removeHandler(test_handler) From b5aaca1ed7cbf41805ab1e2dc2d771b95819f32b Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 23 Sep 2026 16:40:13 +0900 Subject: [PATCH 2/3] fix: skip unreadable RAPL counters in mirror detection at start() Co-Authored-By: Claude Opus 5.5 (1M context) --- codecarbon/core/cpu.py | 1 + tests/test_rapl_mmio_scanning.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 4ad8d839e..d9db5c465 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -893,6 +893,7 @@ def start(self) -> None: (rapl_file.path, float(rapl_file.last_energy)) for rapl_file in self._rapl_files if "dram" not in rapl_file.name.lower() + and rapl_file.last_energy is not None ] self._mirrored_candidates = find_mirrored_counters( counters, SEQUENTIAL_READ_TOLERANCE_KWH diff --git a/tests/test_rapl_mmio_scanning.py b/tests/test_rapl_mmio_scanning.py index 5175e43d8..e1e15eba1 100644 --- a/tests/test_rapl_mmio_scanning.py +++ b/tests/test_rapl_mmio_scanning.py @@ -76,6 +76,20 @@ def test_rapl_start_keeps_dram_when_it_matches_a_package_counter(tmp_path, monke assert "dram" in details +def test_rapl_start_skips_unreadable_counters(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + energy_files = _make_rapl_tree( + tmp_path, [("package-0", 1000000), ("package-1", 1000000)] + ) + + rapl = IntelRAPL(rapl_dir=str(tmp_path)) + energy_files[1].write_text("unreadable") + rapl.start() + + assert rapl._rapl_files[1].last_energy is None + assert len(rapl._mirrored_candidates) == 0 + + def test_rapl_start_reevaluates_mirror_candidates(tmp_path, monkeypatch): """A new start() re-runs the detection instead of compounding drops.""" monkeypatch.setattr(sys, "platform", "linux") From 7848bf8a2f9af2ee1ef4e2f7ea6d544620ae2a07 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Sun, 27 Sep 2026 16:12:54 +0900 Subject: [PATCH 3/3] fix: list RAPL domains in sorted order so indexes are stable --- codecarbon/core/cpu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index d9db5c465..98e46b4f6 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -489,7 +489,7 @@ def _scan_entry_subdirs(self, entry_path: str) -> list: """Scan subdirectories of a RAPL entry for domain directories.""" subdirs = [] try: - for sub in os.listdir(entry_path): + for sub in sorted(os.listdir(entry_path)): sub_path = os.path.join(entry_path, sub) if ":" in sub and os.path.isdir(sub_path): if os.path.exists(os.path.join(sub_path, "energy_uj")): @@ -508,7 +508,7 @@ def _scan_base_entries(self, base: str) -> list: """Scan a base directory for intel-rapl entries.""" domain_dirs = [] try: - for entry in os.listdir(base): + for entry in sorted(os.listdir(base)): if not entry.startswith("intel-rapl"): continue entry_path = os.path.join(base, entry) @@ -537,7 +537,7 @@ def _fallback_collect_domains(self, domain_dirs: list) -> list: if domain_dirs: return domain_dirs try: - for item in os.listdir(self._lin_rapl_dir): + for item in sorted(os.listdir(self._lin_rapl_dir)): if ":" in item: path = os.path.join(self._lin_rapl_dir, item) if os.path.isdir(path) and os.path.exists(