diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 4ad8d839e..98e46b4f6 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -489,7 +489,7 @@ def _scan_entry_subdirs(self, entry_path: str) -> list: """Scan subdirectories of a RAPL entry for domain directories.""" subdirs = [] try: - for sub in os.listdir(entry_path): + for sub in sorted(os.listdir(entry_path)): sub_path = os.path.join(entry_path, sub) if ":" in sub and os.path.isdir(sub_path): if os.path.exists(os.path.join(sub_path, "energy_uj")): @@ -508,7 +508,7 @@ def _scan_base_entries(self, base: str) -> list: """Scan a base directory for intel-rapl entries.""" domain_dirs = [] try: - for entry in os.listdir(base): + for entry in sorted(os.listdir(base)): if not entry.startswith("intel-rapl"): continue entry_path = os.path.join(base, entry) @@ -537,7 +537,7 @@ def _fallback_collect_domains(self, domain_dirs: list) -> list: if domain_dirs: return domain_dirs try: - for item in os.listdir(self._lin_rapl_dir): + for item in sorted(os.listdir(self._lin_rapl_dir)): if ":" in item: path = os.path.join(self._lin_rapl_dir, item) if os.path.isdir(path) and os.path.exists( @@ -893,6 +893,7 @@ def start(self) -> None: (rapl_file.path, float(rapl_file.last_energy)) for rapl_file in self._rapl_files if "dram" not in rapl_file.name.lower() + and rapl_file.last_energy is not None ] self._mirrored_candidates = find_mirrored_counters( counters, SEQUENTIAL_READ_TOLERANCE_KWH diff --git a/codecarbon/core/rapl.py b/codecarbon/core/rapl.py index 46bd6e864..7014323a7 100644 --- a/codecarbon/core/rapl.py +++ b/codecarbon/core/rapl.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import Optional from codecarbon.core.units import Energy, Power, Time from codecarbon.external.logger import logger @@ -64,10 +65,11 @@ class RAPLFile: energy_delta: Energy = field(default_factory=lambda: Energy(0)) # Power based on reading power: Power = field(default_factory=lambda: Power(0)) - # Last energy reading in kWh - last_energy: Energy = field(default_factory=lambda: Energy(0)) + # Last energy reading in kWh, None if it could not be read + last_energy: Optional[Energy] = field(default_factory=lambda: Energy(0)) # Max value energy can hold before it wraps max_energy_reading: Energy = field(default_factory=lambda: Energy(0)) + _warned_uncorrectable_wrap: bool = field(default=False, init=False) def __post_init__(self): self.last_energy = self._get_value() @@ -92,9 +94,9 @@ def __post_init__(self): ) self.max_energy_reading = Energy.from_ujoules(0) - def _get_value(self) -> Energy: + def _get_value(self) -> Optional[Energy]: """ - Reads the value in the file at the path + Reads the value in the file at the path, or None if it cannot be read. """ try: with open(self.path, "r") as f: @@ -110,21 +112,40 @@ def _get_value(self) -> Energy: ) else: logger.debug("Unable to read RAPL value from %s: %s", self.path, e) - return Energy.from_ujoules(0) + return None def start(self) -> None: self.last_energy = self._get_value() + def _skip_sample(self, new_last_energy: Optional[Energy]) -> None: + self.energy_delta = Energy(0) + self.power = Power(0) + self.last_energy = new_last_energy + def delta(self, duration: Time) -> None: """ Compute the energy used since last call. """ new_last_energy = energy = self._get_value() + if energy is None or self.last_energy is None: + # no baseline to compute a delta from: re-baseline silently + self._skip_sample(new_last_energy) + return if self.last_energy > energy: logger.debug( f"In RAPLFile : Current energy value ({energy}) is lower than previous value ({self.last_energy}). Assuming wrap-around! Source file : {self.path}" ) energy = energy + self.max_energy_reading + if self.last_energy > energy: + # still backwards after correction: skip rather than report a negative delta + if not self._warned_uncorrectable_wrap: + self._warned_uncorrectable_wrap = True + logger.warning( + "In RAPLFile : counter went backwards and cannot be corrected for %s; skipping this sample (warned once).", + self.path, + ) + self._skip_sample(new_last_energy) + return self.power = self.power.from_energies_and_delay( energy, self.last_energy, duration ) diff --git a/tests/test_rapl_mmio_scanning.py b/tests/test_rapl_mmio_scanning.py index 5175e43d8..e1e15eba1 100644 --- a/tests/test_rapl_mmio_scanning.py +++ b/tests/test_rapl_mmio_scanning.py @@ -76,6 +76,20 @@ def test_rapl_start_keeps_dram_when_it_matches_a_package_counter(tmp_path, monke assert "dram" in details +def test_rapl_start_skips_unreadable_counters(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + energy_files = _make_rapl_tree( + tmp_path, [("package-0", 1000000), ("package-1", 1000000)] + ) + + rapl = IntelRAPL(rapl_dir=str(tmp_path)) + energy_files[1].write_text("unreadable") + rapl.start() + + assert rapl._rapl_files[1].last_energy is None + assert len(rapl._mirrored_candidates) == 0 + + def test_rapl_start_reevaluates_mirror_candidates(tmp_path, monkeypatch): """A new start() re-runs the detection instead of compounding drops.""" monkeypatch.setattr(sys, "platform", "linux") diff --git a/tests/test_rapl_permissions.py b/tests/test_rapl_permissions.py index 4c522435d..f83bdf4e7 100644 --- a/tests/test_rapl_permissions.py +++ b/tests/test_rapl_permissions.py @@ -110,3 +110,130 @@ def emit(self, record): os.chmod(energy1, stat.S_IMODE(mode_before) or 0o644) except Exception: pass + + +def test_rapl_wraparound_without_max_skips_sample(tmp_path): + """A wrap-around with an unknown max range must not yield a negative delta.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + assert any( + "counter went backwards" in r.getMessage() for r in log_records + ), f"Expected warning, got: {[r.getMessage() for r in log_records]}" + finally: + codecarbon_logger.removeHandler(test_handler) + + +def test_rapl_wraparound_with_max_is_corrected(tmp_path): + """A wrap-around with a known max range is still corrected.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + expected = Energy.from_ujoules(10000 + 4294967295 - 4000000000).kWh + assert rapl_file.energy_delta.kWh == pytest.approx(expected) + + +def test_rapl_read_error_does_not_inject_energy(tmp_path): + """A transient read error must not be mistaken for a wrap-around.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + + # Unreadable content : previously this returned 0 and looked like a wrap. + energy_file.write_text("") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + + # The next sample re-baselines instead of counting the gap twice. + energy_file.write_text("4000010000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + + energy_file.write_text("4000020000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == pytest.approx(Energy.from_ujoules(10000).kWh) + + +def test_rapl_uncorrectable_wrap_warns_once(tmp_path): + """The uncorrectable-wrap warning must not repeat on every cycle.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + for value in ("10000", "9000", "8000"): + energy_file.write_text(value) + rapl_file.delta(Time.from_seconds(10)) + + backwards = [ + r for r in log_records if "counter went backwards" in r.getMessage() + ] + assert len(backwards) == 1, [r.getMessage() for r in log_records] + finally: + codecarbon_logger.removeHandler(test_handler)