diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 4bd0ea99d..9f980a663 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -554,59 +554,66 @@ def __init__( if self._configure_multiple_runs(allow_multiple_runs): return - self._set_from_conf(api_call_interval, "api_call_interval", 8, int) - self._set_from_conf(api_endpoint, "api_endpoint", "https://api.codecarbon.io") - self._set_from_conf(api_key, "api_key", "api_key") - self._configure_electricitymaps_token( - electricitymaps_api_token, co2_signal_api_token - ) - - self._set_from_conf(emissions_endpoint, "emissions_endpoint") - self._set_from_conf(experiment_name, "experiment_name", "base") - self._set_from_conf(gpu_ids, "gpu_ids") - self._set_from_conf(log_level, "log_level", "info") - self._set_from_conf(measure_power_secs, "measure_power_secs", 15, float) - self._set_from_conf(output_dir, "output_dir", ".") - self._set_from_conf(output_file, "output_file", "emissions.csv") - self._set_from_conf(project_name, "project_name", "codecarbon") - self._resolve_output_methods( - output_methods, - save_to_file, - save_to_api, - save_to_logger, - save_to_prometheus, - save_to_logfire, - ) + try: + self._set_from_conf(api_call_interval, "api_call_interval", 8, int) + self._set_from_conf( + api_endpoint, "api_endpoint", "https://api.codecarbon.io" + ) + self._set_from_conf(api_key, "api_key", "api_key") + self._configure_electricitymaps_token( + electricitymaps_api_token, co2_signal_api_token + ) - self._set_from_conf(logging_logger, "logging_logger") - self._set_from_conf(prometheus_url, "prometheus_url", "localhost:9091") - self._set_from_conf(output_handlers, "output_handlers", []) - self._set_from_conf(tracking_mode, "tracking_mode", "machine") - self._set_from_conf(on_csv_write, "on_csv_write", "append") - self._set_from_conf(logger_preamble, "logger_preamble", "") - self._set_from_conf(force_cpu_power, "force_cpu_power", None, float) - self._set_from_conf(force_ram_power, "force_ram_power", None, float) - self._set_from_conf(pue, "pue", 1.0, float) - self._set_from_conf(wue, "wue", 0, float) - self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False, bool) - self._set_from_conf(rapl_include_dram, "rapl_include_dram", False, bool) - self._set_from_conf(rapl_prefer_psys, "rapl_prefer_psys", False, bool) - self._set_from_conf( - experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" - ) + self._set_from_conf(emissions_endpoint, "emissions_endpoint") + self._set_from_conf(experiment_name, "experiment_name", "base") + self._set_from_conf(gpu_ids, "gpu_ids") + self._set_from_conf(log_level, "log_level", "info") + self._set_from_conf(measure_power_secs, "measure_power_secs", 15, float) + self._set_from_conf(output_dir, "output_dir", ".") + self._set_from_conf(output_file, "output_file", "emissions.csv") + self._set_from_conf(project_name, "project_name", "codecarbon") + self._resolve_output_methods( + output_methods, + save_to_file, + save_to_api, + save_to_logger, + save_to_prometheus, + save_to_logfire, + ) - if self.force_carbon_intensity_g_co2e_kwh is not None: - logger.info( - f"Using forced carbon intensity: {self.force_carbon_intensity_g_co2e_kwh} gCO2e/kWh." + self._set_from_conf(logging_logger, "logging_logger") + self._set_from_conf(prometheus_url, "prometheus_url", "localhost:9091") + self._set_from_conf(output_handlers, "output_handlers", []) + self._set_from_conf(tracking_mode, "tracking_mode", "machine") + self._set_from_conf(on_csv_write, "on_csv_write", "append") + self._set_from_conf(logger_preamble, "logger_preamble", "") + self._set_from_conf(force_cpu_power, "force_cpu_power", None, float) + self._set_from_conf(force_ram_power, "force_ram_power", None, float) + self._set_from_conf(pue, "pue", 1.0, float) + self._set_from_conf(wue, "wue", 0, float) + self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False, bool) + self._set_from_conf(rapl_include_dram, "rapl_include_dram", False, bool) + self._set_from_conf(rapl_prefer_psys, "rapl_prefer_psys", False, bool) + self._set_from_conf( + experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) - assert self._tracking_mode in ["machine", "process"] - set_logger_level(self._log_level) - set_logger_format(self._logger_preamble) - self._initialize_runtime_state() - self._initialize_scheduler_state() - self._initialize_emissions_context() - self._init_output_methods(api_key=self._api_key) + if self.force_carbon_intensity_g_co2e_kwh is not None: + logger.info( + f"Using forced carbon intensity: {self.force_carbon_intensity_g_co2e_kwh} gCO2e/kWh." + ) + + assert self._tracking_mode in ["machine", "process"] + set_logger_level(self._log_level) + set_logger_format(self._logger_preamble) + self._initialize_runtime_state() + self._initialize_scheduler_state() + self._initialize_emissions_context() + self._init_output_methods(api_key=self._api_key) + except Exception: + if getattr(self, "_lock", None) is not None: + self._lock.release() + raise def _init_output_methods(self, *, api_key: str = None): """ @@ -1339,7 +1346,6 @@ class OfflineEmissionsTracker(BaseEmissionsTracker): _country_iso_code = None _country_name, _region, country_2letter_iso_code = None, None, None - @suppress(Exception) def __init__( self, *args, @@ -1380,7 +1386,8 @@ def __init__( logger.info("offline tracker init") if self._region is not None: - assert isinstance(self._region, str) + if not isinstance(self._region, str): + raise TypeError(f"region must be a string, got {self._region!r}") self._region: str = self._region.lower() if self._cloud_provider: @@ -1390,7 +1397,11 @@ def __init__( ) if self._country_2letter_iso_code: - assert isinstance(self._country_2letter_iso_code, str) + if not isinstance(self._country_2letter_iso_code, str): + raise TypeError( + "country_2letter_iso_code must be a string, " + f"got {self._country_2letter_iso_code!r}" + ) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() super().__init__(*args, **kwargs) @@ -1650,75 +1661,88 @@ def wrapped_fn(*args, **kwargs): cloud_provider is None or cloud_provider is _sentinel ): raise Exception("Needs ISO Code of the Country for Offline mode") - tracker = OfflineEmissionsTracker( - project_name=project_name, - measure_power_secs=measure_power_secs, - output_dir=output_dir, - output_file=output_file, - output_methods=output_methods, - save_to_file=save_to_file, - save_to_logger=save_to_logger, - logging_logger=logging_logger, - save_to_prometheus=save_to_prometheus, - save_to_logfire=save_to_logfire, - prometheus_url=prometheus_url, - output_handlers=output_handlers, - gpu_ids=gpu_ids, - electricitymaps_api_token=_electricitymaps_token, - tracking_mode=tracking_mode, - log_level=log_level, - on_csv_write=on_csv_write, - logger_preamble=logger_preamble, - country_iso_code=country_iso_code, - region=region, - cloud_provider=cloud_provider, - cloud_region=cloud_region, - country_2letter_iso_code=country_2letter_iso_code, - force_cpu_power=force_cpu_power, - force_ram_power=force_ram_power, - pue=pue, - wue=wue, - force_carbon_intensity_g_co2e_kwh=force_carbon_intensity_g_co2e_kwh, - allow_multiple_runs=allow_multiple_runs, - rapl_include_dram=rapl_include_dram, - rapl_prefer_psys=rapl_prefer_psys, - ) - else: - tracker = EmissionsTracker( - project_name=project_name, - measure_power_secs=measure_power_secs, - api_call_interval=api_call_interval, - api_endpoint=api_endpoint, - api_key=api_key, - output_dir=output_dir, - output_file=output_file, - output_methods=output_methods, - save_to_file=save_to_file, - save_to_api=save_to_api, - save_to_logger=save_to_logger, - logging_logger=logging_logger, - save_to_prometheus=save_to_prometheus, - save_to_logfire=save_to_logfire, - prometheus_url=prometheus_url, - output_handlers=output_handlers, - gpu_ids=gpu_ids, - emissions_endpoint=emissions_endpoint, - experiment_id=experiment_id, - experiment_name=experiment_name, - electricitymaps_api_token=_electricitymaps_token, - tracking_mode=tracking_mode, - log_level=log_level, - on_csv_write=on_csv_write, - logger_preamble=logger_preamble, - force_cpu_power=force_cpu_power, - force_ram_power=force_ram_power, - pue=pue, - wue=wue, - force_carbon_intensity_g_co2e_kwh=force_carbon_intensity_g_co2e_kwh, - allow_multiple_runs=allow_multiple_runs, - rapl_include_dram=rapl_include_dram, - rapl_prefer_psys=rapl_prefer_psys, + try: + if offline and offline is not _sentinel: + tracker = OfflineEmissionsTracker( + project_name=project_name, + measure_power_secs=measure_power_secs, + output_dir=output_dir, + output_file=output_file, + output_methods=output_methods, + save_to_file=save_to_file, + save_to_logger=save_to_logger, + logging_logger=logging_logger, + save_to_prometheus=save_to_prometheus, + save_to_logfire=save_to_logfire, + prometheus_url=prometheus_url, + output_handlers=output_handlers, + gpu_ids=gpu_ids, + electricitymaps_api_token=_electricitymaps_token, + tracking_mode=tracking_mode, + log_level=log_level, + on_csv_write=on_csv_write, + logger_preamble=logger_preamble, + country_iso_code=country_iso_code, + region=region, + cloud_provider=cloud_provider, + cloud_region=cloud_region, + country_2letter_iso_code=country_2letter_iso_code, + force_cpu_power=force_cpu_power, + force_ram_power=force_ram_power, + pue=pue, + wue=wue, + force_carbon_intensity_g_co2e_kwh=force_carbon_intensity_g_co2e_kwh, + allow_multiple_runs=allow_multiple_runs, + rapl_include_dram=rapl_include_dram, + rapl_prefer_psys=rapl_prefer_psys, + ) + else: + tracker = EmissionsTracker( + project_name=project_name, + measure_power_secs=measure_power_secs, + api_call_interval=api_call_interval, + api_endpoint=api_endpoint, + api_key=api_key, + output_dir=output_dir, + output_file=output_file, + output_methods=output_methods, + save_to_file=save_to_file, + save_to_api=save_to_api, + save_to_logger=save_to_logger, + logging_logger=logging_logger, + save_to_prometheus=save_to_prometheus, + save_to_logfire=save_to_logfire, + prometheus_url=prometheus_url, + output_handlers=output_handlers, + gpu_ids=gpu_ids, + emissions_endpoint=emissions_endpoint, + experiment_id=experiment_id, + experiment_name=experiment_name, + electricitymaps_api_token=_electricitymaps_token, + tracking_mode=tracking_mode, + log_level=log_level, + on_csv_write=on_csv_write, + logger_preamble=logger_preamble, + force_cpu_power=force_cpu_power, + force_ram_power=force_ram_power, + pue=pue, + wue=wue, + force_carbon_intensity_g_co2e_kwh=force_carbon_intensity_g_co2e_kwh, + allow_multiple_runs=allow_multiple_runs, + rapl_include_dram=rapl_include_dram, + rapl_prefer_psys=rapl_prefer_psys, + ) + except Exception: + logger.error( + "Tracker not initialized, running the function untracked. " + "Please check the logs.", + exc_info=True, ) + tracker = None + # Called outside the except block so the construction error is not + # chained to exceptions raised by fn. + if tracker is None: + return fn(*args, **kwargs) tracker.start() try: fn_result = fn(*args, **kwargs) diff --git a/docs/reference/api.md b/docs/reference/api.md index 15117b123..f8561acb3 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -8,6 +8,13 @@ Parameters can be set via `EmissionsTracker()`, `OfflineEmissionsTracker()`, the PUE is a multiplication factor provided by the user. Old datacenters have PUE up to 2.2, new greener ones as low as 1.1. +!!! warning "Constructor errors (changed in v3.4.0)" + `OfflineEmissionsTracker(...)` used to swallow every exception raised while + building the tracker, returning a half-initialised object that silently + recorded nothing. It now propagates those errors, like `EmissionsTracker` + always did — for example an `output_dir` that does not exist raises instead of + logging. Code that relied on the constructor never raising needs a `try`/`except`. + !!! note "GPU selection" If you use `CUDA_VISIBLE_DEVICES` or `ROCR_VISIBLE_DEVICES` to set GPUs, CodeCarbon will automatically populate `gpu_ids`. Manual `gpu_ids` overrides this. diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index a2d6b5966..adcec47f2 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -1153,3 +1153,44 @@ def test_cumulative_emissions_with_varying_intensity( # Verification: If it wasn't cumulative, it would be 3.0 kWh * 300 g/kWh = 0.9 kg self.assertLess(data3.emissions, 0.8) + + +class TestConstructorFailureReleasesLock(unittest.TestCase): + """A constructor failure with allow_multiple_runs=False must release the + lock it acquired, so a subsequent tracker can still start.""" + + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.lockfile_path = Path(self.temp_dir.name) / ".codecarbon.lock" + patcher_lock = mock.patch("codecarbon.lock.LOCKFILE", str(self.lockfile_path)) + patcher_lock.start() + self.addCleanup(patcher_lock.stop) + + patcher_conf = mock.patch( + "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + ) + patcher_conf.start() + self.addCleanup(patcher_conf.stop) + + def test_constructor_failure_releases_lock(self): + # A bogus tracking_mode makes __init__ raise after the lock is + # acquired in _configure_multiple_runs(). + with self.assertRaises(AssertionError): + EmissionsTracker( + tracking_mode="not-a-real-mode", + allow_multiple_runs=False, + save_to_file=False, + ) + + self.assertFalse( + self.lockfile_path.exists(), + "constructor failure must release the lock file", + ) + + # A second tracker can now start without hitting the lock. + tracker = EmissionsTracker( + allow_multiple_runs=False, + save_to_file=False, + ) + self.assertTrue(self.lockfile_path.exists()) + tracker._lock.release() diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 36447409d..872bf27b8 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -1,3 +1,4 @@ +import sys import tempfile import time import unittest @@ -6,7 +7,11 @@ import pandas as pd -from codecarbon.emissions_tracker import OfflineEmissionsTracker +from codecarbon.emissions_tracker import ( + EmissionsTracker, + OfflineEmissionsTracker, + track_emissions, +) from tests.testutils import get_custom_mock_open, get_test_data_source @@ -68,6 +73,75 @@ def test_offline_tracker_task(self): self.assertGreater(task_emission_data.emissions, 0.0) self.assertEqual(task_emission_data.country_name, None) + def test_offline_tracker_raises_on_invalid_output_dir(self): + # Configuration errors must not be swallowed by the constructor, + # otherwise the user gets a half-built tracker that silently does + # nothing. Same semantics as the online EmissionsTracker. + with self.assertRaises(OSError): + OfflineEmissionsTracker( + country_iso_code="FRA", + output_dir=str(self.temp_path / "does_not_exist"), + ) + with self.assertRaises(OSError): + EmissionsTracker(output_dir=str(self.temp_path / "does_not_exist")) + + def test_offline_tracker_raises_on_invalid_region(self): + # A second, offline-specific constructor path: the region check runs + # before `super().__init__`, so it also has to reach the caller. + with self.assertRaises(TypeError): + OfflineEmissionsTracker(country_iso_code="FRA", region=123) + + def test_offline_tracker_lowercases_valid_region(self): + # The isinstance check must not reject the normal, valid case. + tracker = OfflineEmissionsTracker( + country_iso_code="FRA", region="Ile-de-France", save_to_file=False + ) + self.assertEqual(tracker._region, "ile-de-france") + + def test_offline_tracker_raises_on_invalid_country_2letter_iso_code(self): + with self.assertRaises(TypeError): + OfflineEmissionsTracker(country_iso_code="FRA", country_2letter_iso_code=42) + + def test_offline_tracker_uppercases_valid_country_2letter_iso_code(self): + tracker = OfflineEmissionsTracker( + country_iso_code="FRA", country_2letter_iso_code="fr", save_to_file=False + ) + self.assertEqual(tracker._country_2letter_iso_code, "FR") + + def test_track_emissions_runs_function_when_tracker_construction_fails(self): + # The decorator must never stop the user's function from running, + # even though direct construction raises. + @track_emissions( + offline=True, + country_iso_code="FRA", + output_dir=str(self.temp_path / "does_not_exist"), + ) + def fn(): + return 42 + + with self.assertLogs("codecarbon", level="ERROR"): + self.assertEqual(fn(), 42) + + def test_untracked_function_errors_are_not_chained_to_construction_error(self): + # The construction error is only logged: it must not become the + # __context__ of the user's own exception or show in sys.exc_info(). + seen = {} + + @track_emissions( + offline=True, + country_iso_code="FRA", + output_dir=str(self.temp_path / "does_not_exist"), + ) + def fn(): + seen["exc_info"] = sys.exc_info() + raise KeyError("user error") + + with self.assertLogs("codecarbon", level="ERROR"): + with self.assertRaises(KeyError) as raised: + fn() + self.assertIsNone(raised.exception.__context__) + self.assertEqual(seen["exc_info"], (None, None, None)) + def test_resolve_offline_country_name_logs_on_invalid_iso(self): tracker = OfflineEmissionsTracker( country_iso_code="INVALID",