Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions codecarbon/emissions_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,8 @@ def _resolve_output_methods(

def _initialize_runtime_state(self) -> None:
self._start_time: Optional[float] = None
# Set by stop(): a stopped tracker cannot be restarted.
self._stopped: bool = False
self._last_measured_time: float = time.perf_counter()
self._total_energy: Energy = Energy.from_energy(kWh=0)
self._total_emissions: float = 0.0
Expand Down Expand Up @@ -708,6 +710,14 @@ def start(self) -> None:
"Another instance of codecarbon is already running. Exiting."
)
return
if self._stopped:
# `start()` is wrapped in @suppress(Exception), so raising here
# would be swallowed: log instead of pretending it worked.
logger.error(
"This tracker was already stopped and cannot be restarted: "
"nothing will be measured. Create a new tracker instead."
)
return
if self._start_time is not None:
logger.warning("Already started tracking")
return
Expand Down Expand Up @@ -745,6 +755,12 @@ def start_task(self, task_name=None) -> None:
"Another instance of codecarbon is already running. Exiting."
)
return
if self._stopped:
logger.error(
"This tracker was already stopped and cannot be restarted: "
"nothing will be measured. Create a new tracker instead."
)
return
try:
self._ensure_emissions_engine()
except Exception:
Expand Down Expand Up @@ -965,6 +981,7 @@ def stop(self) -> Optional[float]:
handler.exit()

self._start_time = None
self._stopped = True

return emissions_data.emissions

Expand Down
66 changes: 66 additions & 0 deletions tests/test_emissions_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,39 @@ def test_offline_tracker_country_name(
self.assertEqual("United States", emissions_df["country_name"].values[0])
self.assertEqual("USA", emissions_df["country_iso_code"].values[0])

def test_stop_releases_the_lock_only_once(
self,
mock_cli_setup,
mock_log_values,
mocked_get_gpu_details,
mocked_env_cloud_details,
mocked_get_gpu_utilization_list,
mocked_is_gpu_details_available,
mocked_is_nvidia_system,
):
with mock.patch("codecarbon.emissions_tracker.Lock") as mock_lock_class:
tracker = OfflineEmissionsTracker(
country_iso_code="USA",
output_dir=self.temp_path,
experiment_id="test",
allow_multiple_runs=False,
)
lock = mock_lock_class.return_value
lock.acquire.assert_called_once()

tracker.start()
heavy_computation(run_time_secs=1)
first_emissions = tracker.stop()
lock.release.assert_called_once()

# A second stop() is a no-op: it must not touch the lock again, which
# by then may belong to another tracker.
second_emissions = tracker.stop()
lock.release.assert_called_once()

self.assertEqual(first_emissions, second_emissions)
self.verify_output_file(self.emissions_file_path, 2)

def test_offline_tracker_invalid_headers(
self,
mock_cli_setup,
Expand Down Expand Up @@ -1153,3 +1186,36 @@ def test_cumulative_emissions_with_varying_intensity(

# Verification: If it wasn't cumulative, it would be 3.0 kWh * 300 g/kWh = 0.9 kg
self.assertLess(data3.emissions, 0.8)


class TestRestartAfterStop(unittest.TestCase):
def test_start_after_stop_is_refused(self):
tracker = OfflineEmissionsTracker(
country_iso_code="FRA",
save_to_file=False,
allow_multiple_runs=True,
measure_power_secs=10,
)
tracker.start()
tracker.stop()
with self.assertLogs("codecarbon", level="ERROR") as logs:
tracker.start()
self.assertIn("cannot be restarted", "".join(logs.output))
# Refused, not half-restarted: nothing was rebuilt.
self.assertIsNone(tracker._scheduler)

def test_start_task_after_stop_is_refused(self):
tracker = OfflineEmissionsTracker(
country_iso_code="FRA",
save_to_file=False,
allow_multiple_runs=True,
measure_power_secs=10,
)
tracker.start()
tracker.stop()
with self.assertLogs("codecarbon", level="ERROR") as logs:
# _scheduler is None after stop(); start_task must hit the
# _stopped guard before touching it, not raise AttributeError.
tracker.start_task()
self.assertIn("cannot be restarted", "".join(logs.output))
self.assertIsNone(tracker._active_task)
Loading