Skip to content

Commit d67048f

Browse files
lyglstOrbax Authors
authored andcommitted
Modify the telemetry for saving metrics.
PiperOrigin-RevId: 929454143
1 parent 4287cc7 commit d67048f

1 file changed

Lines changed: 63 additions & 34 deletions

File tree

checkpoint/orbax/checkpoint/_src/logging/event_tracking.py

Lines changed: 63 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -107,26 +107,39 @@ def record_blocking_completion(self, duration_secs: float):
107107
"""Records the completion of the blocking part of an operation."""
108108
match self._operation_type:
109109
case OperationType.SAVE:
110-
event_name = (
111-
'/jax/checkpoint/write/async/blocking_duration_secs'
112-
if self._async_origin
113-
else '/jax/orbax/write/blocking_duration_secs'
114-
)
115110
record_write_event(self._path)
111+
if multihost.is_primary_host(self._primary_host):
112+
if self._async_origin:
113+
# Keep both /jax/checkpoint/write/... and /jax/orbax/write/...
114+
# as the /jax/checkpoint/write/... event is used for alerting.
115+
jax.monitoring.record_event_duration_secs(
116+
'/jax/checkpoint/write/async/blocking_duration_secs',
117+
duration_secs,
118+
)
119+
jax.monitoring.record_event_duration_secs(
120+
'/jax/orbax/write/async/blocking_duration_secs',
121+
duration_secs,
122+
storage_type=self._storage_type,
123+
)
124+
else:
125+
jax.monitoring.record_event_duration_secs(
126+
'/jax/orbax/write/blocking_duration_secs',
127+
duration_secs,
128+
storage_type=self._storage_type,
129+
)
116130
case OperationType.LOAD:
117-
event_name = (
118-
'/jax/orbax/read/async/blocking_duration_secs'
119-
if self._async_origin
120-
else '/jax/orbax/read/blocking_duration_secs'
121-
)
122131
record_read_event(self._path)
123-
124-
if multihost.is_primary_host(self._primary_host):
125-
jax.monitoring.record_event_duration_secs(
126-
event_name,
127-
duration_secs,
128-
storage_type=self._storage_type,
129-
)
132+
if multihost.is_primary_host(self._primary_host):
133+
event_name = (
134+
'/jax/orbax/read/async/blocking_duration_secs'
135+
if self._async_origin
136+
else '/jax/orbax/read/blocking_duration_secs'
137+
)
138+
jax.monitoring.record_event_duration_secs(
139+
event_name,
140+
duration_secs,
141+
storage_type=self._storage_type,
142+
)
130143

131144
logging.info(
132145
'[process=%s] [%s] Finished blocking %s in %.2f seconds. Continuing %s'
@@ -152,23 +165,39 @@ def record_completion(self, duration_secs: float):
152165
)
153166
match self._operation_type:
154167
case OperationType.SAVE:
155-
duration_event_name = (
156-
'/jax/checkpoint/write/async/total_duration_secs'
157-
if self._async_origin
158-
else '/jax/orbax/write/total_duration_secs'
159-
)
160168
success_event_name = '/jax/orbax/write/success'
169+
if multihost.is_primary_host(self._primary_host):
170+
# Keep both /jax/checkpoint/write/... and /jax/orbax/write/...
171+
# as the /jax/checkpoint/write/... event is used for alerting.
172+
jax.monitoring.record_event(success_event_name)
173+
if self._async_origin:
174+
jax.monitoring.record_event_duration_secs(
175+
'/jax/checkpoint/write/async/total_duration_secs',
176+
duration_secs,
177+
storage_type=self._storage_type,
178+
)
179+
jax.monitoring.record_event_duration_secs(
180+
'/jax/orbax/write/async/total_duration_secs',
181+
duration_secs,
182+
storage_type=self._storage_type,
183+
)
184+
else:
185+
jax.monitoring.record_event_duration_secs(
186+
'/jax/orbax/write/total_duration_secs',
187+
duration_secs,
188+
storage_type=self._storage_type,
189+
)
161190
case OperationType.LOAD:
162-
duration_event_name = (
163-
'/jax/orbax/read/async/total_duration_secs'
164-
if self._async_origin
165-
else '/jax/orbax/read/total_duration_secs'
166-
)
167191
success_event_name = '/jax/orbax/read/success'
168-
if multihost.is_primary_host(self._primary_host):
169-
jax.monitoring.record_event(success_event_name)
170-
jax.monitoring.record_event_duration_secs(
171-
duration_event_name,
172-
duration_secs,
173-
storage_type=self._storage_type,
174-
)
192+
if multihost.is_primary_host(self._primary_host):
193+
jax.monitoring.record_event(success_event_name)
194+
duration_event_name = (
195+
'/jax/orbax/read/async/total_duration_secs'
196+
if self._async_origin
197+
else '/jax/orbax/read/total_duration_secs'
198+
)
199+
jax.monitoring.record_event_duration_secs(
200+
duration_event_name,
201+
duration_secs,
202+
storage_type=self._storage_type,
203+
)

0 commit comments

Comments
 (0)