@@ -107,26 +107,39 @@ def record_blocking_completion(self, duration_secs: float):
107107 """Records the completion of the blocking part of an operation."""
108108 match self ._operation_type :
109109 case OperationType .SAVE :
110- event_name = (
111- '/jax/checkpoint/write/async/blocking_duration_secs'
112- if self ._async_origin
113- else '/jax/orbax/write/blocking_duration_secs'
114- )
115110 record_write_event (self ._path )
111+ if multihost .is_primary_host (self ._primary_host ):
112+ if self ._async_origin :
113+ # Keep both /jax/checkpoint/write/... and /jax/orbax/write/...
114+ # as the /jax/checkpoint/write/... event is used for alerting.
115+ jax .monitoring .record_event_duration_secs (
116+ '/jax/checkpoint/write/async/blocking_duration_secs' ,
117+ duration_secs ,
118+ )
119+ jax .monitoring .record_event_duration_secs (
120+ '/jax/orbax/write/async/blocking_duration_secs' ,
121+ duration_secs ,
122+ storage_type = self ._storage_type ,
123+ )
124+ else :
125+ jax .monitoring .record_event_duration_secs (
126+ '/jax/orbax/write/blocking_duration_secs' ,
127+ duration_secs ,
128+ storage_type = self ._storage_type ,
129+ )
116130 case OperationType .LOAD :
117- event_name = (
118- '/jax/orbax/read/async/blocking_duration_secs'
119- if self ._async_origin
120- else '/jax/orbax/read/blocking_duration_secs'
121- )
122131 record_read_event (self ._path )
123-
124- if multihost .is_primary_host (self ._primary_host ):
125- jax .monitoring .record_event_duration_secs (
126- event_name ,
127- duration_secs ,
128- storage_type = self ._storage_type ,
129- )
132+ if multihost .is_primary_host (self ._primary_host ):
133+ event_name = (
134+ '/jax/orbax/read/async/blocking_duration_secs'
135+ if self ._async_origin
136+ else '/jax/orbax/read/blocking_duration_secs'
137+ )
138+ jax .monitoring .record_event_duration_secs (
139+ event_name ,
140+ duration_secs ,
141+ storage_type = self ._storage_type ,
142+ )
130143
131144 logging .info (
132145 '[process=%s] [%s] Finished blocking %s in %.2f seconds. Continuing %s'
@@ -152,23 +165,39 @@ def record_completion(self, duration_secs: float):
152165 )
153166 match self ._operation_type :
154167 case OperationType .SAVE :
155- duration_event_name = (
156- '/jax/checkpoint/write/async/total_duration_secs'
157- if self ._async_origin
158- else '/jax/orbax/write/total_duration_secs'
159- )
160168 success_event_name = '/jax/orbax/write/success'
169+ if multihost .is_primary_host (self ._primary_host ):
170+ # Keep both /jax/checkpoint/write/... and /jax/orbax/write/...
171+ # as the /jax/checkpoint/write/... event is used for alerting.
172+ jax .monitoring .record_event (success_event_name )
173+ if self ._async_origin :
174+ jax .monitoring .record_event_duration_secs (
175+ '/jax/checkpoint/write/async/total_duration_secs' ,
176+ duration_secs ,
177+ storage_type = self ._storage_type ,
178+ )
179+ jax .monitoring .record_event_duration_secs (
180+ '/jax/orbax/write/async/total_duration_secs' ,
181+ duration_secs ,
182+ storage_type = self ._storage_type ,
183+ )
184+ else :
185+ jax .monitoring .record_event_duration_secs (
186+ '/jax/orbax/write/total_duration_secs' ,
187+ duration_secs ,
188+ storage_type = self ._storage_type ,
189+ )
161190 case OperationType .LOAD :
162- duration_event_name = (
163- '/jax/orbax/read/async/total_duration_secs'
164- if self ._async_origin
165- else '/jax/orbax/read/total_duration_secs'
166- )
167191 success_event_name = '/jax/orbax/read/success'
168- if multihost .is_primary_host (self ._primary_host ):
169- jax .monitoring .record_event (success_event_name )
170- jax .monitoring .record_event_duration_secs (
171- duration_event_name ,
172- duration_secs ,
173- storage_type = self ._storage_type ,
174- )
192+ if multihost .is_primary_host (self ._primary_host ):
193+ jax .monitoring .record_event (success_event_name )
194+ duration_event_name = (
195+ '/jax/orbax/read/async/total_duration_secs'
196+ if self ._async_origin
197+ else '/jax/orbax/read/total_duration_secs'
198+ )
199+ jax .monitoring .record_event_duration_secs (
200+ duration_event_name ,
201+ duration_secs ,
202+ storage_type = self ._storage_type ,
203+ )
0 commit comments