44from typing import Any
55
66import pytest
7- from pydantic import ValidationError
87
98from guidellm .benchmark .entrypoints import resolve_profile
109from guidellm .benchmark .profiles import ProfileFactory , ReplayProfile
1110from guidellm .benchmark .profiles .replay import ReplayProfileArgs
12- from guidellm .data .deserializers import TraceSyntheticDataArgs
1311from guidellm .scheduler import (
14- MaxNumberConstraint ,
15- MaxRequestsConstraintArgs ,
1612 TraceReplayStrategy ,
1713)
1814
1915
20- def _trace_path (tmp_path : Path , lines : list [str ] | None = None ) -> Path :
21- path = tmp_path / "trace.jsonl"
22- path .write_text ("\n " .join (lines or []))
23- return path
24-
25-
2616def _replay_args (** kwargs ) -> ReplayProfileArgs :
2717 payload = {"kind" : "replay" , ** kwargs }
2818 return ReplayProfileArgs .model_validate (payload )
@@ -31,325 +21,15 @@ def _replay_args(**kwargs) -> ReplayProfileArgs:
3121def _replay_profile (
3222 constraints : dict [str , Any ] | None = None , random_seed : int = 42 , ** kwargs
3323) -> ReplayProfile :
34- data = kwargs .pop ("data" , None )
35- data_samples = kwargs .pop ("data_samples" , - 1 )
3624 args = _replay_args (** kwargs )
37- profile_kwargs : dict [str , Any ] = {"data_samples" : data_samples }
38- if data is not None :
39- profile_kwargs ["data" ] = data
4025 return ProfileFactory .create (
41- args , random_seed , constraints = constraints , ** profile_kwargs
26+ args ,
27+ random_seed ,
28+ constraints = constraints ,
4229 )
4330
4431
4532class TestReplayProfile :
46- @pytest .mark .smoke
47- def test_requires_data (self ):
48- """
49- Replay profile requires a trace data source.
50-
51- ## WRITTEN BY AI ##
52- """
53- args = _replay_args ()
54- with pytest .raises (ValueError , match = "exactly one data source" ):
55- ProfileFactory .create (args , 42 )
56-
57- @pytest .mark .smoke
58- def test_rejects_multiple_data_sources (self , tmp_path : Path ):
59- """
60- Replay profile rejects more than one data source.
61-
62- ## WRITTEN BY AI ##
63- """
64- args = _replay_args ()
65- with pytest .raises (ValueError , match = "exactly one data source" ):
66- ProfileFactory .create (
67- args ,
68- 42 ,
69- data = [
70- TraceSyntheticDataArgs (path = tmp_path / "trace-a.jsonl" ),
71- TraceSyntheticDataArgs (path = tmp_path / "trace-b.jsonl" ),
72- ],
73- )
74-
75- @pytest .mark .smoke
76- def test_rejects_missing_or_empty_trace (self , tmp_path : Path ):
77- """
78- Replay profile rejects missing files and empty traces.
79-
80- ## WRITTEN BY AI ##
81- """
82- missing = tmp_path / "missing.jsonl"
83- args = _replay_args ()
84- with pytest .raises (ValueError , match = "not found" ):
85- ProfileFactory .create (args , 42 , data = [TraceSyntheticDataArgs (path = missing )])
86-
87- empty = _trace_path (tmp_path )
88- with pytest .raises (ValueError , match = "empty|No timestamps" ):
89- ProfileFactory .create (args , 42 , data = [TraceSyntheticDataArgs (path = empty )])
90-
91- @pytest .mark .smoke
92- @pytest .mark .parametrize (
93- ("time_scale" , "expected_scale" ),
94- [
95- (None , 1.0 ),
96- (2.0 , 2.0 ),
97- ],
98- )
99- def test_profile_create_resolves_time_scale_and_default_max_requests (
100- self , tmp_path : Path , time_scale , expected_scale
101- ):
102- """
103- Profile.create resolves time scale and default max_requests from trace data.
104-
105- ## WRITTEN BY AI ##
106- """
107- trace = _trace_path (
108- tmp_path ,
109- [
110- '{"timestamp": 5.0, "input_length": 1, "output_length": 1}' ,
111- '{"timestamp": 2.0, "input_length": 2, "output_length": 2}' ,
112- '{"timestamp": 8.0, "input_length": 3, "output_length": 3}' ,
113- ],
114- )
115-
116- payload : dict = {
117- "data" : [TraceSyntheticDataArgs (path = trace )],
118- }
119- if time_scale is not None :
120- payload ["time_scale" ] = time_scale
121-
122- profile = _replay_profile (** payload )
123-
124- assert isinstance (profile , ReplayProfile )
125- assert profile .args .time_scale == expected_scale
126- assert profile .constraints ["max_requests" ] == MaxNumberConstraint (
127- args = MaxRequestsConstraintArgs (count = 3 ), current_index = - 1
128- )
129-
130- @pytest .mark .sanity
131- def test_non_positive_time_scale_is_rejected (self , tmp_path : Path ):
132- """
133- Non-positive time scale values are rejected during argument validation.
134-
135- ## WRITTEN BY AI ##
136- """
137- trace = _trace_path (
138- tmp_path ,
139- ['{"timestamp": 0, "input_length": 1, "output_length": 1}' ],
140- )
141-
142- with pytest .raises (ValidationError ):
143- _replay_profile (
144- time_scale = 0.0 ,
145- data = [TraceSyntheticDataArgs (path = trace )],
146- )
147-
148- @pytest .mark .smoke
149- def test_custom_timestamp_column_via_data_args (self , tmp_path : Path ):
150- """
151- Custom timestamp columns are honored when loading trace timestamps.
152-
153- ## WRITTEN BY AI ##
154- """
155- trace = _trace_path (
156- tmp_path ,
157- [
158- '{"ts": 5.0, "input_length": 100, "output_length": 10}' ,
159- '{"ts": 2.0, "input_length": 200, "output_length": 20}' ,
160- '{"ts": 8.0, "input_length": 300, "output_length": 30}' ,
161- ],
162- )
163-
164- profile = _replay_profile (
165- data = [TraceSyntheticDataArgs (path = trace , timestamp_column = "ts" )],
166- )
167-
168- assert profile .constraints ["max_requests" ] == MaxNumberConstraint (
169- args = MaxRequestsConstraintArgs (count = 3 ), current_index = - 1
170- )
171-
172- @pytest .mark .smoke
173- def test_large_bursty_trace_sets_default_request_constraint (self , tmp_path : Path ):
174- """
175- Default max_requests matches the number of loaded trace events.
176-
177- ## WRITTEN BY AI ##
178- """
179- prompt_lengths = [
180- 6755 ,
181- 7319 ,
182- 7234 ,
183- 2287 ,
184- 9013 ,
185- 6506 ,
186- 4824 ,
187- 3119 ,
188- 23090 ,
189- 3135 ,
190- 26874 ,
191- 10487 ,
192- 17448 ,
193- 6253 ,
194- 6725 ,
195- 13538 ,
196- 87162 ,
197- 6166 ,
198- 6320 ,
199- 2007 ,
200- 3174 ,
201- 3131 ,
202- 3159 ,
203- 6820 ,
204- 3154 ,
205- 9416 ,
206- 7460 ,
207- ]
208- timestamps = [
209- 0.0 ,
210- 0.0 ,
211- 0.0 ,
212- 0.0 ,
213- 0.0 ,
214- 0.0 ,
215- 0.0 ,
216- 0.0 ,
217- 0.0 ,
218- 0.0 ,
219- 0.0 ,
220- 0.5 ,
221- 0.5 ,
222- 0.5 ,
223- 0.5 ,
224- 0.5 ,
225- 0.5 ,
226- 0.5 ,
227- 1.0 ,
228- 1.0 ,
229- 1.0 ,
230- 1.0 ,
231- 1.0 ,
232- 2.0 ,
233- 2.0 ,
234- 2.0 ,
235- 2.0 ,
236- ]
237- trace = _trace_path (
238- tmp_path ,
239- [
240- (
241- f'{{"timestamp": { timestamp } , '
242- f'"input_length": { prompt_length } , "output_length": 1}}'
243- )
244- for timestamp , prompt_length in zip (
245- timestamps , prompt_lengths , strict = True
246- )
247- ],
248- )
249-
250- profile = _replay_profile (data = [TraceSyntheticDataArgs (path = trace )])
251-
252- assert profile .constraints ["max_requests" ] == MaxNumberConstraint (
253- args = MaxRequestsConstraintArgs (count = 27 ), current_index = - 1
254- )
255-
256- @pytest .mark .smoke
257- @pytest .mark .parametrize ("invalid_value" , [None , 123 , False , []])
258- def test_non_string_timestamp_column_rejected_by_pydantic (
259- self , tmp_path : Path , invalid_value
260- ):
261- """
262- Trace data args reject invalid timestamp column types.
263-
264- ## WRITTEN BY AI ##
265- """
266- with pytest .raises (ValidationError ):
267- TraceSyntheticDataArgs (
268- path = tmp_path / "trace.jsonl" ,
269- timestamp_column = invalid_value ,
270- )
271-
272- @pytest .mark .smoke
273- @pytest .mark .parametrize ("invalid_col" , ["" , " " ])
274- def test_blank_timestamp_column_raises_error (self , tmp_path : Path , invalid_col ):
275- """
276- Blank timestamp column names fail when loading trace timestamps.
277-
278- ## WRITTEN BY AI ##
279- """
280- trace = _trace_path (
281- tmp_path ,
282- [
283- '{"timestamp": 10.0, "input_length": 1, "output_length": 1}' ,
284- '{"timestamp": 12.0, "input_length": 2, "output_length": 2}' ,
285- ],
286- )
287-
288- with pytest .raises ((KeyError , ValueError )):
289- _replay_profile (
290- data = [TraceSyntheticDataArgs (path = trace , timestamp_column = invalid_col )],
291- )
292-
293- @pytest .mark .smoke
294- def test_data_samples_truncates_after_sorting_and_preserves_constraints (
295- self , tmp_path : Path
296- ):
297- """
298- data_samples truncates timestamps while preserving explicit constraints.
299-
300- ## WRITTEN BY AI ##
301- """
302- trace = _trace_path (
303- tmp_path ,
304- [
305- '{"timestamp": 5.0, "input_length": 1, "output_length": 1}' ,
306- '{"timestamp": 2.0, "input_length": 2, "output_length": 2}' ,
307- '{"timestamp": 8.0, "input_length": 3, "output_length": 3}' ,
308- '{"timestamp": 1.0, "input_length": 4, "output_length": 4}' ,
309- ],
310- )
311-
312- profile = _replay_profile (
313- data = [TraceSyntheticDataArgs (path = trace )],
314- data_samples = 3 ,
315- constraints = {
316- "max_requests" : {"max_num" : 10 },
317- "max_duration" : {"max_duration" : 0.25 },
318- },
319- )
320-
321- assert profile .constraints == {
322- "max_requests" : {"max_num" : 10 },
323- "max_duration" : {"max_duration" : 0.25 },
324- }
325-
326- @pytest .mark .smoke
327- @pytest .mark .parametrize ("data_samples" , [0 , - 1 ])
328- def test_non_positive_data_samples_do_not_truncate (
329- self , tmp_path : Path , data_samples : int
330- ):
331- """
332- Non-positive data_samples values keep the full trace.
333-
334- ## WRITTEN BY AI ##
335- """
336- trace = _trace_path (
337- tmp_path ,
338- [
339- '{"timestamp": 0, "input_length": 1, "output_length": 1}' ,
340- '{"timestamp": 1.0, "input_length": 2, "output_length": 2}' ,
341- ],
342- )
343-
344- profile = _replay_profile (
345- data = [TraceSyntheticDataArgs (path = trace )],
346- data_samples = data_samples ,
347- )
348-
349- assert profile .constraints ["max_requests" ] == MaxNumberConstraint (
350- args = MaxRequestsConstraintArgs (count = 2 ), current_index = - 1
351- )
352-
35333 @pytest .mark .smoke
35434 @pytest .mark .asyncio
35535 async def test_resolve_profile_passes_replay_specific_kwargs (self , tmp_path : Path ):
@@ -358,23 +38,12 @@ async def test_resolve_profile_passes_replay_specific_kwargs(self, tmp_path: Pat
35838
35939 ## WRITTEN BY AI ##
36040 """
361- trace = _trace_path (
362- tmp_path ,
363- [
364- '{"ts": 5.0, "input_length": 1, "output_length": 1}' ,
365- '{"ts": 2.0, "input_length": 2, "output_length": 2}' ,
366- '{"ts": 8.0, "input_length": 3, "output_length": 3}' ,
367- ],
368- )
369-
37041 profile = await resolve_profile (
37142 profile = ReplayProfileArgs .model_validate (
37243 {"kind" : "replay" , "time_scale" : 2.0 }
37344 ),
37445 constraints = {"max_requests" : {"max_num" : 2 }},
37546 random_seed = 42 ,
376- data = [TraceSyntheticDataArgs (path = trace , timestamp_column = "ts" )],
377- data_samples = 2 ,
37847 )
37948
38049 assert isinstance (profile , ReplayProfile )
@@ -388,14 +57,7 @@ def test_next_strategy_returns_trace_then_none(self, tmp_path: Path):
38857
38958 ## WRITTEN BY AI ##
39059 """
391- trace = _trace_path (
392- tmp_path ,
393- ['{"timestamp": 0, "input_length": 1, "output_length": 1}' ],
394- )
395- profile = _replay_profile (
396- time_scale = 2.0 ,
397- data = [TraceSyntheticDataArgs (path = trace )],
398- )
60+ profile = _replay_profile (time_scale = 2.0 )
39961
40062 strategy = profile .next_strategy (None , None )
40163 assert profile .strategy_types == ["trace" ]
0 commit comments