From ba873dec2731f2e240f65aca3bf366e3f5a89409 Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 11:30:22 +0000 Subject: [PATCH 1/6] enable multi-turn messages Signed-off-by: rebel-jinhwan --- src/guidellm/backends/openai/request_handlers.py | 12 +++++++++++- src/guidellm/data/preprocessors/mappers.py | 5 +++++ src/guidellm/data/schemas.py | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index d3fb3f964..d36e1d9a7 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -451,7 +451,17 @@ def format( if kwargs.get("extras"): arguments.model_combine(kwargs["extras"]) - # Build messages + # Build messages: use multi-turn messages from data if present + raw_messages = data.columns.get("messages_column", []) + if raw_messages and isinstance(raw_messages[0], list) and raw_messages[0]: + first = raw_messages[0] + if all( + isinstance(m, dict) and "role" in m and "content" in m for m in first + ): + arguments.body["messages"] = first + return arguments + + # Fallback: single-turn from prefix_column + text_column (and media) arguments.body["messages"] = [] # Build the system prompt diff --git a/src/guidellm/data/preprocessors/mappers.py b/src/guidellm/data/preprocessors/mappers.py index 99a794a91..598c12f12 100644 --- a/src/guidellm/data/preprocessors/mappers.py +++ b/src/guidellm/data/preprocessors/mappers.py @@ -61,6 +61,11 @@ class GenerativeColumnMapper(DataDependentPreprocessor): "wav", "mp3", ], + "messages_column": [ + "messages", + "conversations", + "conversation", + ], } @classmethod diff --git a/src/guidellm/data/schemas.py b/src/guidellm/data/schemas.py index 16af56dff..797c9a20d 100644 --- a/src/guidellm/data/schemas.py +++ b/src/guidellm/data/schemas.py @@ -23,6 +23,7 @@ "image_column", "video_column", "audio_column", + "messages_column", ] class DataNotSupportedError(Exception): From b1501a8839732da80ef4866a38139bc451ee21f8 Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 12:08:27 +0000 Subject: [PATCH 2/6] fix variable name Signed-off-by: rebel-jinhwan --- src/guidellm/backends/openai/request_handlers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index d36e1d9a7..7f3b98b16 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -451,14 +451,17 @@ def format( if kwargs.get("extras"): arguments.model_combine(kwargs["extras"]) - # Build messages: use multi-turn messages from data if present + # Build messages: use multi-turn from data if present (full conversation per row) + # columns["messages_column"] is a list of length N (one per data source); each + # element is the full messages array for this request, e.g. [system, user, asst, user]. raw_messages = data.columns.get("messages_column", []) if raw_messages and isinstance(raw_messages[0], list) and raw_messages[0]: - first = raw_messages[0] + conversation = raw_messages[0] # full multi-turn list for this row if all( - isinstance(m, dict) and "role" in m and "content" in m for m in first + isinstance(m, dict) and "role" in m and "content" in m + for m in conversation ): - arguments.body["messages"] = first + arguments.body["messages"] = conversation return arguments # Fallback: single-turn from prefix_column + text_column (and media) From d11b239c380d296291097728a53ae6f7f756046e Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 16:04:37 +0000 Subject: [PATCH 3/6] remove unused mappings Signed-off-by: rebel-jinhwan --- src/guidellm/data/preprocessors/mappers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/guidellm/data/preprocessors/mappers.py b/src/guidellm/data/preprocessors/mappers.py index 598c12f12..a947b7606 100644 --- a/src/guidellm/data/preprocessors/mappers.py +++ b/src/guidellm/data/preprocessors/mappers.py @@ -63,8 +63,6 @@ class GenerativeColumnMapper(DataDependentPreprocessor): ], "messages_column": [ "messages", - "conversations", - "conversation", ], } From d80990c1222b28f43e6b502b40805545d2029eb3 Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 16:04:50 +0000 Subject: [PATCH 4/6] fix: quality check issue Signed-off-by: rebel-jinhwan --- src/guidellm/backends/openai/request_handlers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/guidellm/backends/openai/request_handlers.py b/src/guidellm/backends/openai/request_handlers.py index 7f3b98b16..25ccd4b1f 100644 --- a/src/guidellm/backends/openai/request_handlers.py +++ b/src/guidellm/backends/openai/request_handlers.py @@ -451,9 +451,10 @@ def format( if kwargs.get("extras"): arguments.model_combine(kwargs["extras"]) - # Build messages: use multi-turn from data if present (full conversation per row) - # columns["messages_column"] is a list of length N (one per data source); each - # element is the full messages array for this request, e.g. [system, user, asst, user]. + # Build messages: use multi-turn from data if present (full conversation per + # row). columns["messages_column"] is a list of length N (one per data source); + # each element is the full messages array for this request, + # e.g. [system, user, asst, user]. raw_messages = data.columns.get("messages_column", []) if raw_messages and isinstance(raw_messages[0], list) and raw_messages[0]: conversation = raw_messages[0] # full multi-turn list for this row From 8a5c54c3d5f94a271a02b7185f5703d61dc82618 Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 16:06:37 +0000 Subject: [PATCH 5/6] add docs Signed-off-by: rebel-jinhwan --- docs/guides/datasets.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/guides/datasets.md b/docs/guides/datasets.md index e5104d1ec..67ffb0cf3 100644 --- a/docs/guides/datasets.md +++ b/docs/guides/datasets.md @@ -10,7 +10,7 @@ The following arguments can be used to configure datasets and their processing: - `--data`: Specifies the dataset source. This can be a file path, Hugging Face dataset ID, synthetic data configuration, or in-memory data. - `--data-args`: A JSON string or dictionary argument that allows you to control how datasets are parsed and prepared. This includes specific aliases for GuideLLM flows, such as: - - `prompt_column`: Specifies the column name for the prompt. By default, GuideLLM will try the most common column names (e.g., `prompt`, `text`, `input`). + - `prompt_column`: Specifies the column name for the prompt. By default, GuideLLM will try the most common column names (e.g., `prompt`, `text`, `input`). Datasets may instead provide a `messages` column for full multi-turn chat per request. - `prompt_tokens_count_column`: Specifies the column name for the prompt token count. These are used to set the request prompt token count for counting metrics. By default, GuideLLM assumes no token count is provided. - `output_tokens_count_column`: Specifies the column name for the output token count. These are used to set the request output token count for the request and counting metrics. By default, GuideLLM assumes no token count is provided. - `split`: Specifies the dataset split to use (e.g., `train`, `val`, `test`). By default, GuideLLM will try the most common split names (e.g., `train`, `validation`, `test`) if the dataset has splits, otherwise it will use the entire dataset. @@ -131,6 +131,11 @@ GuideLLM supports various file formats for datasets, including text, CSV, JSON, {"prompt": "Hello, how are you?", "output_tokens_count": 5, "additional_column": "foo", "additional_column2": "bar"} {"prompt": "What is your name?", "output_tokens_count": 3, "additional_column": "baz", "additional_column2": "qux"} ``` +- **Multi-turn messages (`.jsonl`, `.json`, `.csv`, etc.)**: When the dataset has a column named `messages` whose value is a list of objects with `role` and `content` (OpenAI-style chat message format), GuideLLM uses that column as the full conversation for each request instead of building a single-turn message from `prefix_column` and `text_column`. Each row represents one request; the column value is the complete message list (e.g. `system`, `user`, `assistant`, `user`). You can combine this with other columns such as `output_tokens_count` (or map a custom name via `--data-column-mapper`). Content may be a string or a list of content parts (e.g. `[{"type": "text", "text": "..."}]`). + ```json + {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is AI?"}, {"role": "assistant", "content": "AI is..."}, {"role": "user", "content": "Give an example."}], "output_tokens_count": 256} + {"messages": [{"role": "user", "content": "Hello."}], "output_tokens_count": 128} + ``` - **JSON files (`.json`)**: Where the entire dataset is represented as a JSON array of objects nested under a specific key. To surface the correct key to use, a `--data-column-mapper` argument must be passed in of `"field": "NAME"` for where the array exists. The objects should include `prompt` or other common names for the prompt which will be used as the prompt column. Additional fields can be included based on the previously mentioned aliases for the `--data-column-mapper` argument. ```json { @@ -163,6 +168,7 @@ Where `.ext` can be any of the supported file formats listed above. - Ensure the file format matches the expected structure for the dataset and is listed as a supported format. - The `--data-column-mapper` argument can be used to specify additional parameters for parsing the dataset, such as the prompt column name or the split to use. +- For multi-turn datasets, a column named `messages` is automatically mapped to `messages_column`; use `--data-column-mapper '{"messages_column": "your_column_name"}'` only if your column has a different name. - A processor/tokenizer is only required if `GUIDELLM__PREFERRED_PROMPT_TOKENS_SOURCE="local"` or `GUIDELLM__PREFERRED_OUTPUT_TOKENS_SOURCE="local"` is set in the environment. In this case, the processor/tokenizer must be specified using the `--processor` argument. If not set, the processor/tokenizer will be set to the model passed in or retrieved from the server. - More information on the supported formats and additional args for the underlying use of `load_dataset` can be found in the [Hugging Face datasets documentation](https://huggingface.co/docs/datasets/en/loading#local-and-remote-files). @@ -312,6 +318,7 @@ When your dataset uses non-standard column names, you can use `--data-column-map - `text_column`: The main prompt text (defaults: `prompt`, `instruction`, `question`, `input`, `context`, `content`, `text`) - `prefix_column`: System prompt or prefix (defaults: `system_prompt`, `system`, `prefix`) +- `messages_column`: Full multi-turn conversation per request (default: `messages`). When present and valid (list of objects with `role` and `content`), the OpenAI-style request handler uses this as the request body’s `messages` instead of building from `prefix_column` and `text_column`. - `prompt_tokens_count_column`: Column containing prompt token counts (defaults: `prompt_tokens_count`, `input_tokens_count`) - `output_tokens_count_column`: Column containing output token counts (defaults: `output_tokens_count`, `completion_tokens_count`) - `image_column`: Image data column From 1c2016e0152347940284b19816961236f0ad6535 Mon Sep 17 00:00:00 2001 From: rebel-jinhwan Date: Tue, 3 Mar 2026 16:33:41 +0000 Subject: [PATCH 6/6] fix doc --- docs/guides/datasets.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/datasets.md b/docs/guides/datasets.md index 67ffb0cf3..1c3e0f794 100644 --- a/docs/guides/datasets.md +++ b/docs/guides/datasets.md @@ -131,7 +131,7 @@ GuideLLM supports various file formats for datasets, including text, CSV, JSON, {"prompt": "Hello, how are you?", "output_tokens_count": 5, "additional_column": "foo", "additional_column2": "bar"} {"prompt": "What is your name?", "output_tokens_count": 3, "additional_column": "baz", "additional_column2": "qux"} ``` -- **Multi-turn messages (`.jsonl`, `.json`, `.csv`, etc.)**: When the dataset has a column named `messages` whose value is a list of objects with `role` and `content` (OpenAI-style chat message format), GuideLLM uses that column as the full conversation for each request instead of building a single-turn message from `prefix_column` and `text_column`. Each row represents one request; the column value is the complete message list (e.g. `system`, `user`, `assistant`, `user`). You can combine this with other columns such as `output_tokens_count` (or map a custom name via `--data-column-mapper`). Content may be a string or a list of content parts (e.g. `[{"type": "text", "text": "..."}]`). +- **Multi-turn messages (`.jsonl`)**: When the dataset has a column named `messages` whose value is a list of objects with `role` and `content` (OpenAI-style chat message format), GuideLLM uses that column as the full conversation for each request instead of building a single-turn message from `prefix_column` and `text_column`. Each row represents one request; the column value is the complete message list (e.g. `system`, `user`, `assistant`, `user`). You can combine this with other columns such as `output_tokens_count` (or map a custom name via `--data-column-mapper`). Content may be a string or a list of content parts (e.g. `[{"type": "text", "text": "..."}]`). ```json {"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is AI?"}, {"role": "assistant", "content": "AI is..."}, {"role": "user", "content": "Give an example."}], "output_tokens_count": 256} {"messages": [{"role": "user", "content": "Hello."}], "output_tokens_count": 128}