From e0081ef8cf0e1e36b4363137de430a73979bc1ab Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 2 Jun 2026 22:49:51 -0700 Subject: [PATCH] [Benchmark] Enable reasoning-model (thinking) benchmarking via `--chat-template-kwargs` for client-rendered datasets (#44244) Signed-off-by: Albert Cheng --- ...est_custom_dataset_chat_template_kwargs.py | 75 +++++++++++++++++++ vllm/benchmarks/datasets/datasets.py | 4 + vllm/benchmarks/serve.py | 9 +++ 3 files changed, 88 insertions(+) create mode 100644 tests/benchmarks/test_custom_dataset_chat_template_kwargs.py diff --git a/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py new file mode 100644 index 0000000000..149ea6625a --- /dev/null +++ b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse +import json +from pathlib import Path + +import pytest + +from vllm.benchmarks.datasets import get_samples + + +class _RecordingTokenizer: + """Minimal tokenizer stub that records the kwargs forwarded to + apply_chat_template, so we can assert chat_template_kwargs propagation + without loading a real model/template.""" + + def __init__(self) -> None: + self.captured_kwargs: dict | None = None + self.chat_template = "dummy-template" + + def apply_chat_template( + self, + conversation, + add_generation_prompt: bool = True, + tokenize: bool = False, + **kwargs, + ) -> str: + self.captured_kwargs = kwargs + return conversation[0]["content"] + + def __call__(self, text: str): + return argparse.Namespace(input_ids=list(range(len(text.split())))) + + +def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace: + return argparse.Namespace( + dataset_name="custom", + dataset_path=dataset_path, + disable_shuffle=True, + num_prompts=1, + custom_output_len=32, + skip_chat_template=False, + chat_template_kwargs=chat_template_kwargs, + no_oversample=False, + seed=0, + request_id_prefix="", + ) + + +def _write_one(path: Path) -> None: + path.write_text(json.dumps({"prompt": "hello world"}) + "\n") + + +@pytest.mark.benchmark +def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None: + """--chat-template-kwargs must reach the client-side apply_chat_template.""" + jsonl = tmp_path / "data.jsonl" + _write_one(jsonl) + + tok = _RecordingTokenizer() + get_samples(_args(str(jsonl), {"thinking": True}), tok) + + assert tok.captured_kwargs == {"thinking": True} + + +@pytest.mark.benchmark +def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None: + """When not provided, no extra kwargs are passed (existing behavior).""" + jsonl = tmp_path / "data.jsonl" + _write_one(jsonl) + + tok = _RecordingTokenizer() + get_samples(_args(str(jsonl), None), tok) + + assert tok.captured_kwargs == {} diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index cc64bd0b94..59e2aa578c 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -2080,6 +2080,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: tokenizer=tokenizer, output_len=args.custom_output_len, skip_chat_template=args.skip_chat_template, + chat_template_kwargs=getattr(args, "chat_template_kwargs", None), request_id_prefix=args.request_id_prefix, no_oversample=args.no_oversample, ) @@ -2409,6 +2410,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: num_requests=args.num_prompts, tokenizer=tokenizer, output_len=args.speed_bench_output_len, + chat_template_kwargs=getattr(args, "chat_template_kwargs", None), enable_multimodal_chat=args.enable_multimodal_chat, request_id_prefix=args.request_id_prefix, no_oversample=args.no_oversample, @@ -2496,6 +2498,7 @@ class CustomDataset(BenchmarkDataset): output_len: int | None = None, enable_multimodal_chat: bool = False, skip_chat_template: bool = False, + chat_template_kwargs: dict | None = None, **kwargs, ) -> list[SampleRequest]: # load all data if needed @@ -2543,6 +2546,7 @@ class CustomDataset(BenchmarkDataset): [{"role": "user", "content": prompt}], add_generation_prompt=True, tokenize=False, + **(chat_template_kwargs or {}), ) prompt_len = len(tokenizer(prompt).input_ids) diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 2bef4b14d8..5ebc297d50 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1609,6 +1609,15 @@ def add_cli_args(parser: argparse.ArgumentParser): "in seconds. Ready check will be skipped by default.", ) + parser.add_argument( + "--chat-template-kwargs", + type=json.loads, + default=None, + help="A JSON string of kwargs forwarded to the tokenizer's " + "apply_chat_template when a dataset renders prompts client-side " + "(e.g. custom / speed_bench). " + "Example: '{\"thinking\": true}' to enable reasoning models.", + ) parser.add_argument( "--extra-body", help="A JSON string representing extra body parameters to include "