[Benchmark] Enable reasoning-model (thinking) benchmarking via --chat-template-kwargs for client-rendered datasets (#44244)

Signed-off-by: Albert Cheng <albertching0112@gmail.com>
This commit is contained in:
Albert Cheng
2026-06-02 22:49:51 -07:00
committed by GitHub
parent f0204358d9
commit e0081ef8cf
3 changed files with 88 additions and 0 deletions
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
import pytest
from vllm.benchmarks.datasets import get_samples
class _RecordingTokenizer:
"""Minimal tokenizer stub that records the kwargs forwarded to
apply_chat_template, so we can assert chat_template_kwargs propagation
without loading a real model/template."""
def __init__(self) -> None:
self.captured_kwargs: dict | None = None
self.chat_template = "dummy-template"
def apply_chat_template(
self,
conversation,
add_generation_prompt: bool = True,
tokenize: bool = False,
**kwargs,
) -> str:
self.captured_kwargs = kwargs
return conversation[0]["content"]
def __call__(self, text: str):
return argparse.Namespace(input_ids=list(range(len(text.split()))))
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=True,
num_prompts=1,
custom_output_len=32,
skip_chat_template=False,
chat_template_kwargs=chat_template_kwargs,
no_oversample=False,
seed=0,
request_id_prefix="",
)
def _write_one(path: Path) -> None:
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
@pytest.mark.benchmark
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), {"thinking": True}), tok)
assert tok.captured_kwargs == {"thinking": True}
@pytest.mark.benchmark
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
"""When not provided, no extra kwargs are passed (existing behavior)."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), None), tok)
assert tok.captured_kwargs == {}
+4
View File
@@ -2080,6 +2080,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
tokenizer=tokenizer,
output_len=args.custom_output_len,
skip_chat_template=args.skip_chat_template,
chat_template_kwargs=getattr(args, "chat_template_kwargs", None),
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
@@ -2409,6 +2410,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
num_requests=args.num_prompts,
tokenizer=tokenizer,
output_len=args.speed_bench_output_len,
chat_template_kwargs=getattr(args, "chat_template_kwargs", None),
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
@@ -2496,6 +2498,7 @@ class CustomDataset(BenchmarkDataset):
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs,
) -> list[SampleRequest]:
# load all data if needed
@@ -2543,6 +2546,7 @@ class CustomDataset(BenchmarkDataset):
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
**(chat_template_kwargs or {}),
)
prompt_len = len(tokenizer(prompt).input_ids)
+9
View File
@@ -1609,6 +1609,15 @@ def add_cli_args(parser: argparse.ArgumentParser):
"in seconds. Ready check will be skipped by default.",
)
parser.add_argument(
"--chat-template-kwargs",
type=json.loads,
default=None,
help="A JSON string of kwargs forwarded to the tokenizer's "
"apply_chat_template when a dataset renders prompts client-side "
"(e.g. custom / speed_bench). "
"Example: '{\"thinking\": true}' to enable reasoning models.",
)
parser.add_argument(
"--extra-body",
help="A JSON string representing extra body parameters to include "