[Benchmark] Enable reasoning-model (thinking) benchmarking via --chat-template-kwargs for client-rendered datasets (#44244)
Signed-off-by: Albert Cheng <albertching0112@gmail.com>
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
|
||||
class _RecordingTokenizer:
|
||||
"""Minimal tokenizer stub that records the kwargs forwarded to
|
||||
apply_chat_template, so we can assert chat_template_kwargs propagation
|
||||
without loading a real model/template."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.captured_kwargs: dict | None = None
|
||||
self.chat_template = "dummy-template"
|
||||
|
||||
def apply_chat_template(
|
||||
self,
|
||||
conversation,
|
||||
add_generation_prompt: bool = True,
|
||||
tokenize: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
self.captured_kwargs = kwargs
|
||||
return conversation[0]["content"]
|
||||
|
||||
def __call__(self, text: str):
|
||||
return argparse.Namespace(input_ids=list(range(len(text.split()))))
|
||||
|
||||
|
||||
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=True,
|
||||
num_prompts=1,
|
||||
custom_output_len=32,
|
||||
skip_chat_template=False,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
no_oversample=False,
|
||||
seed=0,
|
||||
request_id_prefix="",
|
||||
)
|
||||
|
||||
|
||||
def _write_one(path: Path) -> None:
|
||||
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
|
||||
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), {"thinking": True}), tok)
|
||||
|
||||
assert tok.captured_kwargs == {"thinking": True}
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
|
||||
"""When not provided, no extra kwargs are passed (existing behavior)."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), None), tok)
|
||||
|
||||
assert tok.captured_kwargs == {}
|
||||
@@ -2080,6 +2080,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
tokenizer=tokenizer,
|
||||
output_len=args.custom_output_len,
|
||||
skip_chat_template=args.skip_chat_template,
|
||||
chat_template_kwargs=getattr(args, "chat_template_kwargs", None),
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
@@ -2409,6 +2410,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
num_requests=args.num_prompts,
|
||||
tokenizer=tokenizer,
|
||||
output_len=args.speed_bench_output_len,
|
||||
chat_template_kwargs=getattr(args, "chat_template_kwargs", None),
|
||||
enable_multimodal_chat=args.enable_multimodal_chat,
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
@@ -2496,6 +2498,7 @@ class CustomDataset(BenchmarkDataset):
|
||||
output_len: int | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
skip_chat_template: bool = False,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# load all data if needed
|
||||
@@ -2543,6 +2546,7 @@ class CustomDataset(BenchmarkDataset):
|
||||
[{"role": "user", "content": prompt}],
|
||||
add_generation_prompt=True,
|
||||
tokenize=False,
|
||||
**(chat_template_kwargs or {}),
|
||||
)
|
||||
|
||||
prompt_len = len(tokenizer(prompt).input_ids)
|
||||
|
||||
@@ -1609,6 +1609,15 @@ def add_cli_args(parser: argparse.ArgumentParser):
|
||||
"in seconds. Ready check will be skipped by default.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
default=None,
|
||||
help="A JSON string of kwargs forwarded to the tokenizer's "
|
||||
"apply_chat_template when a dataset renders prompts client-side "
|
||||
"(e.g. custom / speed_bench). "
|
||||
"Example: '{\"thinking\": true}' to enable reasoning models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
help="A JSON string representing extra body parameters to include "
|
||||
|
||||
Reference in New Issue
Block a user