diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 6d0b2a01ac..3d8fda95a3 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -405,6 +405,47 @@ vllm bench serve \ Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card. +#### BFCL (Tool-Calling) Benchmark + +The Berkeley Function Calling Leaderboard (BFCL) dataset measures serving +latency and throughput on realistic tool-calling traffic. Each request +carries a per-sample `tools` schema and chat history, so the server must +expose `/v1/chat/completions` with an auto-tool-choice parser enabled. +The benchmark client always uses the `openai-chat` backend. + +Start a tool-parser-enabled server, then run the bench. For example, with +`gpt-oss-20b`: + +```bash +# Server +vllm serve openai/gpt-oss-20b \ + --enable-auto-tool-choice \ + --tool-call-parser openai \ + --reasoning-parser openai_gptoss + +# Client +vllm bench serve \ + --backend openai-chat \ + --endpoint /v1/chat/completions \ + --model openai/gpt-oss-20b \ + --dataset-name hf \ + --dataset-path gorilla-llm/Berkeley-Function-Calling-Leaderboard \ + --bfcl-categories simple,live_simple,multiple \ + --num-prompts 200 +``` + +`--bfcl-categories` is a comma-separated list of BFCL v3 category names +(without the `BFCL_v3_` prefix or `.json` suffix). Defaults to +`simple,live_simple,multiple`. Other supported non-multi-turn categories +include `parallel`, `live_parallel`, `parallel_multiple`, +`live_parallel_multiple`, `irrelevance`, `live_irrelevance`, +`live_relevance`, `java`, `javascript`, and `rest`. Multi-turn categories +are not yet supported. + +The dataset class normalizes BFCL's loose schema dialect (`dict` → +`object`, `float` → `number`, `tuple` → `array`, `any` → `string`) so +modern grammar backends accept the translated tool definitions. + #### Other HuggingFaceDataset Examples ```bash diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 95092734f3..d1a56e83cd 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -504,6 +504,13 @@ Flags: `--tool-call-parser pythonic --chat-template {see_above}` !!! warning Llama's smaller models frequently fail to emit tool calls in the correct format. Results may vary depending on the model. +## Benchmarking Tool-Calling Performance + +To measure serving latency and throughput on realistic tool-calling traffic, +use the BFCL (Berkeley Function Calling Leaderboard) dataset with +`vllm bench serve`. See the [BFCL benchmark example](../benchmarking/cli.md#bfcl-tool-calling-benchmark) +for the full server + client commands. + ## How to Write a Tool Parser Plugin A tool parser plugin is a Python file containing one or more ToolParser implementations. You can write a ToolParser similar to the `Hermes2ProToolParser` in [vllm/tool_parsers/hermes_tool_parser.py](../../vllm/tool_parsers/hermes_tool_parser.py). diff --git a/tests/benchmarks/test_bfcl_dataset.py b/tests/benchmarks/test_bfcl_dataset.py new file mode 100644 index 0000000000..e5110c5098 --- /dev/null +++ b/tests/benchmarks/test_bfcl_dataset.py @@ -0,0 +1,341 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +from vllm.benchmarks.datasets import BFCLDataset, get_samples + + +def _patch_hf_api(side_effect): + """Return a patch context that swaps `hf_api()` to a stub whose + `.hf_hub_download` attribute uses `side_effect`.""" + fake_api = MagicMock() + fake_api.hf_hub_download.side_effect = side_effect + return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api) + + +@pytest.fixture(scope="session") +def hf_tokenizer() -> PreTrainedTokenizerBase: + return AutoTokenizer.from_pretrained("gpt2") + + +_FAKE_ROWS = { + "simple": [ + { + "id": "simple_0", + "question": [ + [ + { + "role": "user", + "content": "What is 2+2?", + } + ] + ], + "function": [ + { + "name": "add", + "description": "Add two numbers.", + "parameters": { + "type": "dict", + "properties": { + "a": {"type": "integer", "description": "first"}, + "b": {"type": "float", "description": "second"}, + }, + "required": ["a", "b"], + }, + } + ], + }, + ], + "live_simple": [ + { + "id": "live_simple_0", + "question": [[{"role": "user", "content": "Tell me the weather."}]], + "function": [ + { + "name": "get_weather", + "description": "Get weather.", + "parameters": { + "type": "dict", + "properties": { + "city": {"type": "any", "description": "city"}, + "coords": {"type": "tuple", "description": "coords"}, + }, + "required": ["city"], + }, + } + ], + }, + ], +} + + +def _write_fake_files(tmp_path: Path) -> dict[str, Path]: + """Write fake BFCL JSONL files mimicking the HF repo layout.""" + paths = {} + for category, rows in _FAKE_ROWS.items(): + p = tmp_path / f"BFCL_v3_{category}.json" + with p.open("w") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + paths[category] = p + return paths + + +def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace: + return argparse.Namespace( + dataset_name="hf", + dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard", + hf_name=None, + hf_subset=None, + hf_split=None, + hf_output_len=64, + disable_shuffle=True, + num_prompts=2, + no_oversample=False, + no_stream=True, + seed=0, + request_id_prefix="", + trust_remote_code=False, + skip_chat_template=False, + enable_multimodal_chat=False, + backend="openai-chat", + bfcl_categories=categories, + ) + + +@pytest.mark.benchmark +def test_bfcl_dataset_translates_schema_and_attaches_tools( + hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path +) -> None: + """BFCLDataset should translate schemas to OpenAI tool format, set + `messages` directly on SampleRequest, and attach tools/tool_choice via + request_overrides.""" + paths = _write_fake_files(tmp_path) + + def fake_download(_repo, filename, **_kwargs): + category = filename.removeprefix("BFCL_v3_").removesuffix(".json") + return str(paths[category]) + + args = _args_for_bfcl(categories=["simple", "live_simple"]) + + with _patch_hf_api(fake_download): + samples = get_samples(args, hf_tokenizer) + + assert len(samples) == 2 + for s in samples: + assert s.chat_messages is not None + assert isinstance(s.chat_messages, list) + assert s.chat_messages[0]["role"] == "user" + assert s.request_overrides is not None + assert "tools" in s.request_overrides + assert s.request_overrides["tool_choice"] == "auto" + # messages must NOT leak into request_overrides — it has its own + # typed field on SampleRequest. + assert "messages" not in s.request_overrides + tools = s.request_overrides["tools"] + assert len(tools) == 1 + tool = tools[0] + assert tool["type"] == "function" + # Translated schema: dict -> object, float -> number, + # any -> string, tuple -> array. + params = tool["function"]["parameters"] + assert params["type"] == "object" + for prop in params["properties"].values(): + assert prop["type"] in {"integer", "number", "string", "array"} + + +@pytest.mark.benchmark +def test_bfcl_dataset_requires_openai_chat_backend( + hf_tokenizer: PreTrainedTokenizerBase, +) -> None: + args = _args_for_bfcl(categories=["simple"]) + args.backend = "openai" + + with pytest.raises(ValueError, match="openai-chat"): + get_samples(args, hf_tokenizer) + + +@pytest.mark.benchmark +def test_bfcl_dataset_missing_category_raises_clear_error( + hf_tokenizer: PreTrainedTokenizerBase, +) -> None: + """A typo'd category should produce an actionable ValueError, not an + opaque huggingface_hub exception.""" + from huggingface_hub.errors import EntryNotFoundError + + args = _args_for_bfcl(categories=["simpl"]) # typo + + def raise_missing(_repo, filename, **_kwargs): + raise EntryNotFoundError(f"404 Not Found: {filename}") + + with ( + _patch_hf_api(raise_missing), + pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"), + ): + get_samples(args, hf_tokenizer) + + +@pytest.mark.benchmark +def test_chat_backend_uses_messages_field_when_set() -> None: + """When RequestFuncInput.chat_messages is set, the chat backend must use + it verbatim and skip default content construction from `prompt`.""" + import asyncio + + from vllm.benchmarks.lib.endpoint_request_func import ( + RequestFuncInput, + async_request_openai_chat_completions, + ) + + captured: dict = {} + + class _FakeResp: + status = 500 + reason = "stop-after-capture" + content = None + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + class _FakeSession: + def post(self, url, json, headers): # noqa: A002 + captured["url"] = url + captured["payload"] = json + return _FakeResp() + + messages = [ + {"role": "system", "content": "you are helpful"}, + {"role": "user", "content": "call add(3, 4)"}, + ] + req = RequestFuncInput( + prompt="IGNORED", + api_url="http://localhost:0/v1/chat/completions", + prompt_len=10, + output_len=16, + model="test-model", + chat_messages=messages, + extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]}, + ) + + asyncio.run( + async_request_openai_chat_completions( + request_func_input=req, session=_FakeSession() + ) + ) + + payload = captured["payload"] + assert payload["messages"] is messages, ( + "chat backend must forward RequestFuncInput.chat_messages verbatim " + "instead of constructing a default user message from `prompt`" + ) + # extra_body still merges in as before (shallow, per-request wins). + assert payload["tools"][0]["function"]["name"] == "add" + + +@pytest.mark.benchmark +def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None: + """prompt_len must reflect tokens from both messages *and* tool schemas, + so percentile buckets and input-distribution summaries aren't biased + low for tool-heavy traffic.""" + paths = _write_fake_files(tmp_path) + + def fake_download(_repo, filename, **_kwargs): + category = filename.removeprefix("BFCL_v3_").removesuffix(".json") + return str(paths[category]) + + captured: dict = {} + + class _FakeTokenizer: + def apply_chat_template( + self, messages, tools=None, tokenize=False, add_generation_prompt=True + ): + captured["tools"] = tools + base = " ".join(m.get("content", "") for m in messages) + tool_text = json.dumps(tools) if tools else "" + return base + " " + tool_text + + def __call__(self, text): + # 1 "token" per whitespace-separated word. + return type("Enc", (), {"input_ids": text.split()})() + + fake = _FakeTokenizer() + args = _args_for_bfcl(categories=["simple"]) + args.num_prompts = 1 + + with _patch_hf_api(fake_download): + samples = get_samples(args, fake) + + assert len(samples) == 1 + assert captured["tools"] is not None, ( + "apply_chat_template must be called with tools= so the schema " + "contributes to the prompt-length estimate" + ) + assert len(captured["tools"]) == 1 + assert captured["tools"][0]["function"]["name"] == "add" + + # Sanity: prompt_len exceeds a messages-only estimate. The fake row's + # user message is "What is 2+2?" (3 whitespace-separated tokens). + assert samples[0].prompt_len > 3 + + +@pytest.mark.benchmark +def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools( + tmp_path: Path, +) -> None: + """Older tokenizers don't accept tools=; fallback must still produce a + non-zero prompt_len without crashing.""" + paths = _write_fake_files(tmp_path) + + def fake_download(_repo, filename, **_kwargs): + category = filename.removeprefix("BFCL_v3_").removesuffix(".json") + return str(paths[category]) + + class _LegacyTokenizer: + def apply_chat_template(self, messages, **kwargs): + if "tools" in kwargs: + raise TypeError("unexpected keyword argument 'tools'") + return " ".join(m.get("content", "") for m in messages) + + def __call__(self, text): + return type("Enc", (), {"input_ids": text.split()})() + + args = _args_for_bfcl(categories=["simple"]) + args.num_prompts = 1 + + with _patch_hf_api(fake_download): + samples = get_samples(args, _LegacyTokenizer()) + + assert len(samples) == 1 + assert samples[0].prompt_len > 0 + + +@pytest.mark.benchmark +def test_bfcl_schema_translation_is_recursive() -> None: + """_translate_schema must recurse into nested properties.""" + input_schema = { + "type": "dict", + "properties": { + "nested": { + "type": "dict", + "properties": { + "value": {"type": "float"}, + "tags": {"type": "tuple", "items": {"type": "any"}}, + }, + } + }, + } + out = BFCLDataset._translate_schema(input_schema) + assert out["type"] == "object" + assert out["properties"]["nested"]["type"] == "object" + assert out["properties"]["nested"]["properties"]["value"]["type"] == "number" + assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array" + nested_props = out["properties"]["nested"]["properties"] + assert nested_props["tags"]["items"]["type"] == "string" diff --git a/vllm/benchmarks/datasets/__init__.py b/vllm/benchmarks/datasets/__init__.py index b989958edc..b003ee4c05 100644 --- a/vllm/benchmarks/datasets/__init__.py +++ b/vllm/benchmarks/datasets/__init__.py @@ -6,6 +6,7 @@ from vllm.benchmarks.datasets.datasets import ( AIMODataset, ASRDataset, BenchmarkDataset, + BFCLDataset, BlazeditDataset, BurstGPTDataset, ConversationDataset, @@ -49,6 +50,7 @@ __all__ = [ "AIMODataset", "ASRDataset", "BenchmarkDataset", + "BFCLDataset", "BlazeditDataset", "BurstGPTDataset", "ConversationDataset", diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 59e2aa578c..3dcff477c4 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -86,6 +86,14 @@ class SampleRequest: lora_request: LoRARequest | None = None request_id: str | None = None timestamp: float | None = None + # Pre-built chat messages. When set, the chat backend uses this list + # directly and skips constructing messages from `prompt` + multimodal + # content. Mutually exclusive with the `prompt`-based path. + chat_messages: list[dict[str, Any]] | None = None + # Per-request fields merged into the request body (e.g. tools, + # tool_choice, response_format). Shallow-merged with --extra-body at + # dispatch time; per-request keys win. + request_overrides: dict | None = None # ----------------------------------------------------------------------------- @@ -1822,6 +1830,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser): "from the sampled HF dataset.", ) + bfcl_group = parser.add_argument_group( + "BFCL dataset options", description=BFCLDataset.__doc__ + ) + bfcl_group.add_argument( + "--bfcl-categories", + type=lambda s: [c.strip() for c in s.split(",") if c.strip()], + default=None, + help="Comma-separated list of BFCL v3 category names (without the " + "'BFCL_v3_' prefix or '.json' suffix) to sample from, e.g. " + "'simple,live_simple,multiple'. Defaults to " + f"'{','.join(BFCLDataset.DEFAULT_CATEGORIES)}'.", + ) + prefix_repetition_group = parser.add_argument_group( "prefix repetition dataset options" ) @@ -2249,6 +2270,20 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: dataset_class = MMStarDataset args.hf_split = args.hf_split if args.hf_split else "val" args.hf_subset = None + elif ( + args.dataset_path in BFCLDataset.SUPPORTED_DATASET_PATHS + or args.hf_name in BFCLDataset.SUPPORTED_DATASET_PATHS + ): + if args.backend != "openai-chat": + raise ValueError( + "BFCL dataset requires the 'openai-chat' backend because " + "it sends per-request tool schemas via chat completions." + ) + dataset_class = BFCLDataset + # BFCL does not use HF splits/subsets; stub values for base init. + args.hf_split = args.hf_split if args.hf_split else "train" + args.hf_subset = None + hf_kwargs = {"categories": args.bfcl_categories} else: supported_datasets = set( [ @@ -4320,6 +4355,221 @@ class MMStarDataset(HuggingFaceDataset): return sampled_requests +# ----------------------------------------------------------------------------- +# BFCL (Berkeley Function Calling Leaderboard) Dataset Implementation +# ----------------------------------------------------------------------------- + + +class BFCLDataset(HuggingFaceDataset): + """Berkeley Function Calling Leaderboard dataset. + + https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard + + BFCL ships one JSON-lines file per category at the repo root (e.g. + ``BFCL_v3_simple.json``, ``BFCL_v3_live_simple.json``) rather than a + single HuggingFace split. Each record has ``{id, question, function}`` + where ``function`` uses a non-OpenAI schema dialect (``"type": "dict"``). + + This dataset loader: + - downloads the selected per-category files via ``hf_hub_download`` + and interleaves rows round-robin so sampling is balanced + - translates BFCL function schemas to OpenAI tool format + - sets :attr:`SampleRequest.chat_messages` directly and attaches + ``tools`` / ``tool_choice`` via :attr:`SampleRequest.request_overrides`, + producing production-alike tool calling traffic when used with an + ``openai-chat`` backend + """ + + DEFAULT_OUTPUT_LEN = 512 + DEFAULT_CATEGORIES = ("simple", "live_simple", "multiple") + SUPPORTED_DATASET_PATHS = { + "gorilla-llm/Berkeley-Function-Calling-Leaderboard", + } + IS_MULTIMODAL = False + + # BFCL primitive type names that are not valid JSON Schema types. + # Map them to the closest JSON Schema equivalent so that grammar + # backends (xgrammar, outlines) accept the translated tool schema. + _TYPE_REMAP = { + "dict": "object", + "float": "number", + "tuple": "array", + "any": "string", + } + + def load_data(self) -> None: + """Defer loading to :meth:`sample` where categories are known.""" + self.data = None + + def _resolve_categories(self, categories: list[str] | None) -> list[str]: + if not categories: + return list(self.DEFAULT_CATEGORIES) + resolved: list[str] = [] + for c in categories: + c = c.strip() + if not c: + continue + resolved.append(c) + return resolved or list(self.DEFAULT_CATEGORIES) + + def _load_category(self, category: str) -> list[dict]: + # Local import: huggingface_hub.errors is a small module and + # importing at call site keeps module import cheap for users who + # never touch BFCL. + from huggingface_hub.errors import EntryNotFoundError + + filename = f"BFCL_v3_{category}.json" + try: + path = hf_api().hf_hub_download( + self.dataset_path, filename, repo_type="dataset" + ) + except EntryNotFoundError as e: + defaults = ", ".join(self.DEFAULT_CATEGORIES) + raise ValueError( + f"BFCL category '{category}' not found: file '{filename}' " + f"does not exist in {self.dataset_path}. Check --bfcl-categories " + f"(defaults: {defaults})." + ) from e + rows: list[dict] = [] + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + @classmethod + def _translate_schema(cls, node: Any) -> Any: + """Recursively translate BFCL-flavored JSON schema to strict JSON Schema.""" + if isinstance(node, dict): + translated = {k: cls._translate_schema(v) for k, v in node.items()} + t = translated.get("type") + if isinstance(t, str) and t in cls._TYPE_REMAP: + translated["type"] = cls._TYPE_REMAP[t] + return translated + if isinstance(node, list): + return [cls._translate_schema(v) for v in node] + return node + + @classmethod + def _to_openai_tools(cls, functions: list[dict]) -> list[dict]: + tools: list[dict] = [] + for fn in functions: + translated = cls._translate_schema(fn) + tools.append({"type": "function", "function": translated}) + return tools + + def sample( + self, + tokenizer: TokenizerLike, + num_requests: int, + request_id_prefix: str = "", + no_oversample: bool = False, + output_len: int | None = None, + categories: list[str] | None = None, + **kwargs, + ) -> list[SampleRequest]: + output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN + categories = self._resolve_categories(categories) + + per_category_rows: list[list[dict]] = [ + self._load_category(c) for c in categories + ] + # Round-robin interleave so that when --disable-shuffle is set, + # taking the first num_requests rows still yields balanced category + # coverage. When shuffle is on (the default) this ordering is + # randomized away, which is fine — the subsequent random sample is + # already balanced in expectation. + interleaved: list[dict] = [] + max_len = max((len(rows) for rows in per_category_rows), default=0) + for i in range(max_len): + for rows in per_category_rows: + if i < len(rows): + interleaved.append(rows[i]) + + if not self.disable_shuffle: + rng = random.Random(self.random_seed) + rng.shuffle(interleaved) + + sampled_requests: list[SampleRequest] = [] + for row in interleaved: + if len(sampled_requests) >= num_requests: + break + question = row.get("question") + functions = row.get("function") + if not question or not functions: + continue + # BFCL question is list[list[dict]] — outer is turns. Use the + # first turn only; skip multi-turn categories in this loader. + if not isinstance(question, list) or not question: + continue + first_turn = question[0] + if not isinstance(first_turn, list) or not first_turn: + continue + messages = first_turn + if not isinstance(functions, list): + functions = [functions] + + tools = self._to_openai_tools(functions) + + # Best-effort prompt length for percentile bucketing. Pass tools= + # so modern chat templates (Llama 3.1+, Qwen, gpt-oss harmony, + # Hermes) render the tool schemas — without this, the estimate + # misses a significant chunk of the true input for BFCL traffic. + # Older tokenizers reject the kwarg; fall back to tools-free. + try: + rendered = tokenizer.apply_chat_template( + messages, + tools=tools, + tokenize=False, + add_generation_prompt=True, + ) + except TypeError: + rendered = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + except Exception as e: + # Unexpected template failure — prompt_len will fall back to a + # plain-text concatenation. Log so the degraded estimate is + # visible instead of silently skewing latency buckets. + logger.warning( + "BFCL: apply_chat_template failed for a sample, falling " + "back to plain-text prompt length: %s", + e, + exc_info=True, + ) + rendered = None + if rendered is not None: + prompt_len = len(tokenizer(rendered).input_ids) + else: + text = "\n".join(m.get("content", "") for m in messages) + prompt_len = len(tokenizer(text).input_ids) + + # The chat backend uses `messages` directly; `prompt` is only + # kept as a fallback string for display/debug. + prompt_text = messages[-1].get("content", "") if messages else "" + + sampled_requests.append( + SampleRequest( + prompt=prompt_text, + prompt_len=prompt_len, + expected_output_len=output_len, + request_id=request_id_prefix + str(len(sampled_requests)), + chat_messages=messages, + request_overrides={ + "tools": tools, + "tool_choice": "auto", + }, + ) + ) + + self.maybe_oversample_requests( + sampled_requests, num_requests, request_id_prefix, no_oversample + ) + return sampled_requests + + # ----------------------------------------------------------------------------- # Speed Bench Dataset Implementation # ----------------------------------------------------------------------------- diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index ab3ae7606a..d282033ba1 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -79,6 +79,10 @@ class RequestFuncInput: ignore_eos: bool = False language: str | None = None request_id: str | None = None + # Pre-built chat messages. When set, `async_request_openai_chat_completions` + # uses this list directly and skips building messages from `prompt` and + # `multi_modal_content`. + chat_messages: list[dict[str, Any]] | None = None @dataclass @@ -343,7 +347,10 @@ async def async_request_openai_chat_completions( api_url = request_func_input.api_url _validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions") - messages = _get_chat_messages(request_func_input, mm_position=mm_position) + if request_func_input.chat_messages is not None: + messages = request_func_input.chat_messages + else: + messages = _get_chat_messages(request_func_input, mm_position=mm_position) payload = { "model": request_func_input.model_name diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 6c55d287a3..8fa29f4df0 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -57,6 +57,14 @@ from vllm.utils.network_utils import join_host_port MILLISECONDS_TO_SECONDS_CONVERSION = 1000 + +def _merge_overrides(base: dict | None, override: dict | None) -> dict | None: + """Shallow merge; per-request wins. Returns None if both are empty.""" + if not base and not override: + return None + return {**(base or {}), **(override or {})} + + TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and ( shutil.which("gnuplot") is not None ) @@ -753,6 +761,8 @@ async def benchmark( input_requests[0].expected_output_len, input_requests[0].multi_modal_data, ) + test_extra_body = _merge_overrides(extra_body, input_requests[0].request_overrides) + test_chat_messages = input_requests[0].chat_messages assert ( test_mm_content is None @@ -773,7 +783,8 @@ async def benchmark( multi_modal_content=test_mm_content, ignore_eos=ignore_eos, extra_headers=extra_headers, - extra_body=extra_body, + extra_body=test_extra_body, + chat_messages=test_chat_messages, ) if ready_check_timeout_sec > 0: @@ -850,7 +861,8 @@ async def benchmark( multi_modal_content=test_mm_content, ignore_eos=ignore_eos, extra_headers=extra_headers, - extra_body=extra_body, + extra_body=test_extra_body, + chat_messages=test_chat_messages, ) profile_output = await request_func( request_func_input=profile_input, session=session @@ -927,6 +939,7 @@ async def benchmark( request.multi_modal_data, request.request_id, ) + per_request_extra_body = _merge_overrides(extra_body, request.request_overrides) req_model_id, req_model_name = model_id, model_name if lora_modules: req_lora_module = next(lora_modules) @@ -943,8 +956,9 @@ async def benchmark( multi_modal_content=mm_content, ignore_eos=ignore_eos, extra_headers=extra_headers, - extra_body=extra_body, + extra_body=per_request_extra_body, request_id=request_id, + chat_messages=request.chat_messages, ) tasks.append( asyncio.create_task(