[Bench] Add BFCL dataset for vllm bench serve tool-calling workloads (#42457)

Signed-off-by: Li Zhang <lzhanga@amazon.com>
Co-authored-by: Li Zhang <lzhanga@amazon.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

docs/benchmarking/cli.md

@@ -405,6 +405,47 @@ vllm bench serve \
 
 Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card.
 
+#### BFCL (Tool-Calling) Benchmark
+
+The Berkeley Function Calling Leaderboard (BFCL) dataset measures serving
+latency and throughput on realistic tool-calling traffic. Each request
+carries a per-sample `tools` schema and chat history, so the server must
+expose `/v1/chat/completions` with an auto-tool-choice parser enabled.
+The benchmark client always uses the `openai-chat` backend.
+
+Start a tool-parser-enabled server, then run the bench. For example, with
+`gpt-oss-20b`:
+
+```bash
+# Server
+vllm serve openai/gpt-oss-20b \
+    --enable-auto-tool-choice \
+    --tool-call-parser openai \
+    --reasoning-parser openai_gptoss
+
+# Client
+vllm bench serve \
+    --backend openai-chat \
+    --endpoint /v1/chat/completions \
+    --model openai/gpt-oss-20b \
+    --dataset-name hf \
+    --dataset-path gorilla-llm/Berkeley-Function-Calling-Leaderboard \
+    --bfcl-categories simple,live_simple,multiple \
+    --num-prompts 200
+```
+
+`--bfcl-categories` is a comma-separated list of BFCL v3 category names
+(without the `BFCL_v3_` prefix or `.json` suffix). Defaults to
+`simple,live_simple,multiple`. Other supported non-multi-turn categories
+include `parallel`, `live_parallel`, `parallel_multiple`,
+`live_parallel_multiple`, `irrelevance`, `live_irrelevance`,
+`live_relevance`, `java`, `javascript`, and `rest`. Multi-turn categories
+are not yet supported.
+
+The dataset class normalizes BFCL's loose schema dialect (`dict` →
+`object`, `float` → `number`, `tuple` → `array`, `any` → `string`) so
+modern grammar backends accept the translated tool definitions.
+
 #### Other HuggingFaceDataset Examples
 
 ```bash

docs/features/tool_calling.md

@@ -504,6 +504,13 @@ Flags: `--tool-call-parser pythonic --chat-template {see_above}`
 !!! warning
     Llama's smaller models frequently fail to emit tool calls in the correct format. Results may vary depending on the model.
 
+## Benchmarking Tool-Calling Performance
+
+To measure serving latency and throughput on realistic tool-calling traffic,
+use the BFCL (Berkeley Function Calling Leaderboard) dataset with
+`vllm bench serve`. See the [BFCL benchmark example](../benchmarking/cli.md#bfcl-tool-calling-benchmark)
+for the full server + client commands.
+
 ## How to Write a Tool Parser Plugin
 
 A tool parser plugin is a Python file containing one or more ToolParser implementations. You can write a ToolParser similar to the `Hermes2ProToolParser` in [vllm/tool_parsers/hermes_tool_parser.py](../../vllm/tool_parsers/hermes_tool_parser.py).

tests/benchmarks/test_bfcl_dataset.py

@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+import json
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
+
+from vllm.benchmarks.datasets import BFCLDataset, get_samples
+
+
+def _patch_hf_api(side_effect):
+    """Return a patch context that swaps `hf_api()` to a stub whose
+    `.hf_hub_download` attribute uses `side_effect`."""
+    fake_api = MagicMock()
+    fake_api.hf_hub_download.side_effect = side_effect
+    return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
+
+
+@pytest.fixture(scope="session")
+def hf_tokenizer() -> PreTrainedTokenizerBase:
+    return AutoTokenizer.from_pretrained("gpt2")
+
+
+_FAKE_ROWS = {
+    "simple": [
+        {
+            "id": "simple_0",
+            "question": [
+                [
+                    {
+                        "role": "user",
+                        "content": "What is 2+2?",
+                    }
+                ]
+            ],
+            "function": [
+                {
+                    "name": "add",
+                    "description": "Add two numbers.",
+                    "parameters": {
+                        "type": "dict",
+                        "properties": {
+                            "a": {"type": "integer", "description": "first"},
+                            "b": {"type": "float", "description": "second"},
+                        },
+                        "required": ["a", "b"],
+                    },
+                }
+            ],
+        },
+    ],
+    "live_simple": [
+        {
+            "id": "live_simple_0",
+            "question": [[{"role": "user", "content": "Tell me the weather."}]],
+            "function": [
+                {
+                    "name": "get_weather",
+                    "description": "Get weather.",
+                    "parameters": {
+                        "type": "dict",
+                        "properties": {
+                            "city": {"type": "any", "description": "city"},
+                            "coords": {"type": "tuple", "description": "coords"},
+                        },
+                        "required": ["city"],
+                    },
+                }
+            ],
+        },
+    ],
+}
+
+
+def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
+    """Write fake BFCL JSONL files mimicking the HF repo layout."""
+    paths = {}
+    for category, rows in _FAKE_ROWS.items():
+        p = tmp_path / f"BFCL_v3_{category}.json"
+        with p.open("w") as f:
+            for row in rows:
+                f.write(json.dumps(row) + "\n")
+        paths[category] = p
+    return paths
+
+
+def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
+    return argparse.Namespace(
+        dataset_name="hf",
+        dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
+        hf_name=None,
+        hf_subset=None,
+        hf_split=None,
+        hf_output_len=64,
+        disable_shuffle=True,
+        num_prompts=2,
+        no_oversample=False,
+        no_stream=True,
+        seed=0,
+        request_id_prefix="",
+        trust_remote_code=False,
+        skip_chat_template=False,
+        enable_multimodal_chat=False,
+        backend="openai-chat",
+        bfcl_categories=categories,
+    )
+
+
+@pytest.mark.benchmark
+def test_bfcl_dataset_translates_schema_and_attaches_tools(
+    hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
+) -> None:
+    """BFCLDataset should translate schemas to OpenAI tool format, set
+    `messages` directly on SampleRequest, and attach tools/tool_choice via
+    request_overrides."""
+    paths = _write_fake_files(tmp_path)
+
+    def fake_download(_repo, filename, **_kwargs):
+        category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
+        return str(paths[category])
+
+    args = _args_for_bfcl(categories=["simple", "live_simple"])
+
+    with _patch_hf_api(fake_download):
+        samples = get_samples(args, hf_tokenizer)
+
+    assert len(samples) == 2
+    for s in samples:
+        assert s.chat_messages is not None
+        assert isinstance(s.chat_messages, list)
+        assert s.chat_messages[0]["role"] == "user"
+        assert s.request_overrides is not None
+        assert "tools" in s.request_overrides
+        assert s.request_overrides["tool_choice"] == "auto"
+        # messages must NOT leak into request_overrides — it has its own
+        # typed field on SampleRequest.
+        assert "messages" not in s.request_overrides
+        tools = s.request_overrides["tools"]
+        assert len(tools) == 1
+        tool = tools[0]
+        assert tool["type"] == "function"
+        # Translated schema: dict -> object, float -> number,
+        # any -> string, tuple -> array.
+        params = tool["function"]["parameters"]
+        assert params["type"] == "object"
+        for prop in params["properties"].values():
+            assert prop["type"] in {"integer", "number", "string", "array"}
+
+
+@pytest.mark.benchmark
+def test_bfcl_dataset_requires_openai_chat_backend(
+    hf_tokenizer: PreTrainedTokenizerBase,
+) -> None:
+    args = _args_for_bfcl(categories=["simple"])
+    args.backend = "openai"
+
+    with pytest.raises(ValueError, match="openai-chat"):
+        get_samples(args, hf_tokenizer)
+
+
+@pytest.mark.benchmark
+def test_bfcl_dataset_missing_category_raises_clear_error(
+    hf_tokenizer: PreTrainedTokenizerBase,
+) -> None:
+    """A typo'd category should produce an actionable ValueError, not an
+    opaque huggingface_hub exception."""
+    from huggingface_hub.errors import EntryNotFoundError
+
+    args = _args_for_bfcl(categories=["simpl"])  # typo
+
+    def raise_missing(_repo, filename, **_kwargs):
+        raise EntryNotFoundError(f"404 Not Found: {filename}")
+
+    with (
+        _patch_hf_api(raise_missing),
+        pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
+    ):
+        get_samples(args, hf_tokenizer)
+
+
+@pytest.mark.benchmark
+def test_chat_backend_uses_messages_field_when_set() -> None:
+    """When RequestFuncInput.chat_messages is set, the chat backend must use
+    it verbatim and skip default content construction from `prompt`."""
+    import asyncio
+
+    from vllm.benchmarks.lib.endpoint_request_func import (
+        RequestFuncInput,
+        async_request_openai_chat_completions,
+    )
+
+    captured: dict = {}
+
+    class _FakeResp:
+        status = 500
+        reason = "stop-after-capture"
+        content = None
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+    class _FakeSession:
+        def post(self, url, json, headers):  # noqa: A002
+            captured["url"] = url
+            captured["payload"] = json
+            return _FakeResp()
+
+    messages = [
+        {"role": "system", "content": "you are helpful"},
+        {"role": "user", "content": "call add(3, 4)"},
+    ]
+    req = RequestFuncInput(
+        prompt="IGNORED",
+        api_url="http://localhost:0/v1/chat/completions",
+        prompt_len=10,
+        output_len=16,
+        model="test-model",
+        chat_messages=messages,
+        extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
+    )
+
+    asyncio.run(
+        async_request_openai_chat_completions(
+            request_func_input=req, session=_FakeSession()
+        )
+    )
+
+    payload = captured["payload"]
+    assert payload["messages"] is messages, (
+        "chat backend must forward RequestFuncInput.chat_messages verbatim "
+        "instead of constructing a default user message from `prompt`"
+    )
+    # extra_body still merges in as before (shallow, per-request wins).
+    assert payload["tools"][0]["function"]["name"] == "add"
+
+
+@pytest.mark.benchmark
+def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
+    """prompt_len must reflect tokens from both messages *and* tool schemas,
+    so percentile buckets and input-distribution summaries aren't biased
+    low for tool-heavy traffic."""
+    paths = _write_fake_files(tmp_path)
+
+    def fake_download(_repo, filename, **_kwargs):
+        category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
+        return str(paths[category])
+
+    captured: dict = {}
+
+    class _FakeTokenizer:
+        def apply_chat_template(
+            self, messages, tools=None, tokenize=False, add_generation_prompt=True
+        ):
+            captured["tools"] = tools
+            base = " ".join(m.get("content", "") for m in messages)
+            tool_text = json.dumps(tools) if tools else ""
+            return base + " " + tool_text
+
+        def __call__(self, text):
+            # 1 "token" per whitespace-separated word.
+            return type("Enc", (), {"input_ids": text.split()})()
+
+    fake = _FakeTokenizer()
+    args = _args_for_bfcl(categories=["simple"])
+    args.num_prompts = 1
+
+    with _patch_hf_api(fake_download):
+        samples = get_samples(args, fake)
+
+    assert len(samples) == 1
+    assert captured["tools"] is not None, (
+        "apply_chat_template must be called with tools= so the schema "
+        "contributes to the prompt-length estimate"
+    )
+    assert len(captured["tools"]) == 1
+    assert captured["tools"][0]["function"]["name"] == "add"
+
+    # Sanity: prompt_len exceeds a messages-only estimate. The fake row's
+    # user message is "What is 2+2?" (3 whitespace-separated tokens).
+    assert samples[0].prompt_len > 3
+
+
+@pytest.mark.benchmark
+def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
+    tmp_path: Path,
+) -> None:
+    """Older tokenizers don't accept tools=; fallback must still produce a
+    non-zero prompt_len without crashing."""
+    paths = _write_fake_files(tmp_path)
+
+    def fake_download(_repo, filename, **_kwargs):
+        category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
+        return str(paths[category])
+
+    class _LegacyTokenizer:
+        def apply_chat_template(self, messages, **kwargs):
+            if "tools" in kwargs:
+                raise TypeError("unexpected keyword argument 'tools'")
+            return " ".join(m.get("content", "") for m in messages)
+
+        def __call__(self, text):
+            return type("Enc", (), {"input_ids": text.split()})()
+
+    args = _args_for_bfcl(categories=["simple"])
+    args.num_prompts = 1
+
+    with _patch_hf_api(fake_download):
+        samples = get_samples(args, _LegacyTokenizer())
+
+    assert len(samples) == 1
+    assert samples[0].prompt_len > 0
+
+
+@pytest.mark.benchmark
+def test_bfcl_schema_translation_is_recursive() -> None:
+    """_translate_schema must recurse into nested properties."""
+    input_schema = {
+        "type": "dict",
+        "properties": {
+            "nested": {
+                "type": "dict",
+                "properties": {
+                    "value": {"type": "float"},
+                    "tags": {"type": "tuple", "items": {"type": "any"}},
+                },
+            }
+        },
+    }
+    out = BFCLDataset._translate_schema(input_schema)
+    assert out["type"] == "object"
+    assert out["properties"]["nested"]["type"] == "object"
+    assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
+    assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
+    nested_props = out["properties"]["nested"]["properties"]
+    assert nested_props["tags"]["items"]["type"] == "string"

vllm/benchmarks/datasets/__init__.py

@@ -6,6 +6,7 @@ from vllm.benchmarks.datasets.datasets import (
     AIMODataset,
     ASRDataset,
     BenchmarkDataset,
+    BFCLDataset,
     BlazeditDataset,
     BurstGPTDataset,
     ConversationDataset,
@@ -49,6 +50,7 @@ __all__ = [
     "AIMODataset",
     "ASRDataset",
     "BenchmarkDataset",
+    "BFCLDataset",
     "BlazeditDataset",
     "BurstGPTDataset",
     "ConversationDataset",

vllm/benchmarks/datasets/datasets.py

@@ -86,6 +86,14 @@ class SampleRequest:
     lora_request: LoRARequest | None = None
     request_id: str | None = None
     timestamp: float | None = None
+    # Pre-built chat messages. When set, the chat backend uses this list
+    # directly and skips constructing messages from `prompt` + multimodal
+    # content. Mutually exclusive with the `prompt`-based path.
+    chat_messages: list[dict[str, Any]] | None = None
+    # Per-request fields merged into the request body (e.g. tools,
+    # tool_choice, response_format). Shallow-merged with --extra-body at
+    # dispatch time; per-request keys win.
+    request_overrides: dict | None = None
 
 
 # -----------------------------------------------------------------------------
@@ -1822,6 +1830,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
         "from the sampled HF dataset.",
     )
 
+    bfcl_group = parser.add_argument_group(
+        "BFCL dataset options", description=BFCLDataset.__doc__
+    )
+    bfcl_group.add_argument(
+        "--bfcl-categories",
+        type=lambda s: [c.strip() for c in s.split(",") if c.strip()],
+        default=None,
+        help="Comma-separated list of BFCL v3 category names (without the "
+        "'BFCL_v3_' prefix or '.json' suffix) to sample from, e.g. "
+        "'simple,live_simple,multiple'. Defaults to "
+        f"'{','.join(BFCLDataset.DEFAULT_CATEGORIES)}'.",
+    )
+
     prefix_repetition_group = parser.add_argument_group(
         "prefix repetition dataset options"
     )
@@ -2249,6 +2270,20 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
             dataset_class = MMStarDataset
             args.hf_split = args.hf_split if args.hf_split else "val"
             args.hf_subset = None
+        elif (
+            args.dataset_path in BFCLDataset.SUPPORTED_DATASET_PATHS
+            or args.hf_name in BFCLDataset.SUPPORTED_DATASET_PATHS
+        ):
+            if args.backend != "openai-chat":
+                raise ValueError(
+                    "BFCL dataset requires the 'openai-chat' backend because "
+                    "it sends per-request tool schemas via chat completions."
+                )
+            dataset_class = BFCLDataset
+            # BFCL does not use HF splits/subsets; stub values for base init.
+            args.hf_split = args.hf_split if args.hf_split else "train"
+            args.hf_subset = None
+            hf_kwargs = {"categories": args.bfcl_categories}
         else:
             supported_datasets = set(
                 [
@@ -4320,6 +4355,221 @@ class MMStarDataset(HuggingFaceDataset):
         return sampled_requests
 
 
+# -----------------------------------------------------------------------------
+# BFCL (Berkeley Function Calling Leaderboard) Dataset Implementation
+# -----------------------------------------------------------------------------
+
+
+class BFCLDataset(HuggingFaceDataset):
+    """Berkeley Function Calling Leaderboard dataset.
+
+    https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard
+
+    BFCL ships one JSON-lines file per category at the repo root (e.g.
+    ``BFCL_v3_simple.json``, ``BFCL_v3_live_simple.json``) rather than a
+    single HuggingFace split. Each record has ``{id, question, function}``
+    where ``function`` uses a non-OpenAI schema dialect (``"type": "dict"``).
+
+    This dataset loader:
+      - downloads the selected per-category files via ``hf_hub_download``
+        and interleaves rows round-robin so sampling is balanced
+      - translates BFCL function schemas to OpenAI tool format
+      - sets :attr:`SampleRequest.chat_messages` directly and attaches
+        ``tools`` / ``tool_choice`` via :attr:`SampleRequest.request_overrides`,
+        producing production-alike tool calling traffic when used with an
+        ``openai-chat`` backend
+    """
+
+    DEFAULT_OUTPUT_LEN = 512
+    DEFAULT_CATEGORIES = ("simple", "live_simple", "multiple")
+    SUPPORTED_DATASET_PATHS = {
+        "gorilla-llm/Berkeley-Function-Calling-Leaderboard",
+    }
+    IS_MULTIMODAL = False
+
+    # BFCL primitive type names that are not valid JSON Schema types.
+    # Map them to the closest JSON Schema equivalent so that grammar
+    # backends (xgrammar, outlines) accept the translated tool schema.
+    _TYPE_REMAP = {
+        "dict": "object",
+        "float": "number",
+        "tuple": "array",
+        "any": "string",
+    }
+
+    def load_data(self) -> None:
+        """Defer loading to :meth:`sample` where categories are known."""
+        self.data = None
+
+    def _resolve_categories(self, categories: list[str] | None) -> list[str]:
+        if not categories:
+            return list(self.DEFAULT_CATEGORIES)
+        resolved: list[str] = []
+        for c in categories:
+            c = c.strip()
+            if not c:
+                continue
+            resolved.append(c)
+        return resolved or list(self.DEFAULT_CATEGORIES)
+
+    def _load_category(self, category: str) -> list[dict]:
+        # Local import: huggingface_hub.errors is a small module and
+        # importing at call site keeps module import cheap for users who
+        # never touch BFCL.
+        from huggingface_hub.errors import EntryNotFoundError
+
+        filename = f"BFCL_v3_{category}.json"
+        try:
+            path = hf_api().hf_hub_download(
+                self.dataset_path, filename, repo_type="dataset"
+            )
+        except EntryNotFoundError as e:
+            defaults = ", ".join(self.DEFAULT_CATEGORIES)
+            raise ValueError(
+                f"BFCL category '{category}' not found: file '{filename}' "
+                f"does not exist in {self.dataset_path}. Check --bfcl-categories "
+                f"(defaults: {defaults})."
+            ) from e
+        rows: list[dict] = []
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                rows.append(json.loads(line))
+        return rows
+
+    @classmethod
+    def _translate_schema(cls, node: Any) -> Any:
+        """Recursively translate BFCL-flavored JSON schema to strict JSON Schema."""
+        if isinstance(node, dict):
+            translated = {k: cls._translate_schema(v) for k, v in node.items()}
+            t = translated.get("type")
+            if isinstance(t, str) and t in cls._TYPE_REMAP:
+                translated["type"] = cls._TYPE_REMAP[t]
+            return translated
+        if isinstance(node, list):
+            return [cls._translate_schema(v) for v in node]
+        return node
+
+    @classmethod
+    def _to_openai_tools(cls, functions: list[dict]) -> list[dict]:
+        tools: list[dict] = []
+        for fn in functions:
+            translated = cls._translate_schema(fn)
+            tools.append({"type": "function", "function": translated})
+        return tools
+
+    def sample(
+        self,
+        tokenizer: TokenizerLike,
+        num_requests: int,
+        request_id_prefix: str = "",
+        no_oversample: bool = False,
+        output_len: int | None = None,
+        categories: list[str] | None = None,
+        **kwargs,
+    ) -> list[SampleRequest]:
+        output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
+        categories = self._resolve_categories(categories)
+
+        per_category_rows: list[list[dict]] = [
+            self._load_category(c) for c in categories
+        ]
+        # Round-robin interleave so that when --disable-shuffle is set,
+        # taking the first num_requests rows still yields balanced category
+        # coverage. When shuffle is on (the default) this ordering is
+        # randomized away, which is fine — the subsequent random sample is
+        # already balanced in expectation.
+        interleaved: list[dict] = []
+        max_len = max((len(rows) for rows in per_category_rows), default=0)
+        for i in range(max_len):
+            for rows in per_category_rows:
+                if i < len(rows):
+                    interleaved.append(rows[i])
+
+        if not self.disable_shuffle:
+            rng = random.Random(self.random_seed)
+            rng.shuffle(interleaved)
+
+        sampled_requests: list[SampleRequest] = []
+        for row in interleaved:
+            if len(sampled_requests) >= num_requests:
+                break
+            question = row.get("question")
+            functions = row.get("function")
+            if not question or not functions:
+                continue
+            # BFCL question is list[list[dict]] — outer is turns. Use the
+            # first turn only; skip multi-turn categories in this loader.
+            if not isinstance(question, list) or not question:
+                continue
+            first_turn = question[0]
+            if not isinstance(first_turn, list) or not first_turn:
+                continue
+            messages = first_turn
+            if not isinstance(functions, list):
+                functions = [functions]
+
+            tools = self._to_openai_tools(functions)
+
+            # Best-effort prompt length for percentile bucketing. Pass tools=
+            # so modern chat templates (Llama 3.1+, Qwen, gpt-oss harmony,
+            # Hermes) render the tool schemas — without this, the estimate
+            # misses a significant chunk of the true input for BFCL traffic.
+            # Older tokenizers reject the kwarg; fall back to tools-free.
+            try:
+                rendered = tokenizer.apply_chat_template(
+                    messages,
+                    tools=tools,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                )
+            except TypeError:
+                rendered = tokenizer.apply_chat_template(
+                    messages, tokenize=False, add_generation_prompt=True
+                )
+            except Exception as e:
+                # Unexpected template failure — prompt_len will fall back to a
+                # plain-text concatenation. Log so the degraded estimate is
+                # visible instead of silently skewing latency buckets.
+                logger.warning(
+                    "BFCL: apply_chat_template failed for a sample, falling "
+                    "back to plain-text prompt length: %s",
+                    e,
+                    exc_info=True,
+                )
+                rendered = None
+            if rendered is not None:
+                prompt_len = len(tokenizer(rendered).input_ids)
+            else:
+                text = "\n".join(m.get("content", "") for m in messages)
+                prompt_len = len(tokenizer(text).input_ids)
+
+            # The chat backend uses `messages` directly; `prompt` is only
+            # kept as a fallback string for display/debug.
+            prompt_text = messages[-1].get("content", "") if messages else ""
+
+            sampled_requests.append(
+                SampleRequest(
+                    prompt=prompt_text,
+                    prompt_len=prompt_len,
+                    expected_output_len=output_len,
+                    request_id=request_id_prefix + str(len(sampled_requests)),
+                    chat_messages=messages,
+                    request_overrides={
+                        "tools": tools,
+                        "tool_choice": "auto",
+                    },
+                )
+            )
+
+        self.maybe_oversample_requests(
+            sampled_requests, num_requests, request_id_prefix, no_oversample
+        )
+        return sampled_requests
+
+
 # -----------------------------------------------------------------------------
 # Speed Bench Dataset Implementation
 # -----------------------------------------------------------------------------

vllm/benchmarks/lib/endpoint_request_func.py

@@ -79,6 +79,10 @@ class RequestFuncInput:
     ignore_eos: bool = False
     language: str | None = None
     request_id: str | None = None
+    # Pre-built chat messages. When set, `async_request_openai_chat_completions`
+    # uses this list directly and skips building messages from `prompt` and
+    # `multi_modal_content`.
+    chat_messages: list[dict[str, Any]] | None = None
 
 
 @dataclass
@@ -343,7 +347,10 @@ async def async_request_openai_chat_completions(
     api_url = request_func_input.api_url
     _validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
 
-    messages = _get_chat_messages(request_func_input, mm_position=mm_position)
+    if request_func_input.chat_messages is not None:
+        messages = request_func_input.chat_messages
+    else:
+        messages = _get_chat_messages(request_func_input, mm_position=mm_position)
 
     payload = {
         "model": request_func_input.model_name

vllm/benchmarks/serve.py

@@ -57,6 +57,14 @@ from vllm.utils.network_utils import join_host_port
 
 MILLISECONDS_TO_SECONDS_CONVERSION = 1000
 
+
+def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
+    """Shallow merge; per-request wins. Returns None if both are empty."""
+    if not base and not override:
+        return None
+    return {**(base or {}), **(override or {})}
+
+
 TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and (
     shutil.which("gnuplot") is not None
 )
@@ -753,6 +761,8 @@ async def benchmark(
         input_requests[0].expected_output_len,
         input_requests[0].multi_modal_data,
     )
+    test_extra_body = _merge_overrides(extra_body, input_requests[0].request_overrides)
+    test_chat_messages = input_requests[0].chat_messages
 
     assert (
         test_mm_content is None
@@ -773,7 +783,8 @@ async def benchmark(
         multi_modal_content=test_mm_content,
         ignore_eos=ignore_eos,
         extra_headers=extra_headers,
-        extra_body=extra_body,
+        extra_body=test_extra_body,
+        chat_messages=test_chat_messages,
     )
 
     if ready_check_timeout_sec > 0:
@@ -850,7 +861,8 @@ async def benchmark(
             multi_modal_content=test_mm_content,
             ignore_eos=ignore_eos,
             extra_headers=extra_headers,
-            extra_body=extra_body,
+            extra_body=test_extra_body,
+            chat_messages=test_chat_messages,
         )
         profile_output = await request_func(
             request_func_input=profile_input, session=session
@@ -927,6 +939,7 @@ async def benchmark(
             request.multi_modal_data,
             request.request_id,
         )
+        per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
         req_model_id, req_model_name = model_id, model_name
         if lora_modules:
             req_lora_module = next(lora_modules)
@@ -943,8 +956,9 @@ async def benchmark(
             multi_modal_content=mm_content,
             ignore_eos=ignore_eos,
             extra_headers=extra_headers,
-            extra_body=extra_body,
+            extra_body=per_request_extra_body,
             request_id=request_id,
+            chat_messages=request.chat_messages,
         )
         tasks.append(
             asyncio.create_task(