[Bench] Add BFCL dataset for vllm bench serve tool-calling workloads (#42457)

Signed-off-by: Li Zhang <lzhanga@amazon.com>
Co-authored-by: Li Zhang <lzhanga@amazon.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Li
2026-06-09 23:59:18 -07:00
committed by GitHub
co-authored by Li Zhang Claude Chauncey
parent 7fdfa6441d
commit 89c6a41001
7 changed files with 666 additions and 4 deletions
+2
View File
@@ -6,6 +6,7 @@ from vllm.benchmarks.datasets.datasets import (
AIMODataset,
ASRDataset,
BenchmarkDataset,
BFCLDataset,
BlazeditDataset,
BurstGPTDataset,
ConversationDataset,
@@ -49,6 +50,7 @@ __all__ = [
"AIMODataset",
"ASRDataset",
"BenchmarkDataset",
"BFCLDataset",
"BlazeditDataset",
"BurstGPTDataset",
"ConversationDataset",
+250
View File
@@ -86,6 +86,14 @@ class SampleRequest:
lora_request: LoRARequest | None = None
request_id: str | None = None
timestamp: float | None = None
# Pre-built chat messages. When set, the chat backend uses this list
# directly and skips constructing messages from `prompt` + multimodal
# content. Mutually exclusive with the `prompt`-based path.
chat_messages: list[dict[str, Any]] | None = None
# Per-request fields merged into the request body (e.g. tools,
# tool_choice, response_format). Shallow-merged with --extra-body at
# dispatch time; per-request keys win.
request_overrides: dict | None = None
# -----------------------------------------------------------------------------
@@ -1822,6 +1830,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"from the sampled HF dataset.",
)
bfcl_group = parser.add_argument_group(
"BFCL dataset options", description=BFCLDataset.__doc__
)
bfcl_group.add_argument(
"--bfcl-categories",
type=lambda s: [c.strip() for c in s.split(",") if c.strip()],
default=None,
help="Comma-separated list of BFCL v3 category names (without the "
"'BFCL_v3_' prefix or '.json' suffix) to sample from, e.g. "
"'simple,live_simple,multiple'. Defaults to "
f"'{','.join(BFCLDataset.DEFAULT_CATEGORIES)}'.",
)
prefix_repetition_group = parser.add_argument_group(
"prefix repetition dataset options"
)
@@ -2249,6 +2270,20 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
dataset_class = MMStarDataset
args.hf_split = args.hf_split if args.hf_split else "val"
args.hf_subset = None
elif (
args.dataset_path in BFCLDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in BFCLDataset.SUPPORTED_DATASET_PATHS
):
if args.backend != "openai-chat":
raise ValueError(
"BFCL dataset requires the 'openai-chat' backend because "
"it sends per-request tool schemas via chat completions."
)
dataset_class = BFCLDataset
# BFCL does not use HF splits/subsets; stub values for base init.
args.hf_split = args.hf_split if args.hf_split else "train"
args.hf_subset = None
hf_kwargs = {"categories": args.bfcl_categories}
else:
supported_datasets = set(
[
@@ -4320,6 +4355,221 @@ class MMStarDataset(HuggingFaceDataset):
return sampled_requests
# -----------------------------------------------------------------------------
# BFCL (Berkeley Function Calling Leaderboard) Dataset Implementation
# -----------------------------------------------------------------------------
class BFCLDataset(HuggingFaceDataset):
"""Berkeley Function Calling Leaderboard dataset.
https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard
BFCL ships one JSON-lines file per category at the repo root (e.g.
``BFCL_v3_simple.json``, ``BFCL_v3_live_simple.json``) rather than a
single HuggingFace split. Each record has ``{id, question, function}``
where ``function`` uses a non-OpenAI schema dialect (``"type": "dict"``).
This dataset loader:
- downloads the selected per-category files via ``hf_hub_download``
and interleaves rows round-robin so sampling is balanced
- translates BFCL function schemas to OpenAI tool format
- sets :attr:`SampleRequest.chat_messages` directly and attaches
``tools`` / ``tool_choice`` via :attr:`SampleRequest.request_overrides`,
producing production-alike tool calling traffic when used with an
``openai-chat`` backend
"""
DEFAULT_OUTPUT_LEN = 512
DEFAULT_CATEGORIES = ("simple", "live_simple", "multiple")
SUPPORTED_DATASET_PATHS = {
"gorilla-llm/Berkeley-Function-Calling-Leaderboard",
}
IS_MULTIMODAL = False
# BFCL primitive type names that are not valid JSON Schema types.
# Map them to the closest JSON Schema equivalent so that grammar
# backends (xgrammar, outlines) accept the translated tool schema.
_TYPE_REMAP = {
"dict": "object",
"float": "number",
"tuple": "array",
"any": "string",
}
def load_data(self) -> None:
"""Defer loading to :meth:`sample` where categories are known."""
self.data = None
def _resolve_categories(self, categories: list[str] | None) -> list[str]:
if not categories:
return list(self.DEFAULT_CATEGORIES)
resolved: list[str] = []
for c in categories:
c = c.strip()
if not c:
continue
resolved.append(c)
return resolved or list(self.DEFAULT_CATEGORIES)
def _load_category(self, category: str) -> list[dict]:
# Local import: huggingface_hub.errors is a small module and
# importing at call site keeps module import cheap for users who
# never touch BFCL.
from huggingface_hub.errors import EntryNotFoundError
filename = f"BFCL_v3_{category}.json"
try:
path = hf_api().hf_hub_download(
self.dataset_path, filename, repo_type="dataset"
)
except EntryNotFoundError as e:
defaults = ", ".join(self.DEFAULT_CATEGORIES)
raise ValueError(
f"BFCL category '{category}' not found: file '{filename}' "
f"does not exist in {self.dataset_path}. Check --bfcl-categories "
f"(defaults: {defaults})."
) from e
rows: list[dict] = []
with open(path) as f:
for line in f:
line = line.strip()
if not line:
continue
rows.append(json.loads(line))
return rows
@classmethod
def _translate_schema(cls, node: Any) -> Any:
"""Recursively translate BFCL-flavored JSON schema to strict JSON Schema."""
if isinstance(node, dict):
translated = {k: cls._translate_schema(v) for k, v in node.items()}
t = translated.get("type")
if isinstance(t, str) and t in cls._TYPE_REMAP:
translated["type"] = cls._TYPE_REMAP[t]
return translated
if isinstance(node, list):
return [cls._translate_schema(v) for v in node]
return node
@classmethod
def _to_openai_tools(cls, functions: list[dict]) -> list[dict]:
tools: list[dict] = []
for fn in functions:
translated = cls._translate_schema(fn)
tools.append({"type": "function", "function": translated})
return tools
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
output_len: int | None = None,
categories: list[str] | None = None,
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
categories = self._resolve_categories(categories)
per_category_rows: list[list[dict]] = [
self._load_category(c) for c in categories
]
# Round-robin interleave so that when --disable-shuffle is set,
# taking the first num_requests rows still yields balanced category
# coverage. When shuffle is on (the default) this ordering is
# randomized away, which is fine — the subsequent random sample is
# already balanced in expectation.
interleaved: list[dict] = []
max_len = max((len(rows) for rows in per_category_rows), default=0)
for i in range(max_len):
for rows in per_category_rows:
if i < len(rows):
interleaved.append(rows[i])
if not self.disable_shuffle:
rng = random.Random(self.random_seed)
rng.shuffle(interleaved)
sampled_requests: list[SampleRequest] = []
for row in interleaved:
if len(sampled_requests) >= num_requests:
break
question = row.get("question")
functions = row.get("function")
if not question or not functions:
continue
# BFCL question is list[list[dict]] — outer is turns. Use the
# first turn only; skip multi-turn categories in this loader.
if not isinstance(question, list) or not question:
continue
first_turn = question[0]
if not isinstance(first_turn, list) or not first_turn:
continue
messages = first_turn
if not isinstance(functions, list):
functions = [functions]
tools = self._to_openai_tools(functions)
# Best-effort prompt length for percentile bucketing. Pass tools=
# so modern chat templates (Llama 3.1+, Qwen, gpt-oss harmony,
# Hermes) render the tool schemas — without this, the estimate
# misses a significant chunk of the true input for BFCL traffic.
# Older tokenizers reject the kwarg; fall back to tools-free.
try:
rendered = tokenizer.apply_chat_template(
messages,
tools=tools,
tokenize=False,
add_generation_prompt=True,
)
except TypeError:
rendered = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
except Exception as e:
# Unexpected template failure — prompt_len will fall back to a
# plain-text concatenation. Log so the degraded estimate is
# visible instead of silently skewing latency buckets.
logger.warning(
"BFCL: apply_chat_template failed for a sample, falling "
"back to plain-text prompt length: %s",
e,
exc_info=True,
)
rendered = None
if rendered is not None:
prompt_len = len(tokenizer(rendered).input_ids)
else:
text = "\n".join(m.get("content", "") for m in messages)
prompt_len = len(tokenizer(text).input_ids)
# The chat backend uses `messages` directly; `prompt` is only
# kept as a fallback string for display/debug.
prompt_text = messages[-1].get("content", "") if messages else ""
sampled_requests.append(
SampleRequest(
prompt=prompt_text,
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(len(sampled_requests)),
chat_messages=messages,
request_overrides={
"tools": tools,
"tool_choice": "auto",
},
)
)
self.maybe_oversample_requests(
sampled_requests, num_requests, request_id_prefix, no_oversample
)
return sampled_requests
# -----------------------------------------------------------------------------
# Speed Bench Dataset Implementation
# -----------------------------------------------------------------------------
+8 -1
View File
@@ -79,6 +79,10 @@ class RequestFuncInput:
ignore_eos: bool = False
language: str | None = None
request_id: str | None = None
# Pre-built chat messages. When set, `async_request_openai_chat_completions`
# uses this list directly and skips building messages from `prompt` and
# `multi_modal_content`.
chat_messages: list[dict[str, Any]] | None = None
@dataclass
@@ -343,7 +347,10 @@ async def async_request_openai_chat_completions(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
if request_func_input.chat_messages is not None:
messages = request_func_input.chat_messages
else:
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
payload = {
"model": request_func_input.model_name
+17 -3
View File
@@ -57,6 +57,14 @@ from vllm.utils.network_utils import join_host_port
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
"""Shallow merge; per-request wins. Returns None if both are empty."""
if not base and not override:
return None
return {**(base or {}), **(override or {})}
TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and (
shutil.which("gnuplot") is not None
)
@@ -753,6 +761,8 @@ async def benchmark(
input_requests[0].expected_output_len,
input_requests[0].multi_modal_data,
)
test_extra_body = _merge_overrides(extra_body, input_requests[0].request_overrides)
test_chat_messages = input_requests[0].chat_messages
assert (
test_mm_content is None
@@ -773,7 +783,8 @@ async def benchmark(
multi_modal_content=test_mm_content,
ignore_eos=ignore_eos,
extra_headers=extra_headers,
extra_body=extra_body,
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
if ready_check_timeout_sec > 0:
@@ -850,7 +861,8 @@ async def benchmark(
multi_modal_content=test_mm_content,
ignore_eos=ignore_eos,
extra_headers=extra_headers,
extra_body=extra_body,
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
profile_output = await request_func(
request_func_input=profile_input, session=session
@@ -927,6 +939,7 @@ async def benchmark(
request.multi_modal_data,
request.request_id,
)
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
req_model_id, req_model_name = model_id, model_name
if lora_modules:
req_lora_module = next(lora_modules)
@@ -943,8 +956,9 @@ async def benchmark(
multi_modal_content=mm_content,
ignore_eos=ignore_eos,
extra_headers=extra_headers,
extra_body=extra_body,
extra_body=per_request_extra_body,
request_id=request_id,
chat_messages=request.chat_messages,
)
tasks.append(
asyncio.create_task(