[Bench] Add BFCL dataset for vllm bench serve tool-calling workloads (#42457)
Signed-off-by: Li Zhang <lzhanga@amazon.com> Co-authored-by: Li Zhang <lzhanga@amazon.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Li Zhang
Claude
Chauncey
parent
7fdfa6441d
commit
89c6a41001
@@ -6,6 +6,7 @@ from vllm.benchmarks.datasets.datasets import (
|
||||
AIMODataset,
|
||||
ASRDataset,
|
||||
BenchmarkDataset,
|
||||
BFCLDataset,
|
||||
BlazeditDataset,
|
||||
BurstGPTDataset,
|
||||
ConversationDataset,
|
||||
@@ -49,6 +50,7 @@ __all__ = [
|
||||
"AIMODataset",
|
||||
"ASRDataset",
|
||||
"BenchmarkDataset",
|
||||
"BFCLDataset",
|
||||
"BlazeditDataset",
|
||||
"BurstGPTDataset",
|
||||
"ConversationDataset",
|
||||
|
||||
@@ -86,6 +86,14 @@ class SampleRequest:
|
||||
lora_request: LoRARequest | None = None
|
||||
request_id: str | None = None
|
||||
timestamp: float | None = None
|
||||
# Pre-built chat messages. When set, the chat backend uses this list
|
||||
# directly and skips constructing messages from `prompt` + multimodal
|
||||
# content. Mutually exclusive with the `prompt`-based path.
|
||||
chat_messages: list[dict[str, Any]] | None = None
|
||||
# Per-request fields merged into the request body (e.g. tools,
|
||||
# tool_choice, response_format). Shallow-merged with --extra-body at
|
||||
# dispatch time; per-request keys win.
|
||||
request_overrides: dict | None = None
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -1822,6 +1830,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
|
||||
"from the sampled HF dataset.",
|
||||
)
|
||||
|
||||
bfcl_group = parser.add_argument_group(
|
||||
"BFCL dataset options", description=BFCLDataset.__doc__
|
||||
)
|
||||
bfcl_group.add_argument(
|
||||
"--bfcl-categories",
|
||||
type=lambda s: [c.strip() for c in s.split(",") if c.strip()],
|
||||
default=None,
|
||||
help="Comma-separated list of BFCL v3 category names (without the "
|
||||
"'BFCL_v3_' prefix or '.json' suffix) to sample from, e.g. "
|
||||
"'simple,live_simple,multiple'. Defaults to "
|
||||
f"'{','.join(BFCLDataset.DEFAULT_CATEGORIES)}'.",
|
||||
)
|
||||
|
||||
prefix_repetition_group = parser.add_argument_group(
|
||||
"prefix repetition dataset options"
|
||||
)
|
||||
@@ -2249,6 +2270,20 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
||||
dataset_class = MMStarDataset
|
||||
args.hf_split = args.hf_split if args.hf_split else "val"
|
||||
args.hf_subset = None
|
||||
elif (
|
||||
args.dataset_path in BFCLDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in BFCLDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
if args.backend != "openai-chat":
|
||||
raise ValueError(
|
||||
"BFCL dataset requires the 'openai-chat' backend because "
|
||||
"it sends per-request tool schemas via chat completions."
|
||||
)
|
||||
dataset_class = BFCLDataset
|
||||
# BFCL does not use HF splits/subsets; stub values for base init.
|
||||
args.hf_split = args.hf_split if args.hf_split else "train"
|
||||
args.hf_subset = None
|
||||
hf_kwargs = {"categories": args.bfcl_categories}
|
||||
else:
|
||||
supported_datasets = set(
|
||||
[
|
||||
@@ -4320,6 +4355,221 @@ class MMStarDataset(HuggingFaceDataset):
|
||||
return sampled_requests
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# BFCL (Berkeley Function Calling Leaderboard) Dataset Implementation
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class BFCLDataset(HuggingFaceDataset):
|
||||
"""Berkeley Function Calling Leaderboard dataset.
|
||||
|
||||
https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard
|
||||
|
||||
BFCL ships one JSON-lines file per category at the repo root (e.g.
|
||||
``BFCL_v3_simple.json``, ``BFCL_v3_live_simple.json``) rather than a
|
||||
single HuggingFace split. Each record has ``{id, question, function}``
|
||||
where ``function`` uses a non-OpenAI schema dialect (``"type": "dict"``).
|
||||
|
||||
This dataset loader:
|
||||
- downloads the selected per-category files via ``hf_hub_download``
|
||||
and interleaves rows round-robin so sampling is balanced
|
||||
- translates BFCL function schemas to OpenAI tool format
|
||||
- sets :attr:`SampleRequest.chat_messages` directly and attaches
|
||||
``tools`` / ``tool_choice`` via :attr:`SampleRequest.request_overrides`,
|
||||
producing production-alike tool calling traffic when used with an
|
||||
``openai-chat`` backend
|
||||
"""
|
||||
|
||||
DEFAULT_OUTPUT_LEN = 512
|
||||
DEFAULT_CATEGORIES = ("simple", "live_simple", "multiple")
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"gorilla-llm/Berkeley-Function-Calling-Leaderboard",
|
||||
}
|
||||
IS_MULTIMODAL = False
|
||||
|
||||
# BFCL primitive type names that are not valid JSON Schema types.
|
||||
# Map them to the closest JSON Schema equivalent so that grammar
|
||||
# backends (xgrammar, outlines) accept the translated tool schema.
|
||||
_TYPE_REMAP = {
|
||||
"dict": "object",
|
||||
"float": "number",
|
||||
"tuple": "array",
|
||||
"any": "string",
|
||||
}
|
||||
|
||||
def load_data(self) -> None:
|
||||
"""Defer loading to :meth:`sample` where categories are known."""
|
||||
self.data = None
|
||||
|
||||
def _resolve_categories(self, categories: list[str] | None) -> list[str]:
|
||||
if not categories:
|
||||
return list(self.DEFAULT_CATEGORIES)
|
||||
resolved: list[str] = []
|
||||
for c in categories:
|
||||
c = c.strip()
|
||||
if not c:
|
||||
continue
|
||||
resolved.append(c)
|
||||
return resolved or list(self.DEFAULT_CATEGORIES)
|
||||
|
||||
def _load_category(self, category: str) -> list[dict]:
|
||||
# Local import: huggingface_hub.errors is a small module and
|
||||
# importing at call site keeps module import cheap for users who
|
||||
# never touch BFCL.
|
||||
from huggingface_hub.errors import EntryNotFoundError
|
||||
|
||||
filename = f"BFCL_v3_{category}.json"
|
||||
try:
|
||||
path = hf_api().hf_hub_download(
|
||||
self.dataset_path, filename, repo_type="dataset"
|
||||
)
|
||||
except EntryNotFoundError as e:
|
||||
defaults = ", ".join(self.DEFAULT_CATEGORIES)
|
||||
raise ValueError(
|
||||
f"BFCL category '{category}' not found: file '{filename}' "
|
||||
f"does not exist in {self.dataset_path}. Check --bfcl-categories "
|
||||
f"(defaults: {defaults})."
|
||||
) from e
|
||||
rows: list[dict] = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
def _translate_schema(cls, node: Any) -> Any:
|
||||
"""Recursively translate BFCL-flavored JSON schema to strict JSON Schema."""
|
||||
if isinstance(node, dict):
|
||||
translated = {k: cls._translate_schema(v) for k, v in node.items()}
|
||||
t = translated.get("type")
|
||||
if isinstance(t, str) and t in cls._TYPE_REMAP:
|
||||
translated["type"] = cls._TYPE_REMAP[t]
|
||||
return translated
|
||||
if isinstance(node, list):
|
||||
return [cls._translate_schema(v) for v in node]
|
||||
return node
|
||||
|
||||
@classmethod
|
||||
def _to_openai_tools(cls, functions: list[dict]) -> list[dict]:
|
||||
tools: list[dict] = []
|
||||
for fn in functions:
|
||||
translated = cls._translate_schema(fn)
|
||||
tools.append({"type": "function", "function": translated})
|
||||
return tools
|
||||
|
||||
def sample(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
num_requests: int,
|
||||
request_id_prefix: str = "",
|
||||
no_oversample: bool = False,
|
||||
output_len: int | None = None,
|
||||
categories: list[str] | None = None,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||
categories = self._resolve_categories(categories)
|
||||
|
||||
per_category_rows: list[list[dict]] = [
|
||||
self._load_category(c) for c in categories
|
||||
]
|
||||
# Round-robin interleave so that when --disable-shuffle is set,
|
||||
# taking the first num_requests rows still yields balanced category
|
||||
# coverage. When shuffle is on (the default) this ordering is
|
||||
# randomized away, which is fine — the subsequent random sample is
|
||||
# already balanced in expectation.
|
||||
interleaved: list[dict] = []
|
||||
max_len = max((len(rows) for rows in per_category_rows), default=0)
|
||||
for i in range(max_len):
|
||||
for rows in per_category_rows:
|
||||
if i < len(rows):
|
||||
interleaved.append(rows[i])
|
||||
|
||||
if not self.disable_shuffle:
|
||||
rng = random.Random(self.random_seed)
|
||||
rng.shuffle(interleaved)
|
||||
|
||||
sampled_requests: list[SampleRequest] = []
|
||||
for row in interleaved:
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
question = row.get("question")
|
||||
functions = row.get("function")
|
||||
if not question or not functions:
|
||||
continue
|
||||
# BFCL question is list[list[dict]] — outer is turns. Use the
|
||||
# first turn only; skip multi-turn categories in this loader.
|
||||
if not isinstance(question, list) or not question:
|
||||
continue
|
||||
first_turn = question[0]
|
||||
if not isinstance(first_turn, list) or not first_turn:
|
||||
continue
|
||||
messages = first_turn
|
||||
if not isinstance(functions, list):
|
||||
functions = [functions]
|
||||
|
||||
tools = self._to_openai_tools(functions)
|
||||
|
||||
# Best-effort prompt length for percentile bucketing. Pass tools=
|
||||
# so modern chat templates (Llama 3.1+, Qwen, gpt-oss harmony,
|
||||
# Hermes) render the tool schemas — without this, the estimate
|
||||
# misses a significant chunk of the true input for BFCL traffic.
|
||||
# Older tokenizers reject the kwarg; fall back to tools-free.
|
||||
try:
|
||||
rendered = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tools=tools,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
except TypeError:
|
||||
rendered = tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
except Exception as e:
|
||||
# Unexpected template failure — prompt_len will fall back to a
|
||||
# plain-text concatenation. Log so the degraded estimate is
|
||||
# visible instead of silently skewing latency buckets.
|
||||
logger.warning(
|
||||
"BFCL: apply_chat_template failed for a sample, falling "
|
||||
"back to plain-text prompt length: %s",
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
rendered = None
|
||||
if rendered is not None:
|
||||
prompt_len = len(tokenizer(rendered).input_ids)
|
||||
else:
|
||||
text = "\n".join(m.get("content", "") for m in messages)
|
||||
prompt_len = len(tokenizer(text).input_ids)
|
||||
|
||||
# The chat backend uses `messages` directly; `prompt` is only
|
||||
# kept as a fallback string for display/debug.
|
||||
prompt_text = messages[-1].get("content", "") if messages else ""
|
||||
|
||||
sampled_requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt_text,
|
||||
prompt_len=prompt_len,
|
||||
expected_output_len=output_len,
|
||||
request_id=request_id_prefix + str(len(sampled_requests)),
|
||||
chat_messages=messages,
|
||||
request_overrides={
|
||||
"tools": tools,
|
||||
"tool_choice": "auto",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
self.maybe_oversample_requests(
|
||||
sampled_requests, num_requests, request_id_prefix, no_oversample
|
||||
)
|
||||
return sampled_requests
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Speed Bench Dataset Implementation
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -79,6 +79,10 @@ class RequestFuncInput:
|
||||
ignore_eos: bool = False
|
||||
language: str | None = None
|
||||
request_id: str | None = None
|
||||
# Pre-built chat messages. When set, `async_request_openai_chat_completions`
|
||||
# uses this list directly and skips building messages from `prompt` and
|
||||
# `multi_modal_content`.
|
||||
chat_messages: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -343,7 +347,10 @@ async def async_request_openai_chat_completions(
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
|
||||
|
||||
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
|
||||
if request_func_input.chat_messages is not None:
|
||||
messages = request_func_input.chat_messages
|
||||
else:
|
||||
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
|
||||
|
||||
payload = {
|
||||
"model": request_func_input.model_name
|
||||
|
||||
@@ -57,6 +57,14 @@ from vllm.utils.network_utils import join_host_port
|
||||
|
||||
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
|
||||
|
||||
|
||||
def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
|
||||
"""Shallow merge; per-request wins. Returns None if both are empty."""
|
||||
if not base and not override:
|
||||
return None
|
||||
return {**(base or {}), **(override or {})}
|
||||
|
||||
|
||||
TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and (
|
||||
shutil.which("gnuplot") is not None
|
||||
)
|
||||
@@ -753,6 +761,8 @@ async def benchmark(
|
||||
input_requests[0].expected_output_len,
|
||||
input_requests[0].multi_modal_data,
|
||||
)
|
||||
test_extra_body = _merge_overrides(extra_body, input_requests[0].request_overrides)
|
||||
test_chat_messages = input_requests[0].chat_messages
|
||||
|
||||
assert (
|
||||
test_mm_content is None
|
||||
@@ -773,7 +783,8 @@ async def benchmark(
|
||||
multi_modal_content=test_mm_content,
|
||||
ignore_eos=ignore_eos,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
|
||||
if ready_check_timeout_sec > 0:
|
||||
@@ -850,7 +861,8 @@ async def benchmark(
|
||||
multi_modal_content=test_mm_content,
|
||||
ignore_eos=ignore_eos,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
profile_output = await request_func(
|
||||
request_func_input=profile_input, session=session
|
||||
@@ -927,6 +939,7 @@ async def benchmark(
|
||||
request.multi_modal_data,
|
||||
request.request_id,
|
||||
)
|
||||
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
||||
req_model_id, req_model_name = model_id, model_name
|
||||
if lora_modules:
|
||||
req_lora_module = next(lora_modules)
|
||||
@@ -943,8 +956,9 @@ async def benchmark(
|
||||
multi_modal_content=mm_content,
|
||||
ignore_eos=ignore_eos,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
extra_body=per_request_extra_body,
|
||||
request_id=request_id,
|
||||
chat_messages=request.chat_messages,
|
||||
)
|
||||
tasks.append(
|
||||
asyncio.create_task(
|
||||
|
||||
Reference in New Issue
Block a user