[Bugfix][Benchmarks] Restore --skip-tokenizer-init with custom dataset (#49180)

Signed-off-by: Michele Gazzetti <michele.gazzetti1@ibm.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
mgazz
2026-07-24 04:30:19 -07:00
committed by GitHub
co-authored by Kevin H. Luu
parent 833483f357
commit a454a1dd25
3 changed files with 185 additions and 9 deletions
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for --skip-tokenizer-init with --dataset-name custom.
Before the fix (introduced by #39896), running:
vllm bench serve \
--backend vllm-pooling \
--dataset-name custom \
--dataset-path <path> \
--model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \
--endpoint /pooling \
--skip-tokenizer-init \
...
raised immediately with:
AssertionError: Tokenizer must be initialized before loading dataset
even though CustomDataset.sample() already handles tokenizer=None.
This test exercises main_async() directly so it catches any regression
re-introduced at the serve.py level, not just inside get_samples().
"""
import argparse
import asyncio
import json
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
import vllm.benchmarks.serve as serve_module
# Exact prompt payload from the failing benchmark run against a
# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format).
_PRITHVI_PROMPT = {
"data": {
"data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif",
"data_format": "url",
"out_data_format": "b64_json",
"indices": [1, 2, 3, 8, 11, 12],
},
"priority": 0,
"softmax": False,
}
def _write_dataset(path: Path) -> None:
path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n")
def _args(dataset_path: str) -> argparse.Namespace:
"""Reproduce the argparse.Namespace that serve.py builds from the
failing command, including skip_tokenizer_init=True."""
return argparse.Namespace(
# dataset
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=False,
num_prompts=1,
custom_output_len=256,
skip_chat_template=True,
chat_template_kwargs=None,
no_oversample=False,
seed=0,
request_id_prefix="bench-",
# model / tokenizer
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
served_model_name=None,
tokenizer=None,
tokenizer_mode="auto",
trust_remote_code=False,
skip_tokenizer_init=True, # <-- the flag under test
# backend / endpoint
backend="vllm-pooling",
base_url="http://127.0.0.1:8000",
host="127.0.0.1",
port=8000,
endpoint="/pooling",
header=None,
insecure=False,
# traffic
request_rate=16.0,
burstiness=1.0,
max_concurrency=None,
# misc serve args that main_async reads before reaching get_samples
plot_timeline=False,
plot_dataset_stats=False,
self_timed=None,
metadata=None,
label=None,
logprobs=None,
use_beam_search=False,
ignore_eos=False,
goodput=None,
percentile_metrics="ttft,tpot,itl,e2el",
metric_percentiles="25,50,75,99",
save_result=False,
append_result=False,
result_dir=".",
result_filename=None,
num_warmups=0,
profile=False,
disable_tqdm=True,
lora_modules=None,
lora_assignment="random",
ramp_up_strategy=None,
ramp_up_start_rps=None,
ramp_up_end_rps=None,
ready_check_timeout_sec=0,
extra_body=None,
top_p=None,
top_k=None,
min_p=None,
temperature=None,
frequency_penalty=None,
presence_penalty=None,
repetition_penalty=None,
save_detailed=False,
input_len=None,
output_len=None,
)
@pytest.mark.benchmark
def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None:
"""main_async must not raise AssertionError when skip_tokenizer_init=True
and dataset_name='custom'.
On main (before the fix) this test fails with:
AssertionError: Tokenizer must be initialized before loading dataset
"""
dataset_path = tmp_path / "dataset_url_input_india.jsonl"
_write_dataset(dataset_path)
args = _args(str(dataset_path))
# Patch benchmark() so we never make real HTTP requests — the regression
# triggers before benchmark() is ever called, so this just keeps the test
# fast and hermetic.
mock_result = {
"completed": 1,
"failed": 0,
"total_input_tokens": 1,
"total_output_tokens": 1,
"request_throughput": 1.0,
"output_throughput": 1.0,
"total_token_throughput": 1.0,
"input_lens": [],
"output_lens": [],
"ttfts": [],
"itls": [],
"generated_texts": [],
"errors": [],
"duration": 1.0,
}
with patch.object(
serve_module, "benchmark", new=AsyncMock(return_value=mock_result)
):
# Must NOT raise AssertionError
asyncio.run(serve_module.main_async(args))
+21 -6
View File
@@ -2091,7 +2091,7 @@ def _parse_range_ratio(value: str) -> RangeRatio:
return json.loads(value)
def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
if not hasattr(args, "request_id_prefix"):
args.request_id_prefix = ""
@@ -2148,6 +2148,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
)
elif args.dataset_name == "sonnet":
assert tokenizer is not None, (
"Tokenizer must be initialized for the 'sonnet' dataset."
)
sonnet_dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
@@ -2317,6 +2320,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
"Multi-modal content is only supported on 'openai-chat' and "
"'openai-audio' backends."
)
assert tokenizer is not None, (
"Tokenizer must be initialized for the 'hf' dataset."
)
input_requests = dataset_class(
dataset_path=args.dataset_path,
dataset_subset=args.hf_subset,
@@ -2338,6 +2344,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
)
elif args.dataset_name == "timed_trace":
assert tokenizer is not None, (
"Tokenizer must be initialized for the 'timed_trace' dataset."
)
dataloader = TimedTrace(**vars(args))
input_requests = dataloader.sample(
num_requests=args.num_prompts,
@@ -2347,6 +2356,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
else:
# For datasets that follow a similar structure, use a mapping.
assert tokenizer is not None, (
f"Tokenizer must be initialized for the '{args.dataset_name}' dataset."
)
dataset_mapping = {
"spec_bench": lambda: SpecBench(
dataset_path=args.dataset_path,
@@ -2532,7 +2544,7 @@ class CustomDataset(BenchmarkDataset):
def sample(
self,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2809,7 +2821,7 @@ class CustomImageDataset(CustomDataset):
def sample(
self,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2907,7 +2919,7 @@ class CustomAudioDataset(CustomDataset):
def sample(
self,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2929,7 +2941,9 @@ class CustomAudioDataset(CustomDataset):
prompt = item.get("prompt", "")
if tokenizer is None:
prompt_len = 1
new_output_len = output_len if output_len not in (None, -1) else 256
new_output_len = (
output_len if (output_len is not None and output_len != -1) else 256
)
mm_content = None
else:
use_chat_template = (
@@ -2979,6 +2993,7 @@ class CustomAudioDataset(CustomDataset):
)
new_output_len = int(item["output_tokens"])
else:
assert output_len is not None
new_output_len = output_len
sampled_requests.append(
SampleRequest(
@@ -3037,7 +3052,7 @@ class SpecBench(CustomDataset):
def sample(
self,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
+2 -3
View File
@@ -557,7 +557,7 @@ def calculate_metrics(
input_requests: list[SampleRequest],
outputs: list[RequestFuncOutput],
dur_s: float,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
selected_percentiles: list[float],
goodput_config_dict: dict[str, float],
) -> tuple[BenchmarkMetrics, list[int]]:
@@ -772,7 +772,7 @@ async def benchmark(
base_url: str,
model_id: str,
model_name: str,
tokenizer: TokenizerLike,
tokenizer: TokenizerLike | None,
input_requests: list[SampleRequest],
logprobs: int | None,
request_rate: float,
@@ -2036,7 +2036,6 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
args.self_timed = False
# Load the dataset.
assert tokenizer is not None, "Tokenizer must be initialized before loading dataset"
input_requests = get_samples(args, tokenizer)
if args.dataset_name in ("random", "prefix_repetition"):