From a454a1dd25a181334f6f126b2a2bec3d7c1d9aab Mon Sep 17 00:00:00 2001 From: mgazz Date: Fri, 24 Jul 2026 12:30:19 +0100 Subject: [PATCH] [Bugfix][Benchmarks] Restore --skip-tokenizer-init with custom dataset (#49180) Signed-off-by: Michele Gazzetti Co-authored-by: Kevin H. Luu --- tests/benchmarks/test_skip_tokenizer_init.py | 162 +++++++++++++++++++ vllm/benchmarks/datasets/datasets.py | 27 +++- vllm/benchmarks/serve.py | 5 +- 3 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 tests/benchmarks/test_skip_tokenizer_init.py diff --git a/tests/benchmarks/test_skip_tokenizer_init.py b/tests/benchmarks/test_skip_tokenizer_init.py new file mode 100644 index 0000000000..e28320b6ae --- /dev/null +++ b/tests/benchmarks/test_skip_tokenizer_init.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test for --skip-tokenizer-init with --dataset-name custom. + +Before the fix (introduced by #39896), running: + + vllm bench serve \ + --backend vllm-pooling \ + --dataset-name custom \ + --dataset-path \ + --model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \ + --endpoint /pooling \ + --skip-tokenizer-init \ + ... + +raised immediately with: + + AssertionError: Tokenizer must be initialized before loading dataset + +even though CustomDataset.sample() already handles tokenizer=None. +This test exercises main_async() directly so it catches any regression +re-introduced at the serve.py level, not just inside get_samples(). +""" + +import argparse +import asyncio +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +import vllm.benchmarks.serve as serve_module + +# Exact prompt payload from the failing benchmark run against a +# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format). +_PRITHVI_PROMPT = { + "data": { + "data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif", + "data_format": "url", + "out_data_format": "b64_json", + "indices": [1, 2, 3, 8, 11, 12], + }, + "priority": 0, + "softmax": False, +} + + +def _write_dataset(path: Path) -> None: + path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n") + + +def _args(dataset_path: str) -> argparse.Namespace: + """Reproduce the argparse.Namespace that serve.py builds from the + failing command, including skip_tokenizer_init=True.""" + return argparse.Namespace( + # dataset + dataset_name="custom", + dataset_path=dataset_path, + disable_shuffle=False, + num_prompts=1, + custom_output_len=256, + skip_chat_template=True, + chat_template_kwargs=None, + no_oversample=False, + seed=0, + request_id_prefix="bench-", + # model / tokenizer + model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", + served_model_name=None, + tokenizer=None, + tokenizer_mode="auto", + trust_remote_code=False, + skip_tokenizer_init=True, # <-- the flag under test + # backend / endpoint + backend="vllm-pooling", + base_url="http://127.0.0.1:8000", + host="127.0.0.1", + port=8000, + endpoint="/pooling", + header=None, + insecure=False, + # traffic + request_rate=16.0, + burstiness=1.0, + max_concurrency=None, + # misc serve args that main_async reads before reaching get_samples + plot_timeline=False, + plot_dataset_stats=False, + self_timed=None, + metadata=None, + label=None, + logprobs=None, + use_beam_search=False, + ignore_eos=False, + goodput=None, + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="25,50,75,99", + save_result=False, + append_result=False, + result_dir=".", + result_filename=None, + num_warmups=0, + profile=False, + disable_tqdm=True, + lora_modules=None, + lora_assignment="random", + ramp_up_strategy=None, + ramp_up_start_rps=None, + ramp_up_end_rps=None, + ready_check_timeout_sec=0, + extra_body=None, + top_p=None, + top_k=None, + min_p=None, + temperature=None, + frequency_penalty=None, + presence_penalty=None, + repetition_penalty=None, + save_detailed=False, + input_len=None, + output_len=None, + ) + + +@pytest.mark.benchmark +def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None: + """main_async must not raise AssertionError when skip_tokenizer_init=True + and dataset_name='custom'. + + On main (before the fix) this test fails with: + AssertionError: Tokenizer must be initialized before loading dataset + """ + dataset_path = tmp_path / "dataset_url_input_india.jsonl" + _write_dataset(dataset_path) + + args = _args(str(dataset_path)) + + # Patch benchmark() so we never make real HTTP requests — the regression + # triggers before benchmark() is ever called, so this just keeps the test + # fast and hermetic. + mock_result = { + "completed": 1, + "failed": 0, + "total_input_tokens": 1, + "total_output_tokens": 1, + "request_throughput": 1.0, + "output_throughput": 1.0, + "total_token_throughput": 1.0, + "input_lens": [], + "output_lens": [], + "ttfts": [], + "itls": [], + "generated_texts": [], + "errors": [], + "duration": 1.0, + } + with patch.object( + serve_module, "benchmark", new=AsyncMock(return_value=mock_result) + ): + # Must NOT raise AssertionError + asyncio.run(serve_module.main_async(args)) diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 83bda31241..de98ab135a 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -2091,7 +2091,7 @@ def _parse_range_ratio(value: str) -> RangeRatio: return json.loads(value) -def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: +def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]: if not hasattr(args, "request_id_prefix"): args.request_id_prefix = "" @@ -2148,6 +2148,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: ) elif args.dataset_name == "sonnet": + assert tokenizer is not None, ( + "Tokenizer must be initialized for the 'sonnet' dataset." + ) sonnet_dataset = SonnetDataset( dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle ) @@ -2317,6 +2320,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: "Multi-modal content is only supported on 'openai-chat' and " "'openai-audio' backends." ) + assert tokenizer is not None, ( + "Tokenizer must be initialized for the 'hf' dataset." + ) input_requests = dataset_class( dataset_path=args.dataset_path, dataset_subset=args.hf_subset, @@ -2338,6 +2344,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: ) elif args.dataset_name == "timed_trace": + assert tokenizer is not None, ( + "Tokenizer must be initialized for the 'timed_trace' dataset." + ) dataloader = TimedTrace(**vars(args)) input_requests = dataloader.sample( num_requests=args.num_prompts, @@ -2347,6 +2356,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: else: # For datasets that follow a similar structure, use a mapping. + assert tokenizer is not None, ( + f"Tokenizer must be initialized for the '{args.dataset_name}' dataset." + ) dataset_mapping = { "spec_bench": lambda: SpecBench( dataset_path=args.dataset_path, @@ -2532,7 +2544,7 @@ class CustomDataset(BenchmarkDataset): def sample( self, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, num_requests: int, request_id_prefix: str = "", no_oversample: bool = False, @@ -2809,7 +2821,7 @@ class CustomImageDataset(CustomDataset): def sample( self, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, num_requests: int, request_id_prefix: str = "", no_oversample: bool = False, @@ -2907,7 +2919,7 @@ class CustomAudioDataset(CustomDataset): def sample( self, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, num_requests: int, request_id_prefix: str = "", no_oversample: bool = False, @@ -2929,7 +2941,9 @@ class CustomAudioDataset(CustomDataset): prompt = item.get("prompt", "") if tokenizer is None: prompt_len = 1 - new_output_len = output_len if output_len not in (None, -1) else 256 + new_output_len = ( + output_len if (output_len is not None and output_len != -1) else 256 + ) mm_content = None else: use_chat_template = ( @@ -2979,6 +2993,7 @@ class CustomAudioDataset(CustomDataset): ) new_output_len = int(item["output_tokens"]) else: + assert output_len is not None new_output_len = output_len sampled_requests.append( SampleRequest( @@ -3037,7 +3052,7 @@ class SpecBench(CustomDataset): def sample( self, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, num_requests: int, request_id_prefix: str = "", no_oversample: bool = False, diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 76993a7d6b..996e334860 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -557,7 +557,7 @@ def calculate_metrics( input_requests: list[SampleRequest], outputs: list[RequestFuncOutput], dur_s: float, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, selected_percentiles: list[float], goodput_config_dict: dict[str, float], ) -> tuple[BenchmarkMetrics, list[int]]: @@ -772,7 +772,7 @@ async def benchmark( base_url: str, model_id: str, model_name: str, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, input_requests: list[SampleRequest], logprobs: int | None, request_rate: float, @@ -2036,7 +2036,6 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: args.self_timed = False # Load the dataset. - assert tokenizer is not None, "Tokenizer must be initialized before loading dataset" input_requests = get_samples(args, tokenizer) if args.dataset_name in ("random", "prefix_repetition"):