[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-375
View File
@@ -1,375 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import tempfile
from pathlib import Path
import pytest
from vllm.benchmarks.sweep import serve as sweep_serve
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
class TestParameterSweepItem:
"""Test ParameterSweepItem functionality."""
@pytest.mark.parametrize(
"input_dict,expected",
[
(
{"compilation_config.use_inductor_graph_partition": False},
"--compilation-config.use_inductor_graph_partition=false",
),
(
{"compilation_config.use_inductor_graph_partition": True},
"--compilation-config.use_inductor_graph_partition=true",
),
],
)
def test_nested_boolean_params(self, input_dict, expected):
"""Test that nested boolean params use =true/false syntax."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert expected in cmd
@pytest.mark.parametrize(
"input_dict,expected",
[
({"enable_prefix_caching": False}, "--no-enable-prefix-caching"),
({"enable_prefix_caching": True}, "--enable-prefix-caching"),
({"disable_log_stats": False}, "--no-disable-log-stats"),
({"disable_log_stats": True}, "--disable-log-stats"),
],
)
def test_non_nested_boolean_params(self, input_dict, expected):
"""Test that non-nested boolean params use --no- prefix."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert expected in cmd
@pytest.mark.parametrize(
"compilation_config",
[
{"cudagraph_mode": "full", "mode": 2, "use_inductor_graph_partition": True},
{
"cudagraph_mode": "piecewise",
"mode": 3,
"use_inductor_graph_partition": False,
},
],
)
def test_nested_dict_value(self, compilation_config):
"""Test that nested dict values are serialized as JSON."""
item = ParameterSweepItem.from_record(
{"compilation_config": compilation_config}
)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert "--compilation-config" in cmd
# The dict should be JSON serialized
idx = cmd.index("--compilation-config")
assert json.loads(cmd[idx + 1]) == compilation_config
@pytest.mark.parametrize(
"input_dict,expected_key,expected_value",
[
({"model": "test-model"}, "--model", "test-model"),
({"max_tokens": 100}, "--max-tokens", "100"),
({"temperature": 0.7}, "--temperature", "0.7"),
],
)
def test_string_and_numeric_values(self, input_dict, expected_key, expected_value):
"""Test that string and numeric values are handled correctly."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve"])
assert expected_key in cmd
assert expected_value in cmd
@pytest.mark.parametrize(
"input_dict,expected_key,key_idx_offset",
[
({"max_tokens": 200}, "--max-tokens", 1),
({"enable_prefix_caching": False}, "--no-enable-prefix-caching", 0),
],
)
def test_replace_existing_parameter(self, input_dict, expected_key, key_idx_offset):
"""Test that existing parameters in cmd are replaced."""
item = ParameterSweepItem.from_record(input_dict)
if key_idx_offset == 1:
# Key-value pair
cmd = item.apply_to_cmd(["vllm", "serve", "--max-tokens", "100", "model"])
assert expected_key in cmd
idx = cmd.index(expected_key)
assert cmd[idx + 1] == "200"
assert "100" not in cmd
else:
# Boolean flag
cmd = item.apply_to_cmd(
["vllm", "serve", "--enable-prefix-caching", "model"]
)
assert expected_key in cmd
assert "--enable-prefix-caching" not in cmd
class TestParameterSweep:
"""Test ParameterSweep functionality."""
def test_from_records_list(self):
"""Test creating ParameterSweep from a list of records."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
assert sweep[0]["max_tokens"] == 100
assert sweep[1]["max_tokens"] == 200
def test_read_from_dict(self):
"""Test creating ParameterSweep from a dict format."""
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
sweep = ParameterSweep.read_from_dict(data)
assert len(sweep) == 2
# Check that items have the _benchmark_name field
names = {item["_benchmark_name"] for item in sweep}
assert names == {"experiment1", "experiment2"}
# Check that parameters are preserved
for item in sweep:
if item["_benchmark_name"] == "experiment1":
assert item["max_tokens"] == 100
assert item["temperature"] == 0.7
elif item["_benchmark_name"] == "experiment2":
assert item["max_tokens"] == 200
assert item["temperature"] == 0.9
def test_read_json_list_format(self):
"""Test reading JSON file with list format."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(records, f)
temp_path = Path(f.name)
try:
sweep = ParameterSweep.read_json(temp_path)
assert len(sweep) == 2
assert sweep[0]["max_tokens"] == 100
assert sweep[1]["max_tokens"] == 200
finally:
temp_path.unlink()
def test_read_json_dict_format(self):
"""Test reading JSON file with dict format."""
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(data, f)
temp_path = Path(f.name)
try:
sweep = ParameterSweep.read_json(temp_path)
assert len(sweep) == 2
# Check that items have the _benchmark_name field
names = {item["_benchmark_name"] for item in sweep}
assert names == {"experiment1", "experiment2"}
finally:
temp_path.unlink()
def test_unique_benchmark_names_validation(self):
"""Test that duplicate _benchmark_name values raise an error."""
# Test with duplicate names in list format
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
]
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
ParameterSweep.from_records(records)
def test_unique_benchmark_names_multiple_duplicates(self):
"""Test validation with multiple duplicate names."""
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
{"_benchmark_name": "exp2", "max_tokens": 300},
{"_benchmark_name": "exp2", "max_tokens": 400},
]
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
ParameterSweep.from_records(records)
def test_no_benchmark_names_allowed(self):
"""Test that records without _benchmark_name are allowed."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
def test_mixed_benchmark_names_allowed(self):
"""Test that mixing records with and without _benchmark_name is allowed."""
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
class TestParameterSweepItemKeyNormalization:
"""Test key normalization in ParameterSweepItem."""
def test_underscore_to_hyphen_conversion(self):
"""Test that underscores are converted to hyphens in CLI."""
item = ParameterSweepItem.from_record({"max_tokens": 100})
cmd = item.apply_to_cmd(["vllm", "serve"])
assert "--max-tokens" in cmd
def test_nested_key_preserves_suffix(self):
"""Test that nested keys preserve the suffix format."""
# The suffix after the dot should preserve underscores
item = ParameterSweepItem.from_record(
{"compilation_config.some_nested_param": "value"}
)
cmd = item.apply_to_cmd(["vllm", "serve"])
# The prefix (compilation_config) gets converted to hyphens,
# but the suffix (some_nested_param) is preserved
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
def test_run_comb_excludes_warmup_from_measured_results(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[tuple[int, int, str]] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 320}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=32,
dry_run=False,
)
assert calls == [
(-1, 32, "warmup.json"),
(0, 320, "run=0.json"),
(1, 320, "run=1.json"),
]
assert measured == [{"run_number": 0}, {"run_number": 1}]
def test_run_comb_continue_on_error_keeps_later_runs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
if run_number == 0:
raise RuntimeError("synthetic run failure")
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 100}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=0,
dry_run=False,
continue_on_error=True,
)
assert calls == [0, 1]
assert measured == [{"run_number": 1}]
assert (base_path / "run=0.failure.json").exists()
assert (base_path / "summary.json").exists()
def test_run_comb_warmup_default_is_backward_compatible(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""Older callers can omit warmup_num_prompts."""
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 10}),
link_vars=[],
base_path=base_path,
num_runs=1,
dry_run=False,
)
assert calls == [0]
assert measured == [{"run_number": 0}]
-330
View File
@@ -1,330 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from pathlib import Path
from typing import Protocol, cast
import numpy as np
import pytest
import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
class _ReadableBinary(Protocol):
def read(self, size: int = -1) -> bytes: ...
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
self.name_or_path = name_or_path
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
class CohereAsrTokenizer(_Tokenizer):
def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None:
super().__init__(name_or_path)
class _CohereNameOnlyTokenizer(_Tokenizer):
def __init__(self) -> None:
super().__init__("cohere/some-local-checkpoint")
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
return cast(TokenizerLike, tokenizer)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
class _FakeFormData:
def __init__(self) -> None:
self.fields: list[tuple[str, object, dict[str, str]]] = []
def add_field(self, name: str, value: object, **kwargs: str) -> None:
self.fields.append((name, value, kwargs))
class _FakeContent:
async def iter_any(self):
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
yield b"data: [DONE]\n\n"
class _FakeResponse:
def __init__(self) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent()
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
class _FakeSession:
def __init__(self) -> None:
self.uploaded_bytes: bytes | None = None
self.upload_filename: str | None = None
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
del url, headers
self.fields = list(data.fields)
_, file_obj, file_kwargs = self.fields[0]
file_obj = cast(_ReadableBinary, file_obj)
self.uploaded_bytes = file_obj.read()
self.upload_filename = file_kwargs.get("filename")
return _FakeResponse()
class _FakeProfileSession:
def __init__(self) -> None:
self.url: str | None = None
self.headers: dict[str, str] | None = None
def post(self, *, url: str, headers: dict[str, str]):
self.url = url
self.headers = headers
return _FakeResponse()
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": str(audio_path),
"bytes": None,
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
assert (
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
)
@pytest.mark.parametrize("has_filepath", [True, False])
def test_asr_dataset_sample_handles_embedded_audio_bytes(
tmp_path: Path, has_filepath: bool
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
test_path = None
if has_filepath:
test_path = audio_path
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": test_path,
"bytes": audio_path.read_bytes(),
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, dict)
audio, sample_rate = samples[0].multi_modal_data["audio"]
assert sample_rate == 16_000
assert isinstance(audio, np.ndarray)
assert audio.size > 0
def test_async_request_openai_audio_handles_local_audio_paths(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.25)
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={"audio_path": str(audio_path)},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == audio_path.name
assert session.uploaded_bytes == audio_path.read_bytes()
assert output.success is True
assert output.generated_text == "hello"
assert output.output_tokens == 1
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
def test_async_request_openai_audio_handles_decoded_audio_arrays(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == "audio.wav"
assert session.uploaded_bytes is not None
assert output.success is True
assert output.generated_text == "hello"
def test_audio_request_rejects_profile_endpoint() -> None:
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/start_profile",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
)
with pytest.raises(ValueError, match="OpenAI Audio API"):
asyncio.run(
request_func_module.async_request_openai_audio(
request_input, _FakeProfileSession()
)
)
@pytest.mark.parametrize("endpoint", ["start_profile", "stop_profile"])
def test_audio_benchmark_profile_request_has_no_inference_payload(
endpoint: str,
) -> None:
session = _FakeProfileSession()
output = asyncio.run(
request_func_module.async_request_profile(
f"http://localhost:8000/{endpoint}",
session,
extra_headers={"X-Test-Header": "profile"},
)
)
assert output.success is True
assert session.url == f"http://localhost:8000/{endpoint}"
assert session.headers == {"X-Test-Header": "profile"}
_COHERE_ASR_PROMPT = (
"<|startofcontext|><|startoftranscript|>"
"<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
"<|notimestamp|><|nodiarize|>"
)
def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
audio_path = tmp_path / "sample.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {"path": str(audio_path), "bytes": None},
"text": "hello world",
}
]
return dataset
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == _COHERE_ASR_PROMPT
def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
tmp_path: Path,
) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == _COHERE_ASR_PROMPT
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == ""
-341
View File
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import BFCLDataset, get_samples
def _patch_hf_api(side_effect):
"""Return a patch context that swaps `hf_api()` to a stub whose
`.hf_hub_download` attribute uses `side_effect`."""
fake_api = MagicMock()
fake_api.hf_hub_download.side_effect = side_effect
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained("openai-community/gpt2")
_FAKE_ROWS = {
"simple": [
{
"id": "simple_0",
"question": [
[
{
"role": "user",
"content": "What is 2+2?",
}
]
],
"function": [
{
"name": "add",
"description": "Add two numbers.",
"parameters": {
"type": "dict",
"properties": {
"a": {"type": "integer", "description": "first"},
"b": {"type": "float", "description": "second"},
},
"required": ["a", "b"],
},
}
],
},
],
"live_simple": [
{
"id": "live_simple_0",
"question": [[{"role": "user", "content": "Tell me the weather."}]],
"function": [
{
"name": "get_weather",
"description": "Get weather.",
"parameters": {
"type": "dict",
"properties": {
"city": {"type": "any", "description": "city"},
"coords": {"type": "tuple", "description": "coords"},
},
"required": ["city"],
},
}
],
},
],
}
def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
"""Write fake BFCL JSONL files mimicking the HF repo layout."""
paths = {}
for category, rows in _FAKE_ROWS.items():
p = tmp_path / f"BFCL_v3_{category}.json"
with p.open("w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
paths[category] = p
return paths
def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="hf",
dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
hf_name=None,
hf_subset=None,
hf_split=None,
hf_output_len=64,
disable_shuffle=True,
num_prompts=2,
no_oversample=False,
no_stream=True,
seed=0,
request_id_prefix="",
trust_remote_code=False,
skip_chat_template=False,
enable_multimodal_chat=False,
backend="openai-chat",
bfcl_categories=categories,
)
@pytest.mark.benchmark
def test_bfcl_dataset_translates_schema_and_attaches_tools(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""BFCLDataset should translate schemas to OpenAI tool format, set
`messages` directly on SampleRequest, and attach tools/tool_choice via
request_overrides."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
args = _args_for_bfcl(categories=["simple", "live_simple"])
with _patch_hf_api(fake_download):
samples = get_samples(args, hf_tokenizer)
assert len(samples) == 2
for s in samples:
assert s.chat_messages is not None
assert isinstance(s.chat_messages, list)
assert s.chat_messages[0]["role"] == "user"
assert s.request_overrides is not None
assert "tools" in s.request_overrides
assert s.request_overrides["tool_choice"] == "auto"
# messages must NOT leak into request_overrides — it has its own
# typed field on SampleRequest.
assert "messages" not in s.request_overrides
tools = s.request_overrides["tools"]
assert len(tools) == 1
tool = tools[0]
assert tool["type"] == "function"
# Translated schema: dict -> object, float -> number,
# any -> string, tuple -> array.
params = tool["function"]["parameters"]
assert params["type"] == "object"
for prop in params["properties"].values():
assert prop["type"] in {"integer", "number", "string", "array"}
@pytest.mark.benchmark
def test_bfcl_dataset_requires_openai_chat_backend(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
args = _args_for_bfcl(categories=["simple"])
args.backend = "openai"
with pytest.raises(ValueError, match="openai-chat"):
get_samples(args, hf_tokenizer)
@pytest.mark.benchmark
def test_bfcl_dataset_missing_category_raises_clear_error(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""A typo'd category should produce an actionable ValueError, not an
opaque huggingface_hub exception."""
from huggingface_hub.errors import EntryNotFoundError
args = _args_for_bfcl(categories=["simpl"]) # typo
def raise_missing(_repo, filename, **_kwargs):
raise EntryNotFoundError(f"404 Not Found: {filename}")
with (
_patch_hf_api(raise_missing),
pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
):
get_samples(args, hf_tokenizer)
@pytest.mark.benchmark
def test_chat_backend_uses_messages_field_when_set() -> None:
"""When RequestFuncInput.chat_messages is set, the chat backend must use
it verbatim and skip default content construction from `prompt`."""
import asyncio
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
async_request_openai_chat_completions,
)
captured: dict = {}
class _FakeResp:
status = 500
reason = "stop-after-capture"
content = None
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
return False
class _FakeSession:
def post(self, url, json, headers): # noqa: A002
captured["url"] = url
captured["payload"] = json
return _FakeResp()
messages = [
{"role": "system", "content": "you are helpful"},
{"role": "user", "content": "call add(3, 4)"},
]
req = RequestFuncInput(
prompt="IGNORED",
api_url="http://localhost:0/v1/chat/completions",
prompt_len=10,
output_len=16,
model="test-model",
chat_messages=messages,
extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
)
asyncio.run(
async_request_openai_chat_completions(
request_func_input=req, session=_FakeSession()
)
)
payload = captured["payload"]
assert payload["messages"] is messages, (
"chat backend must forward RequestFuncInput.chat_messages verbatim "
"instead of constructing a default user message from `prompt`"
)
# extra_body still merges in as before (shallow, per-request wins).
assert payload["tools"][0]["function"]["name"] == "add"
@pytest.mark.benchmark
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
"""prompt_len must reflect tokens from both messages *and* tool schemas,
so percentile buckets and input-distribution summaries aren't biased
low for tool-heavy traffic."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
captured: dict = {}
class _FakeTokenizer:
def apply_chat_template(
self, messages, tools=None, tokenize=False, add_generation_prompt=True
):
captured["tools"] = tools
base = " ".join(m.get("content", "") for m in messages)
tool_text = json.dumps(tools) if tools else ""
return base + " " + tool_text
def __call__(self, text):
# 1 "token" per whitespace-separated word.
return type("Enc", (), {"input_ids": text.split()})()
fake = _FakeTokenizer()
args = _args_for_bfcl(categories=["simple"])
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, fake)
assert len(samples) == 1
assert captured["tools"] is not None, (
"apply_chat_template must be called with tools= so the schema "
"contributes to the prompt-length estimate"
)
assert len(captured["tools"]) == 1
assert captured["tools"][0]["function"]["name"] == "add"
# Sanity: prompt_len exceeds a messages-only estimate. The fake row's
# user message is "What is 2+2?" (3 whitespace-separated tokens).
assert samples[0].prompt_len > 3
@pytest.mark.benchmark
def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
tmp_path: Path,
) -> None:
"""Older tokenizers don't accept tools=; fallback must still produce a
non-zero prompt_len without crashing."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
class _LegacyTokenizer:
def apply_chat_template(self, messages, **kwargs):
if "tools" in kwargs:
raise TypeError("unexpected keyword argument 'tools'")
return " ".join(m.get("content", "") for m in messages)
def __call__(self, text):
return type("Enc", (), {"input_ids": text.split()})()
args = _args_for_bfcl(categories=["simple"])
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, _LegacyTokenizer())
assert len(samples) == 1
assert samples[0].prompt_len > 0
@pytest.mark.benchmark
def test_bfcl_schema_translation_is_recursive() -> None:
"""_translate_schema must recurse into nested properties."""
input_schema = {
"type": "dict",
"properties": {
"nested": {
"type": "dict",
"properties": {
"value": {"type": "float"},
"tags": {"type": "tuple", "items": {"type": "any"}},
},
}
},
}
out = BFCLDataset._translate_schema(input_schema)
assert out["type"] == "object"
assert out["properties"]["nested"]["type"] == "object"
assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
nested_props = out["properties"]["nested"]["properties"]
assert nested_props["tags"]["items"]["type"] == "string"
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
import pytest
from vllm.benchmarks.datasets import get_samples
class _RecordingTokenizer:
"""Minimal tokenizer stub that records the kwargs forwarded to
apply_chat_template, so we can assert chat_template_kwargs propagation
without loading a real model/template."""
def __init__(self) -> None:
self.captured_kwargs: dict | None = None
self.chat_template = "dummy-template"
def apply_chat_template(
self,
conversation,
add_generation_prompt: bool = True,
tokenize: bool = False,
**kwargs,
) -> str:
self.captured_kwargs = kwargs
return conversation[0]["content"]
def __call__(self, text: str):
return argparse.Namespace(input_ids=list(range(len(text.split()))))
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=True,
num_prompts=1,
custom_output_len=32,
skip_chat_template=False,
chat_template_kwargs=chat_template_kwargs,
no_oversample=False,
seed=0,
request_id_prefix="",
)
def _write_one(path: Path) -> None:
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
@pytest.mark.benchmark
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), {"thinking": True}), tok)
assert tok.captured_kwargs == {"thinking": True}
@pytest.mark.benchmark
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
"""When not provided, no extra kwargs are passed (existing behavior)."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), None), tok)
assert tok.captured_kwargs == {}
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import get_samples
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained("openai-community/gpt2")
def _write_jsonl(path: Path, n_rows: int) -> None:
with path.open("w") as f:
for i in range(n_rows):
f.write(json.dumps({"prompt": f"row {i}: unique prompt content."}) + "\n")
def _args_for_custom(dataset_path: str, seed: int) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=False,
num_prompts=30,
custom_output_len=32,
skip_chat_template=True,
no_oversample=False,
seed=seed,
request_id_prefix="",
)
@pytest.mark.benchmark
def test_custom_dataset_seed_propagates(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""--seed must control the CustomDataset shuffle used by get_samples.
Without the fix, CustomDataset was instantiated without random_seed,
so its load-time shuffle always used DEFAULT_SEED=0 regardless of
args.seed, causing every run with --dataset-name custom to pick the
same subset of rows from a larger file.
"""
jsonl = tmp_path / "data.jsonl"
_write_jsonl(jsonl, n_rows=60)
samples_a = get_samples(_args_for_custom(str(jsonl), seed=0), hf_tokenizer)
samples_b = get_samples(_args_for_custom(str(jsonl), seed=42), hf_tokenizer)
prompts_a = {s.prompt for s in samples_a}
prompts_b = {s.prompt for s in samples_b}
assert len(prompts_a) == 30
assert len(prompts_b) == 30
assert prompts_a != prompts_b
@pytest.mark.benchmark
def test_custom_dataset_same_seed_is_deterministic(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""Same --seed must yield the same CustomDataset subset."""
jsonl = tmp_path / "data.jsonl"
_write_jsonl(jsonl, n_rows=60)
samples_a = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
samples_b = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
prompts_a = [s.prompt for s in samples_a]
prompts_b = [s.prompt for s in samples_b]
assert prompts_a == prompts_b
@@ -1,400 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from argparse import Namespace
from io import BytesIO
from pathlib import Path
from typing import Any, cast
import pybase64 as base64
import pytest
from PIL import Image
import vllm.benchmarks.datasets.datasets as datasets_module
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
_get_chat_content,
_get_chat_messages,
)
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
def _tokenizer() -> TokenizerLike:
return cast(TokenizerLike, _Tokenizer())
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
Image.new("RGB", (1, 1), color=color).save(path)
def _decode_data_url(data_url: str) -> tuple[str, bytes]:
prefix, image_base64 = data_url.split(",", 1)
return prefix, base64.b64decode(image_base64)
def _assert_png_data_url(data_url: str) -> None:
prefix, image_bytes = _decode_data_url(data_url)
assert prefix == "data:image/png;base64"
with Image.open(BytesIO(image_bytes)) as image:
image.verify()
def _args_for_custom_image(dataset_path: Path) -> Namespace:
return Namespace(
dataset_name="custom_image",
dataset_path=str(dataset_path),
disable_shuffle=True,
seed=0,
num_prompts=2,
custom_output_len=32,
enable_multimodal_chat=False,
custom_ensure_client_side_data=False,
request_id_prefix="req-",
no_oversample=False,
)
@pytest.mark.benchmark
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
image_c = tmp_path / "chart_c.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the first two charts.",
"image_files": [str(image_a), str(image_b)],
},
{
"content": [
{"type": "text", "text": "Now compare "},
{"type": "image", "image": str(image_c)},
],
},
],
)
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
assert len(samples) == 2
assert samples[0].request_id == "req-0"
assert isinstance(samples[0].multi_modal_data, list)
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
assert samples[1].request_id == "req-1"
assert samples[1].multi_modal_data is None
assert isinstance(samples[1].prompt, list)
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
assert isinstance(samples[1].prompt[1], dict)
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
@pytest.mark.benchmark
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [str(image_a), str(image_b)],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.prompt == "Compare the charts."
assert sample.prompt_len == 3
assert isinstance(sample.multi_modal_data, list)
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
@pytest.mark.benchmark
def test_custom_image_dataset_preserves_interleaved_content_order(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{"type": "text", "text": " with "},
{
"type": "image_url",
"image_url": {
"url": str(image_b),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt_len == 2
assert isinstance(sample.prompt, list)
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
assert [part["type"] for part in parts] == [
"text",
"image_url",
"text",
"image_url",
]
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
assert parts[3]["image_url"] == {
"url": f"file://{image_b}",
"detail": "low",
}
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_content(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
tmp_path: Path,
) -> None:
image = tmp_path / "chart.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Describe "},
{"type": "image", "image": str(image)},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
enable_multimodal_chat=True,
)
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt == [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe "},
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
],
}
]
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_messages(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_image_media_when_requested(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
data_url = "data:image/png;base64,Zm9v"
remote_url = "https://example.com/chart.png"
original_fetch_image = datasets_module.fetch_image
def fake_fetch_image(image_url: str) -> Image.Image:
if image_url == remote_url:
return Image.new("RGB", (1, 1), color=(0, 0, 255))
return original_fetch_image(image_url)
monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [
str(image_a),
image_b.as_uri(),
remote_url,
data_url,
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, list)
image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
_assert_png_data_url(image_urls[0])
_assert_png_data_url(image_urls[1])
_assert_png_data_url(image_urls[2])
assert image_urls[3] == data_url
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_interleaved_image_media(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{
"type": "image_url",
"image_url": {
"url": image_b.as_uri(),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
sample = samples[0]
assert isinstance(sample.prompt, list)
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
_assert_png_data_url(parts[1]["image_url"]["url"])
_assert_png_data_url(parts[2]["image_url"]["url"])
assert parts[2]["image_url"]["detail"] == "low"
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_image_media(
tmp_path: Path,
) -> None:
invalid_image = tmp_path / "not_an_image.png"
invalid_image.write_text("not an image")
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="Invalid image URL"):
dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_content_part(
tmp_path: Path,
) -> None:
jsonl = tmp_path / "images.jsonl"
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -1,510 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Timing-accounting invariants for the streaming benchmark request functions.
serve.py derives TPOT as (latency - ttft) / (output_len - 1), which only means
"average inter-token latency" if latency - ttft == sum(itl). These tests pin
that identity, and pin that the trailing choice-less usage chunk does not
extend the measured end of the request.
Driven against a fake session and a scripted clock: no GPU, model, or socket.
"""
import asyncio
import importlib.util
import json
from pathlib import Path
from typing import Any
import aiohttp
import numpy as np
import pytest
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
async_request_openai_audio,
async_request_openai_chat_completions,
async_request_openai_completions,
)
pytestmark = pytest.mark.skip_global_cleanup
AUDIO_SAMPLE_RATE = 16_000
class _ScriptedClock:
"""A deterministic stand-in for time.perf_counter.
Each read advances the clock by exactly one tick, so what a code path
observes depends only on how many times it reads the clock. A stray or
misattributed read then shows up as a whole tick of drift rather than as
sub-microsecond jitter, which keeps these assertions free of flakiness.
"""
def __init__(self, tick: float = 1.0, start: float = 1000.0) -> None:
self.tick = tick
self._now = start
def __call__(self) -> float:
value = self._now
self._now += self.tick
return value
class _FakeTime:
"""Namespace substituted for the module-level time import."""
def __init__(self, perf_counter) -> None:
self.perf_counter = perf_counter
def _sse(payload: dict[str, Any]) -> bytes:
return f"data: {json.dumps(payload)}\n\n".encode()
def _token_chunk(index: int, *, chat: bool) -> bytes:
if chat:
choice = {"index": 0, "delta": {"content": f"tok{index} "}}
else:
choice = {"index": 0, "text": f"tok{index} "}
return _sse({"id": "cmpl-test", "object": "x", "choices": [choice]})
def _usage_chunk(n_tokens: int) -> bytes:
return _sse(
{
"id": "cmpl-test",
"object": "x",
"choices": [],
"usage": {
"prompt_tokens": 7,
"completion_tokens": n_tokens,
"total_tokens": 7 + n_tokens,
},
}
)
def _build_stream(
n_tokens: int,
*,
chat: bool,
ttft_delay: float = 0.0,
itl_delay: float = 0.0,
usage_delay: float = 0.0,
) -> list[tuple[float, bytes]]:
"""Build ``(delay_before_chunk, payload)`` pairs for one streamed response.
``vllm bench serve`` always requests ``stream_options.include_usage``, so a
choice-less usage chunk always trails the final token.
"""
chunks = [
(ttft_delay if i == 0 else itl_delay, _token_chunk(i, chat=chat))
for i in range(n_tokens)
]
chunks.append((usage_delay, _usage_chunk(n_tokens)))
chunks.append((0.0, b"data: [DONE]\n\n"))
return chunks
class _FakeContent:
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
async def iter_any(self):
for delay, payload in self._chunks:
if delay:
await asyncio.sleep(delay)
yield payload
# The legacy script iterates ``response.content`` directly.
def __aiter__(self):
return self.iter_any()
class _FakeResponse:
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent(chunks)
async def __aenter__(self) -> "_FakeResponse":
return self
async def __aexit__(self, exc_type, exc, tb) -> bool:
return False
class _FakeSession:
"""Returns a scripted SSE stream for any request, without touching a socket."""
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
def post(self, **kwargs: Any) -> _FakeResponse:
del kwargs
return _FakeResponse(self._chunks)
async def __aenter__(self) -> "_FakeSession":
return self
async def __aexit__(self, exc_type, exc, tb) -> bool:
return False
# (id, request function, api url, is_chat_payload)
STREAMING_ENDPOINTS = [
(
"completions",
async_request_openai_completions,
"http://test/v1/completions",
False,
),
(
"chat",
async_request_openai_chat_completions,
"http://test/v1/chat/completions",
True,
),
(
"audio",
async_request_openai_audio,
"http://test/v1/audio/transcriptions",
True,
),
]
def _make_input(api_url: str, n_tokens: int, endpoint_id: str) -> RequestFuncInput:
multi_modal_content = None
if endpoint_id == "audio":
# A short silent clip; only its duration is read by the request func.
multi_modal_content = {
"audio": (
np.zeros(AUDIO_SAMPLE_RATE // 10, dtype=np.float32),
AUDIO_SAMPLE_RATE,
)
}
return RequestFuncInput(
prompt="hello",
api_url=api_url,
prompt_len=7,
output_len=n_tokens,
model="test-model",
multi_modal_content=multi_modal_content,
)
def _run(
request_func,
api_url: str,
endpoint_id: str,
chunks,
n_tokens: int,
*,
expect_success: bool = True,
):
output = asyncio.run(
request_func(
_make_input(api_url, n_tokens, endpoint_id),
_FakeSession(chunks),
)
)
if expect_success:
assert output.success, output.error
return output
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_decode_span_identity_is_exact(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""Latency - ttft must equal sum(itl) exactly.
On a scripted clock this can only drift if a code path reads the clock an
extra time or attributes a read to the wrong event.
"""
tick = 1.0
monkeypatch.setattr(
request_func_module, "time", _FakeTime(_ScriptedClock(tick=tick))
)
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
assert len(output.itl) == n_tokens - 1
assert output.output_tokens == n_tokens
residual = (output.latency - output.ttft) - sum(output.itl)
assert residual == pytest.approx(0.0, abs=tick / 1000), (
f"{endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}; "
f"TPOT is derived from latency - ttft, so it must match the measured "
f"inter-token latencies"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_trailing_usage_chunk_does_not_extend_latency(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""E2E latency must end at the last token, not at the usage trailer.
The usage chunk carries no token, so counting it stretches E2EL (and
therefore TPOT) by the delivery time of one extra SSE frame. The [DONE]
sentinel is already excluded on the same grounds.
"""
tick = 1.0
clock = _ScriptedClock(tick=tick)
monkeypatch.setattr(request_func_module, "time", _FakeTime(clock))
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
# Clock reads: 1 for the start time, then 1 per non-[DONE] chunk. The last
# token is therefore read at start + n_tokens ticks; the usage chunk one
# tick later.
expected_latency = n_tokens * tick
assert output.latency == pytest.approx(expected_latency, abs=tick / 1000), (
f"{endpoint_id}: latency {output.latency} should stop at the final "
f"token ({expected_latency}), not run on to the usage chunk "
f"({expected_latency + tick})"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_decode_span_identity_holds_with_real_clock(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""The identity also holds against the real clock, with real gaps.
The residual differences out the same recorded timestamps, so wall clock
jitter cancels and only float rounding remains; a microsecond bound is
ample rather than tight.
"""
n_tokens = 4
chunks = _build_stream(
n_tokens, chat=chat, ttft_delay=0.02, itl_delay=0.01, usage_delay=0.02
)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
residual = (output.latency - output.ttft) - sum(output.itl)
assert abs(residual) < 1e-6, (
f"{endpoint_id}: (latency - ttft) - sum(itl) drifted by {residual:.9f}s"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_usage_only_stream_is_not_reported_as_success(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""A stream with no token chunk must fail rather than report 0 ms.
Only token chunks move the end of the request, so a degenerate stream
carrying nothing but a usage trailer leaves latency at zero. Reporting that
as a success would feed a zero-duration request into the benchmark
aggregates.
"""
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
output = _run(request_func, api_url, endpoint_id, chunks, 0, expect_success=False)
assert not output.success
assert output.itl == []
assert "TTFT" in output.error
# Significant whitespace at every edge, plus a multibyte character so that a
# split can land inside a UTF-8 sequence.
_BOUNDARY_TEXT = " leading 中 trailing "
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_streamed_output_is_invariant_to_transport_chunk_boundaries(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""iter_any() yields arbitrary transport fragments, so parsing must
give the same result no matter where the byte stream is split."""
if chat:
choice: dict[str, Any] = {"index": 0, "delta": {"content": _BOUNDARY_TEXT}}
else:
choice = {"index": 0, "text": _BOUNDARY_TEXT}
# Not _sse(): its ensure_ascii default would escape the multibyte char.
event = json.dumps(
{"id": "cmpl-test", "object": "x", "choices": [choice]}, ensure_ascii=False
)
stream = f"data: {event}\n\ndata: [DONE]\n\n".encode()
async def run_all_splits() -> list[tuple[int, bool, str, str]]:
failures = []
for split_at in range(1, len(stream)):
chunks = [(0.0, stream[:split_at]), (0.0, stream[split_at:])]
output = await request_func(
_make_input(api_url, 1, endpoint_id),
_FakeSession(chunks),
)
if not output.success or output.generated_text != _BOUNDARY_TEXT:
error = output.error.splitlines()[-1] if output.error else ""
failures.append(
(split_at, output.success, output.generated_text, error)
)
return failures
assert asyncio.run(run_all_splits()) == []
# ``benchmarks/backend_request_func.py`` is a standalone script rather than a
# package module, so it is loaded by path. It also builds its own ClientSession,
# so the scripted stream is injected by swapping the module's aiohttp reference.
_LEGACY_PATH = Path(__file__).parents[2] / "benchmarks" / "backend_request_func.py"
class _FakeAiohttp:
"""Stands in for the legacy module's aiohttp import."""
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
def __getattr__(self, name: str) -> Any:
return getattr(aiohttp, name)
def ClientSession(self, **kwargs: Any) -> _FakeSession: # noqa: N802
del kwargs
return _FakeSession(self._chunks)
def _load_legacy():
spec = importlib.util.spec_from_file_location(
"legacy_backend_request_func", _LEGACY_PATH
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
LEGACY_ENDPOINTS = [
(
"completions",
"async_request_openai_completions",
"http://test/v1/completions",
False,
),
(
"chat",
"async_request_openai_chat_completions",
"http://test/v1/chat/completions",
True,
),
(
"audio",
"async_request_openai_audio",
"http://test/v1/audio/transcriptions",
True,
),
]
def _run_legacy(
monkeypatch, func_name, api_url, endpoint_id, chunks, n_tokens, clock=None
):
module = _load_legacy()
monkeypatch.setattr(module, "aiohttp", _FakeAiohttp(chunks))
if clock is not None:
monkeypatch.setattr(module, "time", _FakeTime(clock))
request_func = getattr(module, func_name)
return asyncio.run(request_func(_make_input(api_url, n_tokens, endpoint_id)))
@pytest.mark.parametrize(
"endpoint_id,func_name,api_url,chat",
LEGACY_ENDPOINTS,
ids=[e[0] for e in LEGACY_ENDPOINTS],
)
def test_legacy_decode_span_identity_is_exact(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
func_name: str,
api_url: str,
chat: bool,
) -> None:
"""benchmarks/backend_request_func.py must honour the same identity."""
tick = 1.0
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run_legacy(
monkeypatch,
func_name,
api_url,
endpoint_id,
chunks,
n_tokens,
clock=_ScriptedClock(tick=tick),
)
assert output.success, output.error
assert len(output.itl) == n_tokens - 1
residual = (output.latency - output.ttft) - sum(output.itl)
assert residual == pytest.approx(0.0, abs=tick / 1000), (
f"legacy {endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}"
)
assert output.latency == pytest.approx(n_tokens * tick, abs=tick / 1000)
@pytest.mark.parametrize(
"endpoint_id,func_name,api_url,chat",
LEGACY_ENDPOINTS,
ids=[e[0] for e in LEGACY_ENDPOINTS],
)
def test_legacy_usage_only_stream_is_not_reported_as_success(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
func_name: str,
api_url: str,
chat: bool,
) -> None:
"""A usage-only stream must fail rather than report a zero-duration success."""
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
output = _run_legacy(monkeypatch, func_name, api_url, endpoint_id, chunks, 0)
assert not output.success
assert output.itl == []
-178
View File
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import subprocess
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.main import run_case
from vllm.benchmarks.forward.profile import summarize
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
from vllm.platforms import current_platform
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
def _group(block_size: int, replay: int = 0, dcp_sharded: bool = False):
return SimpleNamespace(
kv_cache_spec=SimpleNamespace(
block_size=block_size,
prefix_replay_tokens=replay,
dcp_sharded=dcp_sharded,
)
)
def test_build_batch_layout():
"""The runner sees each request kind as the scheduler would schedule it."""
case = parse_case("c=2*1@101,4@64:3,32,8@48c,16@316")
groups = [_group(16), _group(32, replay=128)]
batch = build_batch(case, groups, 1000, 1000, np.random.default_rng(0))
# (scheduled, computed, is_decode) per request: decode, verify, prefill,
# chunked prefill, then a prefix hit replaying the last 128 hit tokens.
assert list(batch.expected.values()) == [
(1, 100, True),
(1, 100, True),
(4, 60, True),
(32, 0, False),
(8, 40, False),
(16 + 128, 300 - 128, False),
]
output = batch.scheduler_output
assert output.total_num_scheduled_tokens == 1 + 1 + 4 + 32 + 8 + 144
assert len(output.scheduled_spec_decode_tokens["bench-2"]) == 3
assert output.scheduled_new_reqs[-1].replay_start == 300 - 128
for req_id, (scheduled, _, _) in batch.expected.items():
assert len(batch.query_token_ids[req_id]) == scheduled
# Groups share one block pool: block IDs must not overlap.
ranges = sorted(r for group in batch.group_block_ranges for r in group)
assert ranges[0][0] == 1
assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:]))
with pytest.raises(ValueError, match="KV blocks"):
build_batch(case, groups, 10, 1000, np.random.default_rng(0))
# DCP spreads each block's tokens over the DCP ranks, like the scheduler.
groups = [_group(16, dcp_sharded=True), _group(16)]
batch = build_batch(
parse_case("c=1@640"), groups, 1000, 1000, np.random.default_rng(0), 4
)
assert batch.group_block_ranges == [[(1, 11)], [(11, 51)]]
def test_jitter_kv_lens():
"""Prefixes vary, capped at the max length; queries and fresh prefills don't.
A case's draw depends only on the seed and its name, so adding or reordering
other cases doesn't change it."""
case = parse_case("c=1000*1@1001,4@64:3,32")
jittered = jitter_kv_lens(case, 0.5, 1400, 0)
assert jitter_kv_lens(case, 0.5, 1400, 0) == jittered
other = jitter_kv_lens(replace(case, name="d"), 0.5, 1400, 0)
assert other.requests != jittered.requests
prefixes = np.array([r.prefix_len for r in jittered.requests[:1000]])
assert prefixes.min() >= 500 and max(r.kv_len for r in jittered.requests) == 1400
assert prefixes.std() > 200
assert [r.query_len for r in jittered.requests] == [
r.query_len for r in case.requests
]
assert jittered.requests[-1] == case.requests[-1]
assert jitter_kv_lens(case, 0, 2048, 0) == case
with pytest.raises(ValueError, match="KV length"):
jitter_kv_lens(case, 0, 1000, 0)
@pytest.mark.parametrize("capacity", [0, 2])
def test_bench_forward_rejects_oversized_verify(capacity):
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
runner = SimpleNamespace(num_speculative_steps=capacity)
case = parse_case("verify=2*4@64:3")
with pytest.raises(
ValueError,
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
):
run_case(runner, case, SimpleNamespace())
def test_profile_summary():
"""Each replay's leading restore/rendezvous activities are left out, and
kernels are grouped by name without template or function arguments."""
def act(name, ts, dur, cat="kernel"):
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
replay = [
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
("void ns::gemm<128>(Params)", "kernel", 10),
("void ns::gemm<64>(Params)", "kernel", 20),
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
]
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
for r in range(2):
for i, (name, cat, dur) in enumerate(replay):
events.append(act(name, 1000 * r + 100 * i, dur, cat))
summary = summarize(events, replays=2, skip=1)
assert summary["pre_forward_us"] == 5
assert summary["busy_us"] == 34
assert summary["span_us"] == 204
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
("ns::gemm", 30, 2),
("ns::norm", 4, 1),
]
with pytest.raises(ValueError, match="replays"):
summarize(events[:-1], replays=2, skip=1)
@pytest.mark.benchmark
# TODO(woosuk): Support ROCm and re-enable this test there.
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Hangs on ROCm; the bench is CUDA-only."
)
def test_bench_forward(tmp_path):
output = tmp_path / "forward.json"
command = [
"vllm",
"bench",
"forward",
"--model",
MODEL_NAME,
"--max-model-len",
"2048",
"--gpu-memory-utilization",
"0.5",
"--case",
"decode=8*1@512",
"--case",
"prefill=256",
"--case",
"mix=128@256,4*1@512",
"--iters",
"3",
"--samples",
"2",
"--output-json",
str(output),
"--profile-dir",
str(tmp_path / "profile"),
]
result = subprocess.run(command, capture_output=True, text=True)
print(result.stdout)
print(result.stderr)
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
results = {row["case"]: row for row in json.loads(output.read_text())}
assert list(results) == ["decode", "prefill", "mix"]
for row in results.values():
rank0 = row["ranks"][0]
assert len(rank0["batch_order"]) == len(row["requests"])
assert rank0["finite"] and rank0["graph_deterministic"]
assert row["median_us"] > 0
profile = rank0["profile"]
assert profile["top_kernels"] and profile["span_us"] > 0
summary = json.loads(Path(profile["summary"]).read_text())
assert Path(summary["trace"]).exists()
-171
View File
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pandas as pd
import pytest
from vllm.benchmarks.sweep.plot import (
PlotEqualTo,
PlotFilterBase,
PlotFilters,
PlotGreaterThan,
PlotGreaterThanOrEqualTo,
PlotLessThan,
PlotLessThanOrEqualTo,
PlotNotEqualTo,
)
class TestPlotFilters:
"""Test PlotFilter functionality including 'inf' edge case."""
def setup_method(self):
"""Create sample DataFrames for testing."""
# DataFrame with numeric values
self.df_numeric = pd.DataFrame(
{
"request_rate": [1.0, 5.0, 10.0, 50.0, 100.0],
"value": [10, 20, 30, 40, 50],
}
)
# DataFrame with float('inf') - note: string "inf" values are coerced
# to float when loading data, so we only test with float('inf')
self.df_inf_float = pd.DataFrame(
{
"request_rate": [1.0, 5.0, 10.0, float("inf"), float("inf")],
"value": [10, 20, 30, 40, 50],
}
)
@pytest.mark.parametrize(
"target,expected_count",
[
("5.0", 1),
("10.0", 1),
("1.0", 1),
],
)
def test_equal_to_numeric(self, target, expected_count):
"""Test PlotEqualTo with numeric values."""
filter_obj = PlotEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
def test_equal_to_inf_float(self):
"""Test PlotEqualTo with float('inf')."""
filter_obj = PlotEqualTo("request_rate", "inf")
result = filter_obj.apply(self.df_inf_float)
# Should match both float('inf') entries because float('inf') == float('inf')
assert len(result) == 2
@pytest.mark.parametrize(
"target,expected_count",
[
("5.0", 4), # All except 5.0
("1.0", 4), # All except 1.0
],
)
def test_not_equal_to_numeric(self, target, expected_count):
"""Test PlotNotEqualTo with numeric values."""
filter_obj = PlotNotEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
def test_not_equal_to_inf_float(self):
"""Test PlotNotEqualTo with float('inf')."""
filter_obj = PlotNotEqualTo("request_rate", "inf")
result = filter_obj.apply(self.df_inf_float)
# Should exclude float('inf') entries
assert len(result) == 3
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 2), # 1.0, 5.0
("50.0", 3), # 1.0, 5.0, 10.0
("5.0", 1), # 1.0
],
)
def test_less_than(self, target, expected_count):
"""Test PlotLessThan with numeric values."""
filter_obj = PlotLessThan("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 3), # 1.0, 5.0, 10.0
("5.0", 2), # 1.0, 5.0
],
)
def test_less_than_or_equal_to(self, target, expected_count):
"""Test PlotLessThanOrEqualTo with numeric values."""
filter_obj = PlotLessThanOrEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 2), # 50.0, 100.0
("5.0", 3), # 10.0, 50.0, 100.0
],
)
def test_greater_than(self, target, expected_count):
"""Test PlotGreaterThan with numeric values."""
filter_obj = PlotGreaterThan("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 3), # 10.0, 50.0, 100.0
("5.0", 4), # 5.0, 10.0, 50.0, 100.0
],
)
def test_greater_than_or_equal_to(self, target, expected_count):
"""Test PlotGreaterThanOrEqualTo with numeric values."""
filter_obj = PlotGreaterThanOrEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"filter_str,expected_var,expected_target,expected_type",
[
("request_rate==5.0", "request_rate", "5.0", PlotEqualTo),
("request_rate!=10.0", "request_rate", "10.0", PlotNotEqualTo),
("request_rate<50.0", "request_rate", "50.0", PlotLessThan),
("request_rate<=50.0", "request_rate", "50.0", PlotLessThanOrEqualTo),
("request_rate>10.0", "request_rate", "10.0", PlotGreaterThan),
("request_rate>=10.0", "request_rate", "10.0", PlotGreaterThanOrEqualTo),
("request_rate==inf", "request_rate", "inf", PlotEqualTo),
("request_rate!='inf'", "request_rate", "inf", PlotNotEqualTo),
],
)
def test_parse_str(self, filter_str, expected_var, expected_target, expected_type):
"""Test parsing filter strings."""
filter_obj = PlotFilterBase.parse_str(filter_str)
assert isinstance(filter_obj, expected_type)
assert filter_obj.var == expected_var
assert filter_obj.target == expected_target
def test_parse_str_inf_edge_case(self):
"""Test parsing 'inf' string in filter."""
filter_obj = PlotFilterBase.parse_str("request_rate==inf")
assert isinstance(filter_obj, PlotEqualTo)
assert filter_obj.var == "request_rate"
assert filter_obj.target == "inf"
def test_parse_multiple_filters(self):
"""Test parsing multiple filters."""
filters = PlotFilters.parse_str("request_rate>5.0,value<=40")
assert len(filters) == 2
assert isinstance(filters[0], PlotGreaterThan)
assert isinstance(filters[1], PlotLessThanOrEqualTo)
def test_parse_empty_filter(self):
"""Test parsing empty filter string."""
filters = PlotFilters.parse_str("")
assert len(filters) == 0
-486
View File
@@ -1,486 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from typing import Any, NamedTuple, cast
import numpy as np
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import (
RandomDataset,
RandomMultiModalDataset,
SampleRequest,
)
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Use a small, commonly available tokenizer
return AutoTokenizer.from_pretrained("openai-community/gpt2")
class Params(NamedTuple):
num_requests: int
prefix_len: int
range_ratio: float
input_len: int
output_len: int
@pytest.fixture(scope="session")
def random_dataset_params() -> Params:
return Params(
num_requests=16, prefix_len=7, range_ratio=0.3, input_len=50, output_len=20
)
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
"""Project a SampleRequest into a comparable tuple."""
assert isinstance(req.prompt, str)
return (req.prompt, req.prompt_len, req.expected_output_len)
def _collect_samples(
dataset: RandomDataset,
tokenizer: PreTrainedTokenizerBase,
num_requests: int = 16,
prefix_len: int = 7,
range_ratio: float = 0.3,
input_len: int = 50,
output_len: int = 20,
) -> list[tuple[str, int, int]]:
samples = dataset.sample(
tokenizer=tokenizer,
num_requests=num_requests,
prefix_len=prefix_len,
range_ratio=range_ratio,
input_len=input_len,
output_len=output_len,
)
return [_fingerprint_sample(s) for s in samples]
@pytest.mark.benchmark
def test_random_dataset_same_seed(
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
) -> None:
"""Same seed should yield identical outputs, even if global RNGs change.
This guards against accidental reliance on Python's random or np.random
in RandomDataset after moving to numpy.default_rng.
"""
p = random_dataset_params
common_seed = 123
dataset_a = RandomDataset(random_seed=common_seed)
dataset_b = RandomDataset(random_seed=common_seed)
a = _collect_samples(
dataset_a,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
# Perturb global RNG state to ensure isolation
random.seed(999)
_ = [random.random() for _ in range(100)]
np.random.seed(888)
_ = [np.random.random() for _ in range(100)]
b = _collect_samples(
dataset_b,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
assert a == b
@pytest.mark.benchmark
def test_random_dataset_different_seeds(
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
) -> None:
"""Different seeds should change outputs with overwhelming likelihood."""
p = random_dataset_params
seed_a = 0
dataset_a = RandomDataset(random_seed=seed_a)
a = _collect_samples(
dataset_a,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
seed_b = 999
dataset_b = RandomDataset(random_seed=seed_b)
# Perturb global RNG with same seed as dataset_a to ensure isolation
random.seed(seed_a)
np.random.seed(seed_a)
b = _collect_samples(
dataset_b,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
assert a != b
# -----------------------------
# RandomMultiModalDataset tests
# -----------------------------
def _mm_fingerprint_sample(
req: SampleRequest,
) -> tuple[str, int, int, int, list[str]]:
"""Create a compact fingerprint for multimodal samples.
Includes:
- prompt string
- prompt_len
- expected_output_len
- count of multimodal items
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
"""
assert isinstance(req.prompt, str)
items = req.multi_modal_data or []
item_prefixes: list[str] = []
for it in items:
if isinstance(it, dict) and it.get("type") == "image_url":
url = it.get("image_url", {}).get("url", "")
# Only keep a short identifying prefix to avoid huge strings
item_prefixes.append(f"image:{url[:22]}")
elif isinstance(it, dict) and it.get("type") == "video_url":
url = it.get("video_url", {}).get("url", "")
item_prefixes.append(f"video:{url[:22]}")
else:
item_prefixes.append("unknown:")
return (
req.prompt,
req.prompt_len,
req.expected_output_len,
len(items),
item_prefixes,
)
def _collect_mm_samples(
dataset: RandomMultiModalDataset,
tokenizer: PreTrainedTokenizerBase,
*,
num_requests: int = 8,
prefix_len: int = 3,
range_ratio: float = 0.0,
input_len: int = 20,
output_len: int = 5,
base_items_per_request: int = 2,
num_mm_items_range_ratio: float = 0.0,
limit_mm_per_prompt: dict[str, int] | None = None,
bucket_config: dict[tuple[int, int, int], float] | None = None,
enable_multimodal_chat: bool = False,
) -> list[SampleRequest]:
if limit_mm_per_prompt is None:
limit_mm_per_prompt = {"image": 5, "video": 0}
if bucket_config is None:
bucket_config = {(32, 32, 1): 0.5, (52, 64, 1): 0.5}
return dataset.sample(
tokenizer=tokenizer,
num_requests=num_requests,
prefix_len=prefix_len,
range_ratio=range_ratio,
input_len=input_len,
output_len=output_len,
base_items_per_request=base_items_per_request,
num_mm_items_range_ratio=num_mm_items_range_ratio,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
enable_multimodal_chat=enable_multimodal_chat,
)
@pytest.mark.benchmark
def test_random_mm_same_seed(hf_tokenizer: PreTrainedTokenizerBase) -> None:
seed = 42
ds_a = RandomMultiModalDataset(random_seed=seed)
ds_b = RandomMultiModalDataset(random_seed=seed)
a = _collect_mm_samples(ds_a, hf_tokenizer)
b = _collect_mm_samples(ds_b, hf_tokenizer)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa == fb
@pytest.mark.benchmark
def test_random_mm_different_seeds(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds_a = RandomMultiModalDataset(random_seed=0)
ds_b = RandomMultiModalDataset(random_seed=999)
a = _collect_mm_samples(ds_a, hf_tokenizer)
b = _collect_mm_samples(ds_b, hf_tokenizer)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa != fb
@pytest.mark.benchmark
def test_random_mm_respects_limits(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Requesting 3 items with a per-prompt limit of 1 should error per current
# design (dataset refuses to silently clamp below the requested baseline).
with pytest.raises(ValueError):
_collect_mm_samples(
ds,
hf_tokenizer,
num_requests=12,
base_items_per_request=3,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 1, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
@pytest.mark.benchmark
def test_random_mm_zero_prob_entries_are_removed(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Second bucket has zero probability and should be ignored after
# normalization
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=6,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 10, "video": 0},
bucket_config={(32, 32, 1): 1.0, (52, 64, 1): 0.0},
)
for s in samples:
assert isinstance(s.multi_modal_data, list)
typed_mm = cast(list[dict[str, Any]], s.multi_modal_data)
for it in typed_mm:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_zero_items(hf_tokenizer: PreTrainedTokenizerBase) -> None:
ds = RandomMultiModalDataset(random_seed=0)
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=0,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 5, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
for s in samples:
assert s.multi_modal_data == []
@pytest.mark.benchmark
def test_random_mm_num_items_per_prompt(hf_tokenizer: PreTrainedTokenizerBase) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Fixed number of images per prompt
# set num_mm_items_range_ratio to 0.0
# TODO: modify video values when video sampling is implemented
samples_fixed_items = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=3,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 3, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
# Must have 5 requests each with 3 mm items per prompt
assert len(samples_fixed_items) == 5
for s in samples_fixed_items:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 3
for it in mm_data:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_bucket_config_not_mutated(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# This bucket config is not normalized to sum to 1
# and has more buckets than requested images
original = {(32, 32, 1): 0.2, (52, 64, 1): 6, (25, 64, 1): 3}
# Keep a snapshot to compare after sampling
snapshot = dict(original)
_ = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=4,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 1, "video": 0},
bucket_config=original,
)
# Ensure the original dict content is unchanged
assert original == snapshot
# Vary number of mm items per prompt
# set num_mm_items_range_ratio to 0.5
samples_varying_items = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=2,
num_mm_items_range_ratio=0.5,
limit_mm_per_prompt={"image": 4, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
# Must have 5 requests each with less than 4 mm items per prompt
# but at least 1 mm item per prompt
assert len(samples_varying_items) == 5
for s in samples_varying_items:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) <= 4
assert len(mm_data) >= 1
for it in mm_data:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
"""Test video sampling functionality in RandomMultiModalDataset."""
ds = RandomMultiModalDataset(random_seed=42)
# Test with video bucket configuration
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
assert len(samples) == 5
# Check that we have both images and videos
video_count = 0
image_count = 0
for s in samples:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
if item.get("type") == "video_url":
video_count += 1
# Verify video URL format
url = item.get("video_url", {}).get("url", "")
assert url.startswith("data:video/mp4;base64,")
elif item.get("type") == "image_url":
image_count += 1
# Verify image URL format
url = item.get("image_url", {}).get("url", "")
assert url.startswith("data:image/jpeg;base64,")
# Should have some videos due to 0.7 probability
assert video_count > 0
assert image_count > 0
@pytest.mark.benchmark
def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
"""Test sampling with only video buckets."""
ds = RandomMultiModalDataset(random_seed=42)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
assert len(samples) == 3
for s in samples:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
assert item.get("type") == "video_url"
url = item.get("video_url", {}).get("url", "")
assert url.startswith("data:video/mp4;base64,")
@pytest.mark.benchmark
def test_random_mm_video_deterministic_sampling(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Test that video sampling is deterministic with same seed."""
seed = 123
ds_a = RandomMultiModalDataset(random_seed=seed)
ds_b = RandomMultiModalDataset(random_seed=seed)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
a = _collect_mm_samples(
ds_a,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
b = _collect_mm_samples(
ds_b,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa == fb
@@ -1,434 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import tempfile
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, cast
from unittest.mock import Mock
import cv2
import pybase64 as base64
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
"""Use a small, commonly available tokenizer."""
return AutoTokenizer.from_pretrained("openai-community/gpt2")
@pytest.fixture
def video_dataset() -> RandomMultiModalDataset:
"""Create a RandomMultiModalDataset instance for testing."""
return RandomMultiModalDataset(random_seed=42)
@pytest.mark.benchmark
def test_generate_synthetic_video_different_seeds():
"""Test that different seeds produce different videos."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=456)
width, height, num_frames = 64, 48, 8
video1 = dataset1.generate_synthetic_video(width, height, num_frames)
video2 = dataset2.generate_synthetic_video(width, height, num_frames)
# Videos should be different due to different seeds
assert video1["bytes"] != video2["bytes"]
@pytest.mark.benchmark
def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
"""Test modality mapping for different configurations."""
# Test image configuration (num_frames = 1)
assert video_dataset.map_config_to_modality((256, 256, 1)) == "image"
assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image"
# Test video configurations (num_frames > 1)
assert video_dataset.map_config_to_modality((256, 256, 8)) == "video"
assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video"
assert video_dataset.map_config_to_modality((64, 64, 32)) == "video"
# Test invalid configurations
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.map_config_to_modality((256, 256, 0))
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.map_config_to_modality((256, 256, -1))
@pytest.mark.benchmark
def test_generate_mm_item_video(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
"""Generate a decodable video without leaving temporary files behind."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
# Test video item generation
video_config = (64, 48, 8) # height, width, num_frames
result = video_dataset.generate_mm_item(video_config)
assert not list(tmp_path.iterdir())
# Check the result structure matches OpenAI API format
assert isinstance(result, dict)
assert result["type"] == "video_url"
assert "video_url" in result
assert "url" in result["video_url"]
# Check that the URL is a data URL with base64 encoded video
url = result["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
# Decode and verify the video content
base64_data = url.split(",")[1]
video_bytes = base64.b64decode(base64_data)
assert len(video_bytes) > 0
# Verify the video can be decoded
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
temp_path = temp_file.name
temp_file.write(video_bytes)
try:
cap = cv2.VideoCapture(temp_path)
assert cap.isOpened()
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
assert frame_count == 8
assert frame_width == 48
assert frame_height == 64
cap.release()
finally:
if os.path.exists(temp_path):
os.unlink(temp_path)
@pytest.mark.benchmark
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
def test_generate_synthetic_video_cleans_up_on_error(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
writer_opened: bool,
):
"""Release the writer and remove temporary storage when encoding fails."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
writer = Mock(spec=cv2.VideoWriter)
writer.isOpened.return_value = writer_opened
writer.write.side_effect = RuntimeError("Failed to write video frame")
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
error = (
"Failed to write video frame"
if writer_opened
else "Failed to create video writer"
)
with pytest.raises(RuntimeError, match=error):
video_dataset.generate_synthetic_video(48, 64, 8)
assert not list(tmp_path.iterdir())
writer.release.assert_called_once_with()
@pytest.mark.benchmark
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
"""Test generating multimodal items for image configurations."""
# Test image item generation
image_config = (64, 48, 1) # height, width, num_frames=1
result = video_dataset.generate_mm_item(image_config)
# Check the result structure matches OpenAI API format
assert isinstance(result, dict)
assert result["type"] == "image_url"
assert "image_url" in result
assert "url" in result["image_url"]
# Check that the URL is a data URL with base64 encoded image
url = result["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
@pytest.mark.benchmark
def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset):
"""Test error handling for invalid configurations."""
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.generate_mm_item((256, 256, 0))
@pytest.mark.benchmark
def test_sample_with_video_buckets(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with video bucket configurations."""
# Configure bucket with video probability > 0
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 5, "video": 3}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=5,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 5
# Check that samples contain both images and videos
video_count = 0
image_count = 0
for sample in samples:
assert isinstance(sample, SampleRequest)
assert sample.multi_modal_data is not None
assert isinstance(sample.multi_modal_data, list)
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) == 2 # base_items_per_request
for item in mm_data:
if item["type"] == "video_url":
video_count += 1
# Verify video URL format
url = item["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
elif item["type"] == "image_url":
image_count += 1
# Verify image URL format
url = item["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
# Should have some videos due to 0.7 probability
assert video_count > 0
assert image_count > 0
@pytest.mark.benchmark
def test_sample_video_only_buckets(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with only video buckets."""
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 2}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 3
for sample in samples:
assert isinstance(sample, SampleRequest)
assert sample.multi_modal_data is not None
assert isinstance(sample.multi_modal_data, list)
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
assert item["type"] == "video_url"
url = item["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
@pytest.mark.benchmark
def test_sample_respects_video_limits(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test that sampling respects video limits per prompt."""
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
# Set very low video limit
limit_mm_per_prompt = {"image": 0, "video": 1}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 3
for sample in samples:
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) <= 1 # Should respect video limit
@pytest.mark.benchmark
def test_sample_mixed_buckets_with_zero_probability(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with mixed buckets including zero probability entries."""
bucket_config = {
(64, 64, 1): 0.5, # Images
(64, 64, 8): 0.5, # Videos
(128, 128, 16): 0.0, # Zero probability videos (should be ignored)
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=4,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 4
# Should only see 64x64 videos, not 128x128 videos
for sample in samples:
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
for item in mm_data:
if item["type"] == "video_url":
# Decode video to verify dimensions
url = item["video_url"]["url"]
base64_data = url.split(",")[1]
video_bytes = base64.b64decode(base64_data)
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa
temp_path = temp_file.name
temp_file.write(video_bytes)
try:
cap = cv2.VideoCapture(temp_path)
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
cap.release()
# Should be 64x64, not 128x128
assert frame_width == 64
assert frame_height == 64
finally:
if os.path.exists(temp_path):
os.unlink(temp_path)
@pytest.mark.benchmark
def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase):
"""Test that sampling with videos is deterministic with same seed."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=123)
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples1 = dataset1.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
samples2 = dataset2.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples1) == len(samples2)
# Compare multimodal data
for s1, s2 in zip(samples1, samples2):
assert s1.multi_modal_data == s2.multi_modal_data
@pytest.mark.benchmark
def test_sample_different_seeds_produce_different_videos(
hf_tokenizer: PreTrainedTokenizerBase,
):
"""Test that different seeds produce different video content."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=456)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
samples1 = dataset1.sample(
tokenizer=hf_tokenizer,
num_requests=2,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
samples2 = dataset2.sample(
tokenizer=hf_tokenizer,
num_requests=2,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
# Video content should be different
for s1, s2 in zip(samples1, samples2):
mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data)
mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data)
assert len(mm_data1) == len(mm_data2) == 1
url1 = mm_data1[0]["video_url"]["url"]
url2 = mm_data2[0]["video_url"]["url"]
assert url1 != url2 # Different video content
@@ -1,453 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
The Responses stream differs from Chat Completions in ways the benchmark client
has to get right: every SSE message is labelled with an ``event:`` line,
reasoning and output text arrive as separate token-bearing events, and a
request ends with ``response.completed`` rather than with its last token. These
tests drive the real request function against a local SSE server so the
resulting latency and usage numbers are covered end to end.
"""
import asyncio
import json
from collections.abc import Sequence
import aiohttp
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
from vllm.benchmarks.lib.endpoint_request_func import (
ASYNC_REQUEST_FUNCS,
OPENAI_COMPATIBLE_BACKENDS,
RequestFuncInput,
RequestFuncOutput,
async_request_openai_responses,
)
# These tests never touch torch, so skip the GPU cleanup fixture.
pytestmark = pytest.mark.skip_global_cleanup
MODEL = "openai/gpt-oss-20b"
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
# that a time to first token measured at the wrong event is unambiguous.
GAP_PREFILL = 0.15
GAP_TOKEN = 0.5
GAP_FINALIZE = 0.3
def sse(event_type: str, payload: dict) -> bytes:
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
``ensure_ascii=False`` matches the server, whose pydantic
``model_dump_json`` emits non-ASCII characters unescaped.
"""
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
return f"event: {event_type}\ndata: {data}\n\n".encode()
def usage_event(
input_tokens: int,
output_tokens: int,
event_type: str = "response.completed",
status: str = "completed",
):
return sse(
event_type,
{
"response": {
"status": status,
"usage": {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": input_tokens + output_tokens,
},
}
},
)
def text_delta(delta: str) -> bytes:
return sse("response.output_text.delta", {"delta": delta})
def reasoning_delta(delta: str) -> bytes:
return sse("response.reasoning_text.delta", {"delta": delta})
# The metadata events the server sends before any token. None of these may
# start the TTFT clock or contribute an ITL sample.
PREAMBLE = (
sse("response.created", {"response": {"status": "in_progress"}})
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
+ sse("response.output_item.added", {"output_index": 0})
+ sse("response.reasoning_part.added", {"output_index": 0})
+ sse("response.content_part.added", {"output_index": 0})
)
# The `.done` family the server sends after the last delta. These repeat the
# accumulated text, so timing them would add bogus ITL samples and counting
# their payload would duplicate the output.
DONE_EVENTS = (
sse("response.reasoning_text.done", {"text": "The user asks "})
+ sse("response.reasoning_part.done", {"output_index": 0})
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
+ sse("response.content_part.done", {"output_index": 0})
+ sse("response.output_item.done", {"output_index": 0})
)
async def _serve(
script: Sequence[bytes | float],
*,
status: int = 200,
chunk_size: int | None = None,
seen: dict | None = None,
**request_kwargs,
) -> RequestFuncOutput:
"""Run the request function against a server replaying ``script``.
Args:
script: Byte strings to write, interleaved with floats to sleep for.
status: HTTP status to respond with. Non-200 skips the script.
chunk_size: If set, split every write into chunks of this many bytes to
emulate fragmented transport.
seen: If given, populated with the request payload.
request_kwargs: Extra fields for the ``RequestFuncInput``.
"""
async def handler(request: web.Request) -> web.StreamResponse:
if seen is not None:
seen["payload"] = await request.json()
seen["headers"] = dict(request.headers)
if status != 200:
return web.Response(status=status, reason="Bad Request")
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
for item in script:
if isinstance(item, float):
await asyncio.sleep(item)
elif chunk_size is None:
await response.write(item)
else:
for i in range(0, len(item), chunk_size):
await response.write(item[i : i + chunk_size])
await response.write_eof()
return response
app = web.Application()
app.router.add_post("/v1/responses", handler)
server = TestServer(app)
await server.start_server()
try:
request_input = RequestFuncInput(
prompt="Why is the sky blue?",
api_url=str(server.make_url("/v1/responses")),
prompt_len=5,
output_len=16,
model=MODEL,
**request_kwargs,
)
async with aiohttp.ClientSession() as session:
return await async_request_openai_responses(request_input, session)
finally:
await server.close()
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
return asyncio.run(_serve(script, **kwargs))
def test_backend_is_registered_as_openai_compatible():
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
# Membership enables --ignore-eos on random datasets and the sampling
# parameter flags, all of which ResponsesRequest accepts.
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
def test_request_payload_is_accepted_by_the_server_schema():
"""Every field sent must be a real ResponsesRequest field.
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
the flags that membership enables have to reach the model, and the server
drops unknown fields without complaining.
"""
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
sampling = {
"temperature": 0.7,
"top_p": 0.9,
"top_k": 20,
"min_p": 0.05,
"frequency_penalty": 0.2,
"presence_penalty": 0.1,
"repetition_penalty": 1.05,
}
seen: dict = {}
output = run(
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
seen=seen,
extra_body=sampling,
ignore_eos=True,
model_name="served-alias",
request_id="bench-7",
extra_headers={"x-team": "perf"},
)
assert output.success
# --served-model-name wins over --model, as on the other backends.
assert seen["payload"]["model"] == "served-alias"
# The route is behind validate_json_request, which 415s a missing type.
assert seen["headers"]["Content-Type"] == "application/json"
assert seen["headers"]["x-request-id"] == "bench-7"
assert seen["headers"]["x-team"] == "perf"
assert seen["payload"] == {
"model": "served-alias",
"input": "Why is the sky blue?",
"max_output_tokens": 16,
"stream": True,
"store": False,
"ignore_eos": True,
**sampling,
}
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
ResponsesRequest.model_validate(seen["payload"])
def test_latency_accounting_matches_the_other_endpoints():
"""`latency - ttft` must equal `sum(itl)`.
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
means average inter-token latency when that identity holds. #55508 fixed the
same accounting on the chat and audio backends; the trailing events after
the last token must not extend the request.
"""
output = run(
[
PREAMBLE,
GAP_PREFILL / 2,
# A keep-alive comment sent during prefill is not a token.
b": keep-alive\n\n",
GAP_PREFILL / 2,
reasoning_delta("The user asks "),
GAP_TOKEN,
text_delta("Rayleigh"),
text_delta(" scattering."),
DONE_EVENTS,
GAP_FINALIZE,
usage_event(5, 8),
]
)
assert output.success
# Reasoning is timed but not returned, matching how `openai-chat` treats
# `DeltaMessage.reasoning`.
assert output.generated_text == "Rayleigh scattering."
# Neither the response.created/in_progress/output_item.added preamble nor
# the keep-alive comment may start the clock, and the reasoning delta must,
# so TTFT lands on the prefill gap rather than near zero or out at the
# first output text delta.
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
# One ITL per token event after the first: the gap before "Rayleigh" and
# the back-to-back delta that follows it.
assert len(output.itl) == 2
assert output.itl[0] >= GAP_TOKEN * 0.5
assert output.itl[1] < GAP_TOKEN * 0.5
# The identity is exact, so no wall-clock margin is involved. The .done
# family and the GAP_FINALIZE wait before response.completed sit outside
# the request, exactly as the trailing usage chunk does on chat.
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
assert output.start_time > 0.0
def test_usage_is_taken_from_the_terminal_event():
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
assert output.success
assert output.prompt_len == 17
assert output.output_tokens == 4
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
def test_incomplete_response_is_a_successful_request(event_type: str):
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
The spec's dedicated `response.incomplete` event is accepted as well.
"""
output = run(
[
PREAMBLE,
text_delta("truncated"),
usage_event(17, 16, event_type=event_type, status="incomplete"),
]
)
assert output.success
assert output.generated_text == "truncated"
assert output.output_tokens == 16
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
"""Splits inside the event line, the JSON, or a multibyte character."""
text = " leading 中 trailing "
output = run(
[PREAMBLE, text_delta(text), usage_event(5, 6)],
chunk_size=chunk_size,
)
assert output.success
assert output.generated_text == text
assert output.output_tokens == 6
@pytest.mark.parametrize(
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
)
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
"""The server emits these unescaped and they are legal inside a JSON string.
`str.splitlines()` treats them as line breaks, which would cut the `data:`
line short and fail the request on valid model output.
"""
text = f"before{char}after"
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
assert output.success, output.error
assert output.generated_text == text
def test_empty_data_line_is_skipped():
"""A bare `data:` line carries no JSON and must not fail the request."""
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
assert output.success, output.error
assert output.generated_text == "ok"
def test_prompt_len_falls_back_when_usage_is_absent():
"""A terminal event without usage keeps the locally computed prompt length."""
output = run(
[
PREAMBLE,
text_delta("hi"),
sse("response.completed", {"response": {"status": "completed"}}),
]
)
assert output.success, output.error
assert output.prompt_len == 5
def test_multi_line_data_payload_is_joined_with_newlines():
"""The SSE spec allows one payload to span several `data:` lines."""
script: list[bytes | float] = [
PREAMBLE,
b"event: response.output_text.delta\n"
b'data: {"type": "response.output_text.delta",\n'
b'data: "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success, output.error
assert output.generated_text == "ok"
def test_done_sentinel_is_skipped():
"""Other Responses servers send `[DONE]`, but vLLM does not."""
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
assert output.success, output.error
assert output.generated_text == "ok"
def test_error_before_any_token_reports_the_server_error():
"""The server's message must win over the generic no-TTFT message."""
output = run(
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
)
assert not output.success
assert "boom" in output.error
def test_http_error_is_not_reported_as_success():
output = run([], status=400)
assert not output.success
assert output.error
def test_stream_without_tokens_is_not_reported_as_success():
"""A stream that finishes without generating anything has no TTFT."""
output = run([PREAMBLE, usage_event(5, 0)])
assert not output.success
assert "TTFT" in output.error
def test_truncated_stream_is_not_reported_as_success():
"""A stream cut short after a token has no usage and an understated E2EL."""
output = run([PREAMBLE, text_delta("partial")])
assert not output.success
assert "terminal event" in output.error
def test_mid_stream_error_is_not_reported_as_success():
"""An error payload mid-stream must not be counted as a success.
vLLM cannot emit this today: its streaming error path fails to validate
against the event union, so a generation failure truncates the stream
instead. Proxies and other Responses servers do send it.
"""
output = run(
[
PREAMBLE,
text_delta("partial"),
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
]
)
assert not output.success
assert "boom" in output.error
def test_data_only_framing_is_still_parsed():
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
script: list[bytes | float] = [
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success
assert output.generated_text == "ok"
def test_endpoint_path_is_validated():
request_input = RequestFuncInput(
prompt="hi",
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=1,
output_len=1,
model=MODEL,
)
async def call() -> None:
async with aiohttp.ClientSession() as session:
await async_request_openai_responses(request_input, session)
with pytest.raises(ValueError, match="responses"):
asyncio.run(call())
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Keep the `vllm bench serve` flag snapshot in sync for the Rust CLI parity test.
`rust/src/bench/tests/cli_parity.rs` asserts that every flag in the snapshot is
either a known `vllm-bench` clap flag/alias or explicitly allowlisted as
Python-only, so `VLLM_USE_RUST_BENCH=1` delegation cannot silently reject
documented flags. This test keeps the snapshot itself current.
Run this file directly to regenerate the snapshot:
.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
"""
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
SNAPSHOT = REPO_ROOT / "rust" / "src" / "bench" / "tests" / "python_serve_flags.txt"
HEADER = """\
# Long CLI flags of the Python `vllm bench serve` parser
# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
"""
def _current_flags() -> list[str]:
from vllm.benchmarks.serve import add_cli_args
from vllm.utils.argparse_utils import FlexibleArgumentParser
parser = FlexibleArgumentParser()
add_cli_args(parser)
return sorted(
{
opt
for action in parser._actions
for opt in action.option_strings
if opt.startswith("--") and opt != "--help"
}
)
def _snapshot_flags() -> list[str]:
lines = SNAPSHOT.read_text().splitlines()
return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
def test_serve_flag_snapshot_is_current():
assert SNAPSHOT.is_file(), f"missing snapshot {SNAPSHOT}"
assert _current_flags() == _snapshot_flags(), (
"`vllm bench serve` flags changed. Regenerate the snapshot with "
"`.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py`, "
"then for any added flag either support it in rust/src/bench/src/cli.rs "
"or add it to PYTHON_ONLY in rust/src/bench/tests/cli_parity.rs"
)
if __name__ == "__main__":
SNAPSHOT.write_text(HEADER + "\n".join(_current_flags()) + "\n")
print(f"wrote {SNAPSHOT}")
-258
View File
@@ -1,258 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
from vllm.benchmarks.datasets.utils import get_sampling_params
from vllm.tokenizers import TokenizerLike
class _FakeTokenizer(TokenizerLike):
"""Minimal tokenizer implementing the TokenizerLike protocol
for testing get_sampling_params."""
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
self._vocab_size = vocab_size
self._num_special_tokens = num_special_tokens
# -- Properties required by TokenizerLike --
@classmethod
def from_pretrained(cls, path_or_repo_id, *a, **kw):
return cls()
@property
def vocab_size(self) -> int:
return self._vocab_size
@property
def all_special_tokens(self) -> list[str]:
return []
@property
def all_special_ids(self) -> list[int]:
return []
@property
def bos_token_id(self) -> int:
return 0
@property
def eos_token_id(self) -> int:
return 1
@property
def pad_token_id(self) -> int:
return 2
@property
def is_fast(self) -> bool:
return False
@property
def max_token_id(self) -> int:
return self._vocab_size - 1
@property
def max_chars_per_token(self) -> int:
return 4
@property
def truncation_side(self) -> str:
return "right"
def num_special_tokens_to_add(self) -> int:
return self._num_special_tokens
def __call__(self, text, text_pair=None, **kw): # type: ignore[override]
raise NotImplementedError
def get_vocab(self) -> dict[str, int]:
return {}
def get_added_vocab(self) -> dict[str, int]:
return {}
def encode(self, text, **kw) -> list[int]: # type: ignore[override]
raise NotImplementedError
def apply_chat_template(self, messages, **kw): # type: ignore[override]
raise NotImplementedError
def convert_tokens_to_ids(self, tokens):
raise NotImplementedError
def convert_tokens_to_string(self, tokens: list[str]) -> str:
raise NotImplementedError
def decode(self, ids, skip_special_tokens: bool = False) -> str:
raise NotImplementedError
def convert_ids_to_tokens(
self, ids, skip_special_tokens: bool = False
) -> list[str]:
raise NotImplementedError
class TestGetSamplingParams:
"""Tests for ``get_sampling_params`` in ``vllm.benchmarks.datasets.shared``."""
# -- helpers --
@staticmethod
def _tok(vocab_size: int = 1000, num_special: int = 0) -> _FakeTokenizer:
return _FakeTokenizer(vocab_size=vocab_size, num_special_tokens=num_special)
# -- return shape / dtype --
def test_returns_three_arrays(self):
rng = np.random.default_rng(0)
result = get_sampling_params(rng, 5, 0.0, 100, 50, self._tok())
assert len(result) == 3
for arr in result:
assert isinstance(arr, np.ndarray)
@pytest.mark.parametrize("n", [1, 10, 100])
def test_output_length_matches_num_requests(self, n: int):
rng = np.random.default_rng(42)
input_lens, output_lens, offsets = get_sampling_params(
rng, n, 0.0, 64, 32, self._tok()
)
assert input_lens.shape == (n,)
assert output_lens.shape == (n,)
assert offsets.shape == (n,)
# -- fixed lengths (range_ratio = 0) --
def test_zero_range_ratio_gives_constant_lengths(self):
rng = np.random.default_rng(7)
input_lens, output_lens, _ = get_sampling_params(
rng, 20, 0.0, 128, 64, self._tok()
)
assert np.all(input_lens == 128)
assert np.all(output_lens == 64)
def test_special_tokens_subtracted_from_input_only(self):
rng = np.random.default_rng(7)
input_lens, output_lens, _ = get_sampling_params(
rng, 10, 0.0, 100, 50, self._tok(num_special=4)
)
# real_input_len = 100 - 4 = 96, range_ratio 0 → all 96
assert np.all(input_lens == 96)
# special tokens are not subtracted from output length
assert np.all(output_lens == 50)
# -- range ratios --
def test_input_range_bounds(self):
rng = np.random.default_rng(0)
ratio = 0.5
base = 200
input_lens, _, _ = get_sampling_params(
rng, 500, {"input": ratio, "output": 0.0}, base, 50, self._tok()
)
lo = int(np.floor(base * (1 - ratio)))
hi = int(np.ceil(base * (1 + ratio)))
assert np.all(input_lens >= lo)
assert np.all(input_lens <= hi)
def test_output_range_bounds(self):
rng = np.random.default_rng(0)
ratio = 0.3
base = 100
_, output_lens, _ = get_sampling_params(
rng, 500, {"input": 0.0, "output": ratio}, 50, base, self._tok()
)
lo = max(1, int(np.floor(base * (1 - ratio))))
hi = int(np.ceil(base * (1 + ratio)))
assert np.all(output_lens >= lo)
assert np.all(output_lens <= hi)
def test_output_low_clamped_to_one(self):
"""Even with a high ratio that would push output_low to 0,
the function clamps it to 1."""
rng = np.random.default_rng(0)
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
_, output_lens, _ = get_sampling_params(
rng, 50, {"input": 0.0, "output": 0.99}, 100, 1, self._tok()
)
assert np.all(output_lens >= 1)
# -- offsets bounded by vocab_size --
@pytest.mark.parametrize("vocab", [100, 32000, 128256])
def test_offsets_within_vocab(self, vocab: int):
rng = np.random.default_rng(0)
_, _, offsets = get_sampling_params(
rng, 200, 0.0, 64, 32, self._tok(vocab_size=vocab)
)
assert np.all(offsets >= 0)
assert np.all(offsets < vocab)
# -- reproducibility --
def test_same_seed_same_results(self):
tok = self._tok()
rr = {"input": 0.3, "output": 0.2}
a = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
b = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
for arr_a, arr_b in zip(a, b):
np.testing.assert_array_equal(arr_a, arr_b)
def test_different_seed_different_results(self):
tok = self._tok()
rr = {"input": 0.3, "output": 0.2}
a = get_sampling_params(np.random.default_rng(0), 50, rr, 256, 64, tok)
b = get_sampling_params(np.random.default_rng(1), 50, rr, 256, 64, tok)
# Extremely unlikely all three arrays match with different seeds
assert not all(np.array_equal(arr_a, arr_b) for arr_a, arr_b in zip(a, b))
# -- validation / error paths --
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
def test_invalid_input_range_ratio(self, bad_ratio: float):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="input_range_ratio"):
get_sampling_params(
rng, 10, {"input": bad_ratio, "output": 0.0}, 100, 50, self._tok()
)
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
def test_invalid_output_range_ratio(self, bad_ratio: float):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="output_range_ratio"):
get_sampling_params(
rng, 10, {"input": 0.0, "output": bad_ratio}, 100, 50, self._tok()
)
def test_invalid_dict_missing_keys(self):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="input.*output"):
get_sampling_params(rng, 10, {"input": 0.1}, 100, 50, self._tok())
def test_input_len_zero_with_special_tokens(self):
"""input_len < num_special_tokens → real_input_len = 0, which is fine
(range [0, 0])."""
rng = np.random.default_rng(0)
input_lens, _, _ = get_sampling_params(
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
)
# real_input_len = max(0, 5 - 10) = 0
assert np.all(input_lens == 0)
# -- edge cases --
def test_single_request(self):
rng = np.random.default_rng(0)
i, o, off = get_sampling_params(rng, 1, 0.0, 100, 50, self._tok())
assert i.shape == (1,)
assert o.shape == (1,)
assert off.shape == (1,)
def test_large_num_requests(self):
rng = np.random.default_rng(0)
i, o, off = get_sampling_params(rng, 10_000, 0.5, 512, 128, self._tok())
assert i.shape == (10_000,)
assert o.shape == (10_000,)
assert off.shape == (10_000,)
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for --skip-tokenizer-init with --dataset-name custom.
Before the fix (introduced by #39896), running:
vllm bench serve \
--backend vllm-pooling \
--dataset-name custom \
--dataset-path <path> \
--model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \
--endpoint /pooling \
--skip-tokenizer-init \
...
raised immediately with:
AssertionError: Tokenizer must be initialized before loading dataset
even though CustomDataset.sample() already handles tokenizer=None.
This test exercises main_async() directly so it catches any regression
re-introduced at the serve.py level, not just inside get_samples().
"""
import argparse
import asyncio
import json
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
import vllm.benchmarks.serve as serve_module
# Exact prompt payload from the failing benchmark run against a
# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format).
_PRITHVI_PROMPT = {
"data": {
"data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif",
"data_format": "url",
"out_data_format": "b64_json",
"indices": [1, 2, 3, 8, 11, 12],
},
"priority": 0,
"softmax": False,
}
def _write_dataset(path: Path) -> None:
path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n")
def _args(dataset_path: str) -> argparse.Namespace:
"""Reproduce the argparse.Namespace that serve.py builds from the
failing command, including skip_tokenizer_init=True."""
return argparse.Namespace(
# dataset
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=False,
num_prompts=1,
custom_output_len=256,
skip_chat_template=True,
chat_template_kwargs=None,
no_oversample=False,
seed=0,
request_id_prefix="bench-",
# model / tokenizer
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
served_model_name=None,
tokenizer=None,
tokenizer_mode="auto",
trust_remote_code=False,
skip_tokenizer_init=True, # <-- the flag under test
# backend / endpoint
backend="vllm-pooling",
base_url="http://127.0.0.1:8000",
host="127.0.0.1",
port=8000,
endpoint="/pooling",
header=None,
insecure=False,
# traffic
request_rate=16.0,
burstiness=1.0,
max_concurrency=None,
probe_request_rate=0.0,
# misc serve args that main_async reads before reaching get_samples
plot_timeline=False,
plot_dataset_stats=False,
self_timed=None,
metadata=None,
label=None,
logprobs=None,
use_beam_search=False,
ignore_eos=False,
goodput=None,
percentile_metrics="ttft,tpot,itl,e2el",
metric_percentiles="25,50,75,99",
save_result=False,
append_result=False,
result_dir=".",
result_filename=None,
num_warmups=0,
profile=False,
disable_tqdm=True,
lora_modules=None,
lora_assignment="random",
ramp_up_strategy=None,
ramp_up_start_rps=None,
ramp_up_end_rps=None,
ready_check_timeout_sec=0,
extra_body=None,
top_p=None,
top_k=None,
min_p=None,
temperature=None,
frequency_penalty=None,
presence_penalty=None,
repetition_penalty=None,
save_detailed=False,
input_len=None,
output_len=None,
)
@pytest.mark.benchmark
def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None:
"""main_async must not raise AssertionError when skip_tokenizer_init=True
and dataset_name='custom'.
On main (before the fix) this test fails with:
AssertionError: Tokenizer must be initialized before loading dataset
"""
dataset_path = tmp_path / "dataset_url_input_india.jsonl"
_write_dataset(dataset_path)
args = _args(str(dataset_path))
# Patch benchmark() so we never make real HTTP requests — the regression
# triggers before benchmark() is ever called, so this just keeps the test
# fast and hermetic.
mock_result = {
"completed": 1,
"failed": 0,
"total_input_tokens": 1,
"total_output_tokens": 1,
"request_throughput": 1.0,
"output_throughput": 1.0,
"total_token_throughput": 1.0,
"input_lens": [],
"output_lens": [],
"ttfts": [],
"itls": [],
"generated_texts": [],
"errors": [],
"duration": 1.0,
}
with patch.object(
serve_module, "benchmark", new=AsyncMock(return_value=mock_result)
):
# Must NOT raise AssertionError
asyncio.run(serve_module.main_async(args))
-369
View File
@@ -1,369 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ``vllm bench throughput`` CLI.
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
(serve has no analogue), keeping the two benchmarks aligned without duplicating
serve-side dataset coverage.
"""
import subprocess
from argparse import Namespace
from types import SimpleNamespace
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import SampleRequest
from vllm.benchmarks.throughput import (
_run_vllm_chat_requests,
_to_serve_args,
add_cli_args,
assign_loras,
get_requests,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
@pytest.mark.benchmark
def test_bench_throughput():
command = [
"vllm",
"bench",
"throughput",
"--model",
MODEL_NAME,
"--input-len",
"32",
"--output-len",
"1",
"--enforce-eager",
"--load-format",
"dummy",
]
result = subprocess.run(command, capture_output=True, text=True)
print(result.stdout)
print(result.stderr)
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
def test_bench_throughput_accepts_custom_audio_args():
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"custom_audio",
"--dataset-path",
"audio.jsonl",
"--no-oversample",
"--custom-output-len",
"32",
"--enable-multimodal-chat",
]
)
assert args.dataset_name == "custom_audio"
assert args.no_oversample
assert args.custom_output_len == 32
assert args.enable_multimodal_chat
def test_vllm_chat_requests_include_multimodal_content():
class FakeLLM:
def __init__(self):
self.prompts = None
def chat(self, prompts, sampling_params, use_tqdm):
del sampling_params, use_tqdm
self.prompts = prompts
return []
llm = FakeLLM()
audio_content = {
"type": "input_audio",
"input_audio": {"data": "abc", "format": "wav"},
}
request = SampleRequest(
prompt="Transcribe this audio.",
prompt_len=1,
expected_output_len=8,
multi_modal_data=audio_content,
)
_run_vllm_chat_requests(
llm,
[request],
n=1,
disable_detokenize=False,
do_profile=False,
prequeue_requests=False,
)
assert llm.prompts == [
[
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe this audio."},
audio_content,
],
}
]
]
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Small, commonly available tokenizer.
return AutoTokenizer.from_pretrained("openai-community/gpt2")
# -----------------------------
# LoRA assignment (throughput-only post-processing)
# -----------------------------
def _sr(i: int) -> SampleRequest:
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
def _lora_args(
*,
lora_path: str | None,
max_loras: int,
lora_assignment: str = "random",
) -> SimpleNamespace:
return SimpleNamespace(
lora_path=lora_path,
max_loras=max_loras,
lora_assignment=lora_assignment,
)
@pytest.mark.benchmark
def test_assign_loras_noop_without_lora_path() -> None:
"""No lora_path -> requests returned unchanged, no LoRA attached."""
reqs = [_sr(i) for i in range(5)]
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
assert out is reqs
assert all(r.lora_request is None for r in out)
@pytest.mark.benchmark
def test_assign_loras_round_robin() -> None:
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
reqs = [_sr(i) for i in range(6)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
)
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
@pytest.mark.benchmark
def test_assign_loras_random_in_range() -> None:
"""Random assignment stays in [1, max_loras] and covers all IDs."""
reqs = [_sr(i) for i in range(200)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
)
ids = [r.lora_request.lora_int_id for r in out]
assert all(1 <= i <= 5 for i in ids)
assert len(set(ids)) == 5
# -----------------------------
# Issue #50838: MMVU via throughput
# -----------------------------
@pytest.mark.benchmark
def test_get_requests_resolves_mmvu(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
Previously throughput's validate_args rejected MMVU ("is not supported by
hf dataset"). It now resolves through the shared get_samples. The HF
download is stubbed so the resolution + dispatch path is exercised without
network access.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubMMVUDataset:
# MMVU is multimodal in content but, like the real class, does not set
# IS_MULTIMODAL -- so it is not subject to the backend gate.
IS_MULTIMODAL = False
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"yale-nlp/MMVU",
"--backend",
"vllm-chat",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
@pytest.mark.benchmark
def test_get_requests_allows_multimodal_on_plain_vllm_backend(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Regression: --backend vllm must accept multimodal datasets such as ASR.
_run_vllm_requests forwards multi_modal_data straight to LLM.generate, so
the vllm-chat-only gate in get_samples rejected a backend that always
worked.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubASRDataset:
IS_MULTIMODAL = True
SUPPORTED_DATASET_PATHS = {"openslr/librispeech_asr"}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "ASRDataset", _StubASRDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"openslr/librispeech_asr",
"--backend",
"vllm",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
def test_to_serve_args_supports_mm_processor_flags() -> None:
"""The adapter must serve callers without throughput's legacy flags.
``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
nor --backend, and leaves --random-prefix-len at its default of 0. The
adapter must not read the legacy flags unconditionally, and must keep the
explicit 0 instead of coercing it to None (which later crashed
RandomMultiModalDataset.get_prefix on ``None <= 0``).
"""
args = Namespace(
random_input_len=32,
random_output_len=1,
random_prefix_len=0,
)
serve_args = _to_serve_args(args)
assert serve_args.random_input_len == 32
assert serve_args.random_output_len == 1
assert serve_args.random_prefix_len == 0
assert serve_args.backend == "vllm-chat"
assert serve_args.enable_multimodal_chat
assert serve_args.no_oversample is False
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
"""Callers that only set the legacy flags still get them mapped."""
args = Namespace(
input_len=256,
output_len=8,
prefix_len=16,
backend="vllm",
)
serve_args = _to_serve_args(args)
assert serve_args.random_input_len == 256
assert serve_args.random_output_len == 8
assert serve_args.random_prefix_len == 16
assert serve_args.backend == "vllm"
assert not serve_args.enable_multimodal_chat
@pytest.mark.benchmark
def test_get_requests_random_mm_with_mm_processor_args(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Regression: bench mm-processor's random-mm sampling must not crash.
get_requests feeds mm-processor's parsed args (no legacy length/backend
flags) through the shared get_samples dispatch. With the default
--random-prefix-len 0 this previously crashed inside
RandomMultiModalDataset.get_prefix with a TypeError.
"""
from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
parser = FlexibleArgumentParser()
add_mm_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"random-mm",
"--num-prompts",
"3",
"--random-input-len",
"32",
"--random-output-len",
"1",
"--random-mm-base-items-per-request",
"1",
"--random-mm-num-mm-items-range-ratio",
"0",
"--random-mm-limit-mm-per-prompt",
'{"image": 1}',
"--random-mm-bucket-config",
"{(224, 224, 1): 1.0}",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
# mm-processor drives llm.chat, so prompts must be chat-formatted with
# multimodal content attached.
for req in requests:
assert isinstance(req.prompt, list)
assert req.prompt[0]["role"] == "user"
assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from pathlib import Path
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import CustomDataset
from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices_jsonl
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Use a small, commonly available tokenizer
return AutoTokenizer.from_pretrained("openai-community/gpt2")
text_content = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident,
sunt in culpa qui officia deserunt mollit anim id est laborum.
"""
@pytest.mark.benchmark
def test_create_txt_slices_jsonl(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""Test that create_txt_slices_jsonl produces valid JSONL for CustomDataset."""
txt_path = tmp_path / "input.txt"
jsonl_path = tmp_path / "input.txt.jsonl"
txt_path.write_text(text_content)
create_txt_slices_jsonl(
input_path=str(txt_path),
output_path=str(jsonl_path),
tokenizer_name="openai-community/gpt2",
num_prompts=10,
input_len=10,
output_len=10,
)
# Verify the JSONL file is valid and has the expected structure
records = [json.loads(line) for line in jsonl_path.read_text().splitlines()]
assert len(records) == 10
for record in records:
assert "prompt" in record
assert "output_tokens" in record
assert isinstance(record["prompt"], str)
assert record["output_tokens"] == 10
# Verify the JSONL file can be loaded by CustomDataset
dataset = CustomDataset(dataset_path=str(jsonl_path))
samples = dataset.sample(
tokenizer=hf_tokenizer,
num_requests=10,
output_len=10,
skip_chat_template=True,
)
assert len(samples) == 10
assert all(sample.expected_output_len == 10 for sample in samples)