[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,375 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep import serve as sweep_serve
|
||||
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
|
||||
|
||||
|
||||
class TestParameterSweepItem:
|
||||
"""Test ParameterSweepItem functionality."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected",
|
||||
[
|
||||
(
|
||||
{"compilation_config.use_inductor_graph_partition": False},
|
||||
"--compilation-config.use_inductor_graph_partition=false",
|
||||
),
|
||||
(
|
||||
{"compilation_config.use_inductor_graph_partition": True},
|
||||
"--compilation-config.use_inductor_graph_partition=true",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nested_boolean_params(self, input_dict, expected):
|
||||
"""Test that nested boolean params use =true/false syntax."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert expected in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected",
|
||||
[
|
||||
({"enable_prefix_caching": False}, "--no-enable-prefix-caching"),
|
||||
({"enable_prefix_caching": True}, "--enable-prefix-caching"),
|
||||
({"disable_log_stats": False}, "--no-disable-log-stats"),
|
||||
({"disable_log_stats": True}, "--disable-log-stats"),
|
||||
],
|
||||
)
|
||||
def test_non_nested_boolean_params(self, input_dict, expected):
|
||||
"""Test that non-nested boolean params use --no- prefix."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert expected in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compilation_config",
|
||||
[
|
||||
{"cudagraph_mode": "full", "mode": 2, "use_inductor_graph_partition": True},
|
||||
{
|
||||
"cudagraph_mode": "piecewise",
|
||||
"mode": 3,
|
||||
"use_inductor_graph_partition": False,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_nested_dict_value(self, compilation_config):
|
||||
"""Test that nested dict values are serialized as JSON."""
|
||||
item = ParameterSweepItem.from_record(
|
||||
{"compilation_config": compilation_config}
|
||||
)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert "--compilation-config" in cmd
|
||||
# The dict should be JSON serialized
|
||||
idx = cmd.index("--compilation-config")
|
||||
assert json.loads(cmd[idx + 1]) == compilation_config
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected_key,expected_value",
|
||||
[
|
||||
({"model": "test-model"}, "--model", "test-model"),
|
||||
({"max_tokens": 100}, "--max-tokens", "100"),
|
||||
({"temperature": 0.7}, "--temperature", "0.7"),
|
||||
],
|
||||
)
|
||||
def test_string_and_numeric_values(self, input_dict, expected_key, expected_value):
|
||||
"""Test that string and numeric values are handled correctly."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
assert expected_key in cmd
|
||||
assert expected_value in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected_key,key_idx_offset",
|
||||
[
|
||||
({"max_tokens": 200}, "--max-tokens", 1),
|
||||
({"enable_prefix_caching": False}, "--no-enable-prefix-caching", 0),
|
||||
],
|
||||
)
|
||||
def test_replace_existing_parameter(self, input_dict, expected_key, key_idx_offset):
|
||||
"""Test that existing parameters in cmd are replaced."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
|
||||
if key_idx_offset == 1:
|
||||
# Key-value pair
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "--max-tokens", "100", "model"])
|
||||
assert expected_key in cmd
|
||||
idx = cmd.index(expected_key)
|
||||
assert cmd[idx + 1] == "200"
|
||||
assert "100" not in cmd
|
||||
else:
|
||||
# Boolean flag
|
||||
cmd = item.apply_to_cmd(
|
||||
["vllm", "serve", "--enable-prefix-caching", "model"]
|
||||
)
|
||||
assert expected_key in cmd
|
||||
assert "--enable-prefix-caching" not in cmd
|
||||
|
||||
|
||||
class TestParameterSweep:
|
||||
"""Test ParameterSweep functionality."""
|
||||
|
||||
def test_from_records_list(self):
|
||||
"""Test creating ParameterSweep from a list of records."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
assert sweep[0]["max_tokens"] == 100
|
||||
assert sweep[1]["max_tokens"] == 200
|
||||
|
||||
def test_read_from_dict(self):
|
||||
"""Test creating ParameterSweep from a dict format."""
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
sweep = ParameterSweep.read_from_dict(data)
|
||||
assert len(sweep) == 2
|
||||
|
||||
# Check that items have the _benchmark_name field
|
||||
names = {item["_benchmark_name"] for item in sweep}
|
||||
assert names == {"experiment1", "experiment2"}
|
||||
|
||||
# Check that parameters are preserved
|
||||
for item in sweep:
|
||||
if item["_benchmark_name"] == "experiment1":
|
||||
assert item["max_tokens"] == 100
|
||||
assert item["temperature"] == 0.7
|
||||
elif item["_benchmark_name"] == "experiment2":
|
||||
assert item["max_tokens"] == 200
|
||||
assert item["temperature"] == 0.9
|
||||
|
||||
def test_read_json_list_format(self):
|
||||
"""Test reading JSON file with list format."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
||||
json.dump(records, f)
|
||||
temp_path = Path(f.name)
|
||||
|
||||
try:
|
||||
sweep = ParameterSweep.read_json(temp_path)
|
||||
assert len(sweep) == 2
|
||||
assert sweep[0]["max_tokens"] == 100
|
||||
assert sweep[1]["max_tokens"] == 200
|
||||
finally:
|
||||
temp_path.unlink()
|
||||
|
||||
def test_read_json_dict_format(self):
|
||||
"""Test reading JSON file with dict format."""
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
||||
json.dump(data, f)
|
||||
temp_path = Path(f.name)
|
||||
|
||||
try:
|
||||
sweep = ParameterSweep.read_json(temp_path)
|
||||
assert len(sweep) == 2
|
||||
|
||||
# Check that items have the _benchmark_name field
|
||||
names = {item["_benchmark_name"] for item in sweep}
|
||||
assert names == {"experiment1", "experiment2"}
|
||||
finally:
|
||||
temp_path.unlink()
|
||||
|
||||
def test_unique_benchmark_names_validation(self):
|
||||
"""Test that duplicate _benchmark_name values raise an error."""
|
||||
# Test with duplicate names in list format
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
|
||||
ParameterSweep.from_records(records)
|
||||
|
||||
def test_unique_benchmark_names_multiple_duplicates(self):
|
||||
"""Test validation with multiple duplicate names."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 300},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 400},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
|
||||
ParameterSweep.from_records(records)
|
||||
|
||||
def test_no_benchmark_names_allowed(self):
|
||||
"""Test that records without _benchmark_name are allowed."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
|
||||
def test_mixed_benchmark_names_allowed(self):
|
||||
"""Test that mixing records with and without _benchmark_name is allowed."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
|
||||
|
||||
class TestParameterSweepItemKeyNormalization:
|
||||
"""Test key normalization in ParameterSweepItem."""
|
||||
|
||||
def test_underscore_to_hyphen_conversion(self):
|
||||
"""Test that underscores are converted to hyphens in CLI."""
|
||||
item = ParameterSweepItem.from_record({"max_tokens": 100})
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
assert "--max-tokens" in cmd
|
||||
|
||||
def test_nested_key_preserves_suffix(self):
|
||||
"""Test that nested keys preserve the suffix format."""
|
||||
# The suffix after the dot should preserve underscores
|
||||
item = ParameterSweepItem.from_record(
|
||||
{"compilation_config.some_nested_param": "value"}
|
||||
)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
# The prefix (compilation_config) gets converted to hyphens,
|
||||
# but the suffix (some_nested_param) is preserved
|
||||
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
|
||||
|
||||
|
||||
def test_run_comb_excludes_warmup_from_measured_results(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[tuple[int, int, str]] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(
|
||||
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
|
||||
)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 320}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=32,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(-1, 32, "warmup.json"),
|
||||
(0, 320, "run=0.json"),
|
||||
(1, 320, "run=1.json"),
|
||||
]
|
||||
assert measured == [{"run_number": 0}, {"run_number": 1}]
|
||||
|
||||
|
||||
def test_run_comb_continue_on_error_keeps_later_runs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
if run_number == 0:
|
||||
raise RuntimeError("synthetic run failure")
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 100}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=0,
|
||||
dry_run=False,
|
||||
continue_on_error=True,
|
||||
)
|
||||
|
||||
assert calls == [0, 1]
|
||||
assert measured == [{"run_number": 1}]
|
||||
assert (base_path / "run=0.failure.json").exists()
|
||||
assert (base_path / "summary.json").exists()
|
||||
|
||||
|
||||
def test_run_comb_warmup_default_is_backward_compatible(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
"""Older callers can omit warmup_num_prompts."""
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 10}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=1,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [0]
|
||||
assert measured == [{"run_number": 0}]
|
||||
@@ -1,330 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Protocol, cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _ReadableBinary(Protocol):
|
||||
def read(self, size: int = -1) -> bytes: ...
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
|
||||
self.name_or_path = name_or_path
|
||||
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
class CohereAsrTokenizer(_Tokenizer):
|
||||
def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None:
|
||||
super().__init__(name_or_path)
|
||||
|
||||
|
||||
class _CohereNameOnlyTokenizer(_Tokenizer):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("cohere/some-local-checkpoint")
|
||||
|
||||
|
||||
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
|
||||
return cast(TokenizerLike, tokenizer)
|
||||
|
||||
|
||||
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
|
||||
num_samples = int(duration_s * sample_rate)
|
||||
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
|
||||
|
||||
|
||||
class _FakeFormData:
|
||||
def __init__(self) -> None:
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] = []
|
||||
|
||||
def add_field(self, name: str, value: object, **kwargs: str) -> None:
|
||||
self.fields.append((name, value, kwargs))
|
||||
|
||||
|
||||
class _FakeContent:
|
||||
async def iter_any(self):
|
||||
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
|
||||
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self) -> None:
|
||||
self.status = 200
|
||||
self.reason = "OK"
|
||||
self.content = _FakeContent()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.uploaded_bytes: bytes | None = None
|
||||
self.upload_filename: str | None = None
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
|
||||
|
||||
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
|
||||
del url, headers
|
||||
self.fields = list(data.fields)
|
||||
_, file_obj, file_kwargs = self.fields[0]
|
||||
file_obj = cast(_ReadableBinary, file_obj)
|
||||
self.uploaded_bytes = file_obj.read()
|
||||
self.upload_filename = file_kwargs.get("filename")
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
class _FakeProfileSession:
|
||||
def __init__(self) -> None:
|
||||
self.url: str | None = None
|
||||
self.headers: dict[str, str] | None = None
|
||||
|
||||
def post(self, *, url: str, headers: dict[str, str]):
|
||||
self.url = url
|
||||
self.headers = headers
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": str(audio_path),
|
||||
"bytes": None,
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
|
||||
assert (
|
||||
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_filepath", [True, False])
|
||||
def test_asr_dataset_sample_handles_embedded_audio_bytes(
|
||||
tmp_path: Path, has_filepath: bool
|
||||
) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
test_path = None
|
||||
if has_filepath:
|
||||
test_path = audio_path
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": test_path,
|
||||
"bytes": audio_path.read_bytes(),
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert isinstance(samples[0].multi_modal_data, dict)
|
||||
audio, sample_rate = samples[0].multi_modal_data["audio"]
|
||||
assert sample_rate == 16_000
|
||||
assert isinstance(audio, np.ndarray)
|
||||
assert audio.size > 0
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_local_audio_paths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.25)
|
||||
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={"audio_path": str(audio_path)},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == audio_path.name
|
||||
assert session.uploaded_bytes == audio_path.read_bytes()
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
assert output.output_tokens == 1
|
||||
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_decoded_audio_arrays(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={
|
||||
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
|
||||
},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == "audio.wav"
|
||||
assert session.uploaded_bytes is not None
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
|
||||
|
||||
def test_audio_request_rejects_profile_endpoint() -> None:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/start_profile",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="OpenAI Audio API"):
|
||||
asyncio.run(
|
||||
request_func_module.async_request_openai_audio(
|
||||
request_input, _FakeProfileSession()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["start_profile", "stop_profile"])
|
||||
def test_audio_benchmark_profile_request_has_no_inference_payload(
|
||||
endpoint: str,
|
||||
) -> None:
|
||||
session = _FakeProfileSession()
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_profile(
|
||||
f"http://localhost:8000/{endpoint}",
|
||||
session,
|
||||
extra_headers={"X-Test-Header": "profile"},
|
||||
)
|
||||
)
|
||||
|
||||
assert output.success is True
|
||||
assert session.url == f"http://localhost:8000/{endpoint}"
|
||||
assert session.headers == {"X-Test-Header": "profile"}
|
||||
|
||||
|
||||
_COHERE_ASR_PROMPT = (
|
||||
"<|startofcontext|><|startoftranscript|>"
|
||||
"<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
|
||||
"<|notimestamp|><|nodiarize|>"
|
||||
)
|
||||
|
||||
|
||||
def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
|
||||
audio_path = tmp_path / "sample.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {"path": str(audio_path), "bytes": None},
|
||||
"text": "hello world",
|
||||
}
|
||||
]
|
||||
return dataset
|
||||
|
||||
|
||||
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == _COHERE_ASR_PROMPT
|
||||
|
||||
|
||||
def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == _COHERE_ASR_PROMPT
|
||||
|
||||
|
||||
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == ""
|
||||
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import BFCLDataset, get_samples
|
||||
|
||||
|
||||
def _patch_hf_api(side_effect):
|
||||
"""Return a patch context that swaps `hf_api()` to a stub whose
|
||||
`.hf_hub_download` attribute uses `side_effect`."""
|
||||
fake_api = MagicMock()
|
||||
fake_api.hf_hub_download.side_effect = side_effect
|
||||
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
_FAKE_ROWS = {
|
||||
"simple": [
|
||||
{
|
||||
"id": "simple_0",
|
||||
"question": [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2+2?",
|
||||
}
|
||||
]
|
||||
],
|
||||
"function": [
|
||||
{
|
||||
"name": "add",
|
||||
"description": "Add two numbers.",
|
||||
"parameters": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"a": {"type": "integer", "description": "first"},
|
||||
"b": {"type": "float", "description": "second"},
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"live_simple": [
|
||||
{
|
||||
"id": "live_simple_0",
|
||||
"question": [[{"role": "user", "content": "Tell me the weather."}]],
|
||||
"function": [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather.",
|
||||
"parameters": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"city": {"type": "any", "description": "city"},
|
||||
"coords": {"type": "tuple", "description": "coords"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
|
||||
"""Write fake BFCL JSONL files mimicking the HF repo layout."""
|
||||
paths = {}
|
||||
for category, rows in _FAKE_ROWS.items():
|
||||
p = tmp_path / f"BFCL_v3_{category}.json"
|
||||
with p.open("w") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
paths[category] = p
|
||||
return paths
|
||||
|
||||
|
||||
def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="hf",
|
||||
dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
|
||||
hf_name=None,
|
||||
hf_subset=None,
|
||||
hf_split=None,
|
||||
hf_output_len=64,
|
||||
disable_shuffle=True,
|
||||
num_prompts=2,
|
||||
no_oversample=False,
|
||||
no_stream=True,
|
||||
seed=0,
|
||||
request_id_prefix="",
|
||||
trust_remote_code=False,
|
||||
skip_chat_template=False,
|
||||
enable_multimodal_chat=False,
|
||||
backend="openai-chat",
|
||||
bfcl_categories=categories,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_translates_schema_and_attaches_tools(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""BFCLDataset should translate schemas to OpenAI tool format, set
|
||||
`messages` directly on SampleRequest, and attach tools/tool_choice via
|
||||
request_overrides."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
args = _args_for_bfcl(categories=["simple", "live_simple"])
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, hf_tokenizer)
|
||||
|
||||
assert len(samples) == 2
|
||||
for s in samples:
|
||||
assert s.chat_messages is not None
|
||||
assert isinstance(s.chat_messages, list)
|
||||
assert s.chat_messages[0]["role"] == "user"
|
||||
assert s.request_overrides is not None
|
||||
assert "tools" in s.request_overrides
|
||||
assert s.request_overrides["tool_choice"] == "auto"
|
||||
# messages must NOT leak into request_overrides — it has its own
|
||||
# typed field on SampleRequest.
|
||||
assert "messages" not in s.request_overrides
|
||||
tools = s.request_overrides["tools"]
|
||||
assert len(tools) == 1
|
||||
tool = tools[0]
|
||||
assert tool["type"] == "function"
|
||||
# Translated schema: dict -> object, float -> number,
|
||||
# any -> string, tuple -> array.
|
||||
params = tool["function"]["parameters"]
|
||||
assert params["type"] == "object"
|
||||
for prop in params["properties"].values():
|
||||
assert prop["type"] in {"integer", "number", "string", "array"}
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_requires_openai_chat_backend(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.backend = "openai"
|
||||
|
||||
with pytest.raises(ValueError, match="openai-chat"):
|
||||
get_samples(args, hf_tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_missing_category_raises_clear_error(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""A typo'd category should produce an actionable ValueError, not an
|
||||
opaque huggingface_hub exception."""
|
||||
from huggingface_hub.errors import EntryNotFoundError
|
||||
|
||||
args = _args_for_bfcl(categories=["simpl"]) # typo
|
||||
|
||||
def raise_missing(_repo, filename, **_kwargs):
|
||||
raise EntryNotFoundError(f"404 Not Found: {filename}")
|
||||
|
||||
with (
|
||||
_patch_hf_api(raise_missing),
|
||||
pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
|
||||
):
|
||||
get_samples(args, hf_tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_backend_uses_messages_field_when_set() -> None:
|
||||
"""When RequestFuncInput.chat_messages is set, the chat backend must use
|
||||
it verbatim and skip default content construction from `prompt`."""
|
||||
import asyncio
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
async_request_openai_chat_completions,
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
class _FakeResp:
|
||||
status = 500
|
||||
reason = "stop-after-capture"
|
||||
content = None
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
class _FakeSession:
|
||||
def post(self, url, json, headers): # noqa: A002
|
||||
captured["url"] = url
|
||||
captured["payload"] = json
|
||||
return _FakeResp()
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "you are helpful"},
|
||||
{"role": "user", "content": "call add(3, 4)"},
|
||||
]
|
||||
req = RequestFuncInput(
|
||||
prompt="IGNORED",
|
||||
api_url="http://localhost:0/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=16,
|
||||
model="test-model",
|
||||
chat_messages=messages,
|
||||
extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
async_request_openai_chat_completions(
|
||||
request_func_input=req, session=_FakeSession()
|
||||
)
|
||||
)
|
||||
|
||||
payload = captured["payload"]
|
||||
assert payload["messages"] is messages, (
|
||||
"chat backend must forward RequestFuncInput.chat_messages verbatim "
|
||||
"instead of constructing a default user message from `prompt`"
|
||||
)
|
||||
# extra_body still merges in as before (shallow, per-request wins).
|
||||
assert payload["tools"][0]["function"]["name"] == "add"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
||||
"""prompt_len must reflect tokens from both messages *and* tool schemas,
|
||||
so percentile buckets and input-distribution summaries aren't biased
|
||||
low for tool-heavy traffic."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
class _FakeTokenizer:
|
||||
def apply_chat_template(
|
||||
self, messages, tools=None, tokenize=False, add_generation_prompt=True
|
||||
):
|
||||
captured["tools"] = tools
|
||||
base = " ".join(m.get("content", "") for m in messages)
|
||||
tool_text = json.dumps(tools) if tools else ""
|
||||
return base + " " + tool_text
|
||||
|
||||
def __call__(self, text):
|
||||
# 1 "token" per whitespace-separated word.
|
||||
return type("Enc", (), {"input_ids": text.split()})()
|
||||
|
||||
fake = _FakeTokenizer()
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, fake)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert captured["tools"] is not None, (
|
||||
"apply_chat_template must be called with tools= so the schema "
|
||||
"contributes to the prompt-length estimate"
|
||||
)
|
||||
assert len(captured["tools"]) == 1
|
||||
assert captured["tools"][0]["function"]["name"] == "add"
|
||||
|
||||
# Sanity: prompt_len exceeds a messages-only estimate. The fake row's
|
||||
# user message is "What is 2+2?" (3 whitespace-separated tokens).
|
||||
assert samples[0].prompt_len > 3
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Older tokenizers don't accept tools=; fallback must still produce a
|
||||
non-zero prompt_len without crashing."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
class _LegacyTokenizer:
|
||||
def apply_chat_template(self, messages, **kwargs):
|
||||
if "tools" in kwargs:
|
||||
raise TypeError("unexpected keyword argument 'tools'")
|
||||
return " ".join(m.get("content", "") for m in messages)
|
||||
|
||||
def __call__(self, text):
|
||||
return type("Enc", (), {"input_ids": text.split()})()
|
||||
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, _LegacyTokenizer())
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt_len > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_schema_translation_is_recursive() -> None:
|
||||
"""_translate_schema must recurse into nested properties."""
|
||||
input_schema = {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"nested": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"value": {"type": "float"},
|
||||
"tags": {"type": "tuple", "items": {"type": "any"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
out = BFCLDataset._translate_schema(input_schema)
|
||||
assert out["type"] == "object"
|
||||
assert out["properties"]["nested"]["type"] == "object"
|
||||
assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
|
||||
assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
|
||||
nested_props = out["properties"]["nested"]["properties"]
|
||||
assert nested_props["tags"]["items"]["type"] == "string"
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
|
||||
class _RecordingTokenizer:
|
||||
"""Minimal tokenizer stub that records the kwargs forwarded to
|
||||
apply_chat_template, so we can assert chat_template_kwargs propagation
|
||||
without loading a real model/template."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.captured_kwargs: dict | None = None
|
||||
self.chat_template = "dummy-template"
|
||||
|
||||
def apply_chat_template(
|
||||
self,
|
||||
conversation,
|
||||
add_generation_prompt: bool = True,
|
||||
tokenize: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
self.captured_kwargs = kwargs
|
||||
return conversation[0]["content"]
|
||||
|
||||
def __call__(self, text: str):
|
||||
return argparse.Namespace(input_ids=list(range(len(text.split()))))
|
||||
|
||||
|
||||
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=True,
|
||||
num_prompts=1,
|
||||
custom_output_len=32,
|
||||
skip_chat_template=False,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
no_oversample=False,
|
||||
seed=0,
|
||||
request_id_prefix="",
|
||||
)
|
||||
|
||||
|
||||
def _write_one(path: Path) -> None:
|
||||
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
|
||||
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), {"thinking": True}), tok)
|
||||
|
||||
assert tok.captured_kwargs == {"thinking": True}
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
|
||||
"""When not provided, no extra kwargs are passed (existing behavior)."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), None), tok)
|
||||
|
||||
assert tok.captured_kwargs == {}
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, n_rows: int) -> None:
|
||||
with path.open("w") as f:
|
||||
for i in range(n_rows):
|
||||
f.write(json.dumps({"prompt": f"row {i}: unique prompt content."}) + "\n")
|
||||
|
||||
|
||||
def _args_for_custom(dataset_path: str, seed: int) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=False,
|
||||
num_prompts=30,
|
||||
custom_output_len=32,
|
||||
skip_chat_template=True,
|
||||
no_oversample=False,
|
||||
seed=seed,
|
||||
request_id_prefix="",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_dataset_seed_propagates(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""--seed must control the CustomDataset shuffle used by get_samples.
|
||||
|
||||
Without the fix, CustomDataset was instantiated without random_seed,
|
||||
so its load-time shuffle always used DEFAULT_SEED=0 regardless of
|
||||
args.seed, causing every run with --dataset-name custom to pick the
|
||||
same subset of rows from a larger file.
|
||||
"""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_jsonl(jsonl, n_rows=60)
|
||||
|
||||
samples_a = get_samples(_args_for_custom(str(jsonl), seed=0), hf_tokenizer)
|
||||
samples_b = get_samples(_args_for_custom(str(jsonl), seed=42), hf_tokenizer)
|
||||
|
||||
prompts_a = {s.prompt for s in samples_a}
|
||||
prompts_b = {s.prompt for s in samples_b}
|
||||
|
||||
assert len(prompts_a) == 30
|
||||
assert len(prompts_b) == 30
|
||||
assert prompts_a != prompts_b
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_dataset_same_seed_is_deterministic(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""Same --seed must yield the same CustomDataset subset."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_jsonl(jsonl, n_rows=60)
|
||||
|
||||
samples_a = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
|
||||
samples_b = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
|
||||
|
||||
prompts_a = [s.prompt for s in samples_a]
|
||||
prompts_b = [s.prompt for s in samples_b]
|
||||
|
||||
assert prompts_a == prompts_b
|
||||
@@ -1,400 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
_get_chat_content,
|
||||
_get_chat_messages,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, _Tokenizer())
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
|
||||
|
||||
def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
|
||||
Image.new("RGB", (1, 1), color=color).save(path)
|
||||
|
||||
|
||||
def _decode_data_url(data_url: str) -> tuple[str, bytes]:
|
||||
prefix, image_base64 = data_url.split(",", 1)
|
||||
return prefix, base64.b64decode(image_base64)
|
||||
|
||||
|
||||
def _assert_png_data_url(data_url: str) -> None:
|
||||
prefix, image_bytes = _decode_data_url(data_url)
|
||||
assert prefix == "data:image/png;base64"
|
||||
with Image.open(BytesIO(image_bytes)) as image:
|
||||
image.verify()
|
||||
|
||||
|
||||
def _args_for_custom_image(dataset_path: Path) -> Namespace:
|
||||
return Namespace(
|
||||
dataset_name="custom_image",
|
||||
dataset_path=str(dataset_path),
|
||||
disable_shuffle=True,
|
||||
seed=0,
|
||||
num_prompts=2,
|
||||
custom_output_len=32,
|
||||
enable_multimodal_chat=False,
|
||||
custom_ensure_client_side_data=False,
|
||||
request_id_prefix="req-",
|
||||
no_oversample=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
image_c = tmp_path / "chart_c.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the first two charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
},
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Now compare "},
|
||||
{"type": "image", "image": str(image_c)},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
|
||||
|
||||
assert len(samples) == 2
|
||||
assert samples[0].request_id == "req-0"
|
||||
assert isinstance(samples[0].multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
assert samples[1].request_id == "req-1"
|
||||
assert samples[1].multi_modal_data is None
|
||||
assert isinstance(samples[1].prompt, list)
|
||||
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
|
||||
assert isinstance(samples[1].prompt[1], dict)
|
||||
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.prompt == "Compare the charts."
|
||||
assert sample.prompt_len == 3
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare "},
|
||||
{"type": "image", "image": str(image_a)},
|
||||
{"type": "text", "text": " with "},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": str(image_b),
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt_len == 2
|
||||
assert isinstance(sample.prompt, list)
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
assert [part["type"] for part in parts] == [
|
||||
"text",
|
||||
"image_url",
|
||||
"text",
|
||||
"image_url",
|
||||
]
|
||||
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert parts[3]["image_url"] == {
|
||||
"url": f"file://{image_b}",
|
||||
"detail": "low",
|
||||
}
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_content(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image = tmp_path / "chart.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image", "image": str(image)},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
enable_multimodal_chat=True,
|
||||
)
|
||||
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_messages(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_encodes_image_media_when_requested(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart b.png"
|
||||
_write_png(image_a, color=(255, 0, 0))
|
||||
_write_png(image_b, color=(0, 255, 0))
|
||||
data_url = "data:image/png;base64,Zm9v"
|
||||
remote_url = "https://example.com/chart.png"
|
||||
original_fetch_image = datasets_module.fetch_image
|
||||
|
||||
def fake_fetch_image(image_url: str) -> Image.Image:
|
||||
if image_url == remote_url:
|
||||
return Image.new("RGB", (1, 1), color=(0, 0, 255))
|
||||
return original_fetch_image(image_url)
|
||||
|
||||
monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
|
||||
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the charts.",
|
||||
"image_files": [
|
||||
str(image_a),
|
||||
image_b.as_uri(),
|
||||
remote_url,
|
||||
data_url,
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert isinstance(samples[0].multi_modal_data, list)
|
||||
image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
|
||||
|
||||
_assert_png_data_url(image_urls[0])
|
||||
_assert_png_data_url(image_urls[1])
|
||||
_assert_png_data_url(image_urls[2])
|
||||
assert image_urls[3] == data_url
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
_write_png(image_a, color=(255, 0, 0))
|
||||
_write_png(image_b, color=(0, 255, 0))
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare "},
|
||||
{"type": "image", "image": str(image_a)},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": image_b.as_uri(),
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
sample = samples[0]
|
||||
assert isinstance(sample.prompt, list)
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
_assert_png_data_url(parts[1]["image_url"]["url"])
|
||||
_assert_png_data_url(parts[2]["image_url"]["url"])
|
||||
assert parts[2]["image_url"]["detail"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_rejects_invalid_image_media(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
invalid_image = tmp_path / "not_an_image.png"
|
||||
invalid_image.write_text("not an image")
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="Invalid image URL"):
|
||||
dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_rejects_invalid_content_part(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
|
||||
dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -1,510 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Timing-accounting invariants for the streaming benchmark request functions.
|
||||
|
||||
serve.py derives TPOT as (latency - ttft) / (output_len - 1), which only means
|
||||
"average inter-token latency" if latency - ttft == sum(itl). These tests pin
|
||||
that identity, and pin that the trailing choice-less usage chunk does not
|
||||
extend the measured end of the request.
|
||||
|
||||
Driven against a fake session and a scripted clock: no GPU, model, or socket.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
async_request_openai_audio,
|
||||
async_request_openai_chat_completions,
|
||||
async_request_openai_completions,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
AUDIO_SAMPLE_RATE = 16_000
|
||||
|
||||
|
||||
class _ScriptedClock:
|
||||
"""A deterministic stand-in for time.perf_counter.
|
||||
|
||||
Each read advances the clock by exactly one tick, so what a code path
|
||||
observes depends only on how many times it reads the clock. A stray or
|
||||
misattributed read then shows up as a whole tick of drift rather than as
|
||||
sub-microsecond jitter, which keeps these assertions free of flakiness.
|
||||
"""
|
||||
|
||||
def __init__(self, tick: float = 1.0, start: float = 1000.0) -> None:
|
||||
self.tick = tick
|
||||
self._now = start
|
||||
|
||||
def __call__(self) -> float:
|
||||
value = self._now
|
||||
self._now += self.tick
|
||||
return value
|
||||
|
||||
|
||||
class _FakeTime:
|
||||
"""Namespace substituted for the module-level time import."""
|
||||
|
||||
def __init__(self, perf_counter) -> None:
|
||||
self.perf_counter = perf_counter
|
||||
|
||||
|
||||
def _sse(payload: dict[str, Any]) -> bytes:
|
||||
return f"data: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
def _token_chunk(index: int, *, chat: bool) -> bytes:
|
||||
if chat:
|
||||
choice = {"index": 0, "delta": {"content": f"tok{index} "}}
|
||||
else:
|
||||
choice = {"index": 0, "text": f"tok{index} "}
|
||||
return _sse({"id": "cmpl-test", "object": "x", "choices": [choice]})
|
||||
|
||||
|
||||
def _usage_chunk(n_tokens: int) -> bytes:
|
||||
return _sse(
|
||||
{
|
||||
"id": "cmpl-test",
|
||||
"object": "x",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 7,
|
||||
"completion_tokens": n_tokens,
|
||||
"total_tokens": 7 + n_tokens,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_stream(
|
||||
n_tokens: int,
|
||||
*,
|
||||
chat: bool,
|
||||
ttft_delay: float = 0.0,
|
||||
itl_delay: float = 0.0,
|
||||
usage_delay: float = 0.0,
|
||||
) -> list[tuple[float, bytes]]:
|
||||
"""Build ``(delay_before_chunk, payload)`` pairs for one streamed response.
|
||||
|
||||
``vllm bench serve`` always requests ``stream_options.include_usage``, so a
|
||||
choice-less usage chunk always trails the final token.
|
||||
"""
|
||||
chunks = [
|
||||
(ttft_delay if i == 0 else itl_delay, _token_chunk(i, chat=chat))
|
||||
for i in range(n_tokens)
|
||||
]
|
||||
chunks.append((usage_delay, _usage_chunk(n_tokens)))
|
||||
chunks.append((0.0, b"data: [DONE]\n\n"))
|
||||
return chunks
|
||||
|
||||
|
||||
class _FakeContent:
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
async def iter_any(self):
|
||||
for delay, payload in self._chunks:
|
||||
if delay:
|
||||
await asyncio.sleep(delay)
|
||||
yield payload
|
||||
|
||||
# The legacy script iterates ``response.content`` directly.
|
||||
def __aiter__(self):
|
||||
return self.iter_any()
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self.status = 200
|
||||
self.reason = "OK"
|
||||
self.content = _FakeContent(chunks)
|
||||
|
||||
async def __aenter__(self) -> "_FakeResponse":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Returns a scripted SSE stream for any request, without touching a socket."""
|
||||
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
def post(self, **kwargs: Any) -> _FakeResponse:
|
||||
del kwargs
|
||||
return _FakeResponse(self._chunks)
|
||||
|
||||
async def __aenter__(self) -> "_FakeSession":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
# (id, request function, api url, is_chat_payload)
|
||||
STREAMING_ENDPOINTS = [
|
||||
(
|
||||
"completions",
|
||||
async_request_openai_completions,
|
||||
"http://test/v1/completions",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"chat",
|
||||
async_request_openai_chat_completions,
|
||||
"http://test/v1/chat/completions",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"audio",
|
||||
async_request_openai_audio,
|
||||
"http://test/v1/audio/transcriptions",
|
||||
True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _make_input(api_url: str, n_tokens: int, endpoint_id: str) -> RequestFuncInput:
|
||||
multi_modal_content = None
|
||||
if endpoint_id == "audio":
|
||||
# A short silent clip; only its duration is read by the request func.
|
||||
multi_modal_content = {
|
||||
"audio": (
|
||||
np.zeros(AUDIO_SAMPLE_RATE // 10, dtype=np.float32),
|
||||
AUDIO_SAMPLE_RATE,
|
||||
)
|
||||
}
|
||||
return RequestFuncInput(
|
||||
prompt="hello",
|
||||
api_url=api_url,
|
||||
prompt_len=7,
|
||||
output_len=n_tokens,
|
||||
model="test-model",
|
||||
multi_modal_content=multi_modal_content,
|
||||
)
|
||||
|
||||
|
||||
def _run(
|
||||
request_func,
|
||||
api_url: str,
|
||||
endpoint_id: str,
|
||||
chunks,
|
||||
n_tokens: int,
|
||||
*,
|
||||
expect_success: bool = True,
|
||||
):
|
||||
output = asyncio.run(
|
||||
request_func(
|
||||
_make_input(api_url, n_tokens, endpoint_id),
|
||||
_FakeSession(chunks),
|
||||
)
|
||||
)
|
||||
if expect_success:
|
||||
assert output.success, output.error
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_decode_span_identity_is_exact(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""Latency - ttft must equal sum(itl) exactly.
|
||||
|
||||
On a scripted clock this can only drift if a code path reads the clock an
|
||||
extra time or attributes a read to the wrong event.
|
||||
"""
|
||||
tick = 1.0
|
||||
monkeypatch.setattr(
|
||||
request_func_module, "time", _FakeTime(_ScriptedClock(tick=tick))
|
||||
)
|
||||
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
assert len(output.itl) == n_tokens - 1
|
||||
assert output.output_tokens == n_tokens
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert residual == pytest.approx(0.0, abs=tick / 1000), (
|
||||
f"{endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}; "
|
||||
f"TPOT is derived from latency - ttft, so it must match the measured "
|
||||
f"inter-token latencies"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_trailing_usage_chunk_does_not_extend_latency(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""E2E latency must end at the last token, not at the usage trailer.
|
||||
|
||||
The usage chunk carries no token, so counting it stretches E2EL (and
|
||||
therefore TPOT) by the delivery time of one extra SSE frame. The [DONE]
|
||||
sentinel is already excluded on the same grounds.
|
||||
"""
|
||||
tick = 1.0
|
||||
clock = _ScriptedClock(tick=tick)
|
||||
monkeypatch.setattr(request_func_module, "time", _FakeTime(clock))
|
||||
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
# Clock reads: 1 for the start time, then 1 per non-[DONE] chunk. The last
|
||||
# token is therefore read at start + n_tokens ticks; the usage chunk one
|
||||
# tick later.
|
||||
expected_latency = n_tokens * tick
|
||||
assert output.latency == pytest.approx(expected_latency, abs=tick / 1000), (
|
||||
f"{endpoint_id}: latency {output.latency} should stop at the final "
|
||||
f"token ({expected_latency}), not run on to the usage chunk "
|
||||
f"({expected_latency + tick})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_decode_span_identity_holds_with_real_clock(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""The identity also holds against the real clock, with real gaps.
|
||||
|
||||
The residual differences out the same recorded timestamps, so wall clock
|
||||
jitter cancels and only float rounding remains; a microsecond bound is
|
||||
ample rather than tight.
|
||||
"""
|
||||
n_tokens = 4
|
||||
chunks = _build_stream(
|
||||
n_tokens, chat=chat, ttft_delay=0.02, itl_delay=0.01, usage_delay=0.02
|
||||
)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert abs(residual) < 1e-6, (
|
||||
f"{endpoint_id}: (latency - ttft) - sum(itl) drifted by {residual:.9f}s"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_usage_only_stream_is_not_reported_as_success(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""A stream with no token chunk must fail rather than report 0 ms.
|
||||
|
||||
Only token chunks move the end of the request, so a degenerate stream
|
||||
carrying nothing but a usage trailer leaves latency at zero. Reporting that
|
||||
as a success would feed a zero-duration request into the benchmark
|
||||
aggregates.
|
||||
"""
|
||||
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, 0, expect_success=False)
|
||||
|
||||
assert not output.success
|
||||
assert output.itl == []
|
||||
assert "TTFT" in output.error
|
||||
|
||||
|
||||
# Significant whitespace at every edge, plus a multibyte character so that a
|
||||
# split can land inside a UTF-8 sequence.
|
||||
_BOUNDARY_TEXT = " leading 中 trailing "
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_streamed_output_is_invariant_to_transport_chunk_boundaries(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""iter_any() yields arbitrary transport fragments, so parsing must
|
||||
give the same result no matter where the byte stream is split."""
|
||||
if chat:
|
||||
choice: dict[str, Any] = {"index": 0, "delta": {"content": _BOUNDARY_TEXT}}
|
||||
else:
|
||||
choice = {"index": 0, "text": _BOUNDARY_TEXT}
|
||||
# Not _sse(): its ensure_ascii default would escape the multibyte char.
|
||||
event = json.dumps(
|
||||
{"id": "cmpl-test", "object": "x", "choices": [choice]}, ensure_ascii=False
|
||||
)
|
||||
stream = f"data: {event}\n\ndata: [DONE]\n\n".encode()
|
||||
|
||||
async def run_all_splits() -> list[tuple[int, bool, str, str]]:
|
||||
failures = []
|
||||
for split_at in range(1, len(stream)):
|
||||
chunks = [(0.0, stream[:split_at]), (0.0, stream[split_at:])]
|
||||
output = await request_func(
|
||||
_make_input(api_url, 1, endpoint_id),
|
||||
_FakeSession(chunks),
|
||||
)
|
||||
if not output.success or output.generated_text != _BOUNDARY_TEXT:
|
||||
error = output.error.splitlines()[-1] if output.error else ""
|
||||
failures.append(
|
||||
(split_at, output.success, output.generated_text, error)
|
||||
)
|
||||
return failures
|
||||
|
||||
assert asyncio.run(run_all_splits()) == []
|
||||
|
||||
|
||||
# ``benchmarks/backend_request_func.py`` is a standalone script rather than a
|
||||
# package module, so it is loaded by path. It also builds its own ClientSession,
|
||||
# so the scripted stream is injected by swapping the module's aiohttp reference.
|
||||
_LEGACY_PATH = Path(__file__).parents[2] / "benchmarks" / "backend_request_func.py"
|
||||
|
||||
|
||||
class _FakeAiohttp:
|
||||
"""Stands in for the legacy module's aiohttp import."""
|
||||
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(aiohttp, name)
|
||||
|
||||
def ClientSession(self, **kwargs: Any) -> _FakeSession: # noqa: N802
|
||||
del kwargs
|
||||
return _FakeSession(self._chunks)
|
||||
|
||||
|
||||
def _load_legacy():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"legacy_backend_request_func", _LEGACY_PATH
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
LEGACY_ENDPOINTS = [
|
||||
(
|
||||
"completions",
|
||||
"async_request_openai_completions",
|
||||
"http://test/v1/completions",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"chat",
|
||||
"async_request_openai_chat_completions",
|
||||
"http://test/v1/chat/completions",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"audio",
|
||||
"async_request_openai_audio",
|
||||
"http://test/v1/audio/transcriptions",
|
||||
True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _run_legacy(
|
||||
monkeypatch, func_name, api_url, endpoint_id, chunks, n_tokens, clock=None
|
||||
):
|
||||
module = _load_legacy()
|
||||
monkeypatch.setattr(module, "aiohttp", _FakeAiohttp(chunks))
|
||||
if clock is not None:
|
||||
monkeypatch.setattr(module, "time", _FakeTime(clock))
|
||||
request_func = getattr(module, func_name)
|
||||
return asyncio.run(request_func(_make_input(api_url, n_tokens, endpoint_id)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,func_name,api_url,chat",
|
||||
LEGACY_ENDPOINTS,
|
||||
ids=[e[0] for e in LEGACY_ENDPOINTS],
|
||||
)
|
||||
def test_legacy_decode_span_identity_is_exact(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
func_name: str,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""benchmarks/backend_request_func.py must honour the same identity."""
|
||||
tick = 1.0
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run_legacy(
|
||||
monkeypatch,
|
||||
func_name,
|
||||
api_url,
|
||||
endpoint_id,
|
||||
chunks,
|
||||
n_tokens,
|
||||
clock=_ScriptedClock(tick=tick),
|
||||
)
|
||||
|
||||
assert output.success, output.error
|
||||
assert len(output.itl) == n_tokens - 1
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert residual == pytest.approx(0.0, abs=tick / 1000), (
|
||||
f"legacy {endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}"
|
||||
)
|
||||
assert output.latency == pytest.approx(n_tokens * tick, abs=tick / 1000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,func_name,api_url,chat",
|
||||
LEGACY_ENDPOINTS,
|
||||
ids=[e[0] for e in LEGACY_ENDPOINTS],
|
||||
)
|
||||
def test_legacy_usage_only_stream_is_not_reported_as_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
func_name: str,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""A usage-only stream must fail rather than report a zero-duration success."""
|
||||
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
|
||||
output = _run_legacy(monkeypatch, func_name, api_url, endpoint_id, chunks, 0)
|
||||
|
||||
assert not output.success
|
||||
assert output.itl == []
|
||||
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.main import run_case
|
||||
from vllm.benchmarks.forward.profile import summarize
|
||||
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
|
||||
def _group(block_size: int, replay: int = 0, dcp_sharded: bool = False):
|
||||
return SimpleNamespace(
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
prefix_replay_tokens=replay,
|
||||
dcp_sharded=dcp_sharded,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_build_batch_layout():
|
||||
"""The runner sees each request kind as the scheduler would schedule it."""
|
||||
case = parse_case("c=2*1@101,4@64:3,32,8@48c,16@316")
|
||||
groups = [_group(16), _group(32, replay=128)]
|
||||
batch = build_batch(case, groups, 1000, 1000, np.random.default_rng(0))
|
||||
|
||||
# (scheduled, computed, is_decode) per request: decode, verify, prefill,
|
||||
# chunked prefill, then a prefix hit replaying the last 128 hit tokens.
|
||||
assert list(batch.expected.values()) == [
|
||||
(1, 100, True),
|
||||
(1, 100, True),
|
||||
(4, 60, True),
|
||||
(32, 0, False),
|
||||
(8, 40, False),
|
||||
(16 + 128, 300 - 128, False),
|
||||
]
|
||||
output = batch.scheduler_output
|
||||
assert output.total_num_scheduled_tokens == 1 + 1 + 4 + 32 + 8 + 144
|
||||
assert len(output.scheduled_spec_decode_tokens["bench-2"]) == 3
|
||||
assert output.scheduled_new_reqs[-1].replay_start == 300 - 128
|
||||
for req_id, (scheduled, _, _) in batch.expected.items():
|
||||
assert len(batch.query_token_ids[req_id]) == scheduled
|
||||
# Groups share one block pool: block IDs must not overlap.
|
||||
ranges = sorted(r for group in batch.group_block_ranges for r in group)
|
||||
assert ranges[0][0] == 1
|
||||
assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:]))
|
||||
|
||||
with pytest.raises(ValueError, match="KV blocks"):
|
||||
build_batch(case, groups, 10, 1000, np.random.default_rng(0))
|
||||
|
||||
# DCP spreads each block's tokens over the DCP ranks, like the scheduler.
|
||||
groups = [_group(16, dcp_sharded=True), _group(16)]
|
||||
batch = build_batch(
|
||||
parse_case("c=1@640"), groups, 1000, 1000, np.random.default_rng(0), 4
|
||||
)
|
||||
assert batch.group_block_ranges == [[(1, 11)], [(11, 51)]]
|
||||
|
||||
|
||||
def test_jitter_kv_lens():
|
||||
"""Prefixes vary, capped at the max length; queries and fresh prefills don't.
|
||||
A case's draw depends only on the seed and its name, so adding or reordering
|
||||
other cases doesn't change it."""
|
||||
case = parse_case("c=1000*1@1001,4@64:3,32")
|
||||
jittered = jitter_kv_lens(case, 0.5, 1400, 0)
|
||||
assert jitter_kv_lens(case, 0.5, 1400, 0) == jittered
|
||||
other = jitter_kv_lens(replace(case, name="d"), 0.5, 1400, 0)
|
||||
assert other.requests != jittered.requests
|
||||
prefixes = np.array([r.prefix_len for r in jittered.requests[:1000]])
|
||||
assert prefixes.min() >= 500 and max(r.kv_len for r in jittered.requests) == 1400
|
||||
assert prefixes.std() > 200
|
||||
assert [r.query_len for r in jittered.requests] == [
|
||||
r.query_len for r in case.requests
|
||||
]
|
||||
assert jittered.requests[-1] == case.requests[-1]
|
||||
assert jitter_kv_lens(case, 0, 2048, 0) == case
|
||||
with pytest.raises(ValueError, match="KV length"):
|
||||
jitter_kv_lens(case, 0, 1000, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 2])
|
||||
def test_bench_forward_rejects_oversized_verify(capacity):
|
||||
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
|
||||
runner = SimpleNamespace(num_speculative_steps=capacity)
|
||||
case = parse_case("verify=2*4@64:3")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
|
||||
):
|
||||
run_case(runner, case, SimpleNamespace())
|
||||
|
||||
|
||||
def test_profile_summary():
|
||||
"""Each replay's leading restore/rendezvous activities are left out, and
|
||||
kernels are grouped by name without template or function arguments."""
|
||||
|
||||
def act(name, ts, dur, cat="kernel"):
|
||||
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
|
||||
|
||||
replay = [
|
||||
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
|
||||
("void ns::gemm<128>(Params)", "kernel", 10),
|
||||
("void ns::gemm<64>(Params)", "kernel", 20),
|
||||
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
|
||||
]
|
||||
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
|
||||
for r in range(2):
|
||||
for i, (name, cat, dur) in enumerate(replay):
|
||||
events.append(act(name, 1000 * r + 100 * i, dur, cat))
|
||||
summary = summarize(events, replays=2, skip=1)
|
||||
assert summary["pre_forward_us"] == 5
|
||||
assert summary["busy_us"] == 34
|
||||
assert summary["span_us"] == 204
|
||||
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
|
||||
("ns::gemm", 30, 2),
|
||||
("ns::norm", 4, 1),
|
||||
]
|
||||
with pytest.raises(ValueError, match="replays"):
|
||||
summarize(events[:-1], replays=2, skip=1)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Hangs on ROCm; the bench is CUDA-only."
|
||||
)
|
||||
def test_bench_forward(tmp_path):
|
||||
output = tmp_path / "forward.json"
|
||||
command = [
|
||||
"vllm",
|
||||
"bench",
|
||||
"forward",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--case",
|
||||
"decode=8*1@512",
|
||||
"--case",
|
||||
"prefill=256",
|
||||
"--case",
|
||||
"mix=128@256,4*1@512",
|
||||
"--iters",
|
||||
"3",
|
||||
"--samples",
|
||||
"2",
|
||||
"--output-json",
|
||||
str(output),
|
||||
"--profile-dir",
|
||||
str(tmp_path / "profile"),
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
print(result.stderr)
|
||||
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
|
||||
|
||||
results = {row["case"]: row for row in json.loads(output.read_text())}
|
||||
assert list(results) == ["decode", "prefill", "mix"]
|
||||
for row in results.values():
|
||||
rank0 = row["ranks"][0]
|
||||
assert len(rank0["batch_order"]) == len(row["requests"])
|
||||
assert rank0["finite"] and rank0["graph_deterministic"]
|
||||
assert row["median_us"] > 0
|
||||
profile = rank0["profile"]
|
||||
assert profile["top_kernels"] and profile["span_us"] > 0
|
||||
summary = json.loads(Path(profile["summary"]).read_text())
|
||||
assert Path(summary["trace"]).exists()
|
||||
@@ -1,171 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep.plot import (
|
||||
PlotEqualTo,
|
||||
PlotFilterBase,
|
||||
PlotFilters,
|
||||
PlotGreaterThan,
|
||||
PlotGreaterThanOrEqualTo,
|
||||
PlotLessThan,
|
||||
PlotLessThanOrEqualTo,
|
||||
PlotNotEqualTo,
|
||||
)
|
||||
|
||||
|
||||
class TestPlotFilters:
|
||||
"""Test PlotFilter functionality including 'inf' edge case."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Create sample DataFrames for testing."""
|
||||
# DataFrame with numeric values
|
||||
self.df_numeric = pd.DataFrame(
|
||||
{
|
||||
"request_rate": [1.0, 5.0, 10.0, 50.0, 100.0],
|
||||
"value": [10, 20, 30, 40, 50],
|
||||
}
|
||||
)
|
||||
|
||||
# DataFrame with float('inf') - note: string "inf" values are coerced
|
||||
# to float when loading data, so we only test with float('inf')
|
||||
self.df_inf_float = pd.DataFrame(
|
||||
{
|
||||
"request_rate": [1.0, 5.0, 10.0, float("inf"), float("inf")],
|
||||
"value": [10, 20, 30, 40, 50],
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("5.0", 1),
|
||||
("10.0", 1),
|
||||
("1.0", 1),
|
||||
],
|
||||
)
|
||||
def test_equal_to_numeric(self, target, expected_count):
|
||||
"""Test PlotEqualTo with numeric values."""
|
||||
filter_obj = PlotEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
def test_equal_to_inf_float(self):
|
||||
"""Test PlotEqualTo with float('inf')."""
|
||||
filter_obj = PlotEqualTo("request_rate", "inf")
|
||||
result = filter_obj.apply(self.df_inf_float)
|
||||
# Should match both float('inf') entries because float('inf') == float('inf')
|
||||
assert len(result) == 2
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("5.0", 4), # All except 5.0
|
||||
("1.0", 4), # All except 1.0
|
||||
],
|
||||
)
|
||||
def test_not_equal_to_numeric(self, target, expected_count):
|
||||
"""Test PlotNotEqualTo with numeric values."""
|
||||
filter_obj = PlotNotEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
def test_not_equal_to_inf_float(self):
|
||||
"""Test PlotNotEqualTo with float('inf')."""
|
||||
filter_obj = PlotNotEqualTo("request_rate", "inf")
|
||||
result = filter_obj.apply(self.df_inf_float)
|
||||
# Should exclude float('inf') entries
|
||||
assert len(result) == 3
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 2), # 1.0, 5.0
|
||||
("50.0", 3), # 1.0, 5.0, 10.0
|
||||
("5.0", 1), # 1.0
|
||||
],
|
||||
)
|
||||
def test_less_than(self, target, expected_count):
|
||||
"""Test PlotLessThan with numeric values."""
|
||||
filter_obj = PlotLessThan("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 3), # 1.0, 5.0, 10.0
|
||||
("5.0", 2), # 1.0, 5.0
|
||||
],
|
||||
)
|
||||
def test_less_than_or_equal_to(self, target, expected_count):
|
||||
"""Test PlotLessThanOrEqualTo with numeric values."""
|
||||
filter_obj = PlotLessThanOrEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 2), # 50.0, 100.0
|
||||
("5.0", 3), # 10.0, 50.0, 100.0
|
||||
],
|
||||
)
|
||||
def test_greater_than(self, target, expected_count):
|
||||
"""Test PlotGreaterThan with numeric values."""
|
||||
filter_obj = PlotGreaterThan("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 3), # 10.0, 50.0, 100.0
|
||||
("5.0", 4), # 5.0, 10.0, 50.0, 100.0
|
||||
],
|
||||
)
|
||||
def test_greater_than_or_equal_to(self, target, expected_count):
|
||||
"""Test PlotGreaterThanOrEqualTo with numeric values."""
|
||||
filter_obj = PlotGreaterThanOrEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filter_str,expected_var,expected_target,expected_type",
|
||||
[
|
||||
("request_rate==5.0", "request_rate", "5.0", PlotEqualTo),
|
||||
("request_rate!=10.0", "request_rate", "10.0", PlotNotEqualTo),
|
||||
("request_rate<50.0", "request_rate", "50.0", PlotLessThan),
|
||||
("request_rate<=50.0", "request_rate", "50.0", PlotLessThanOrEqualTo),
|
||||
("request_rate>10.0", "request_rate", "10.0", PlotGreaterThan),
|
||||
("request_rate>=10.0", "request_rate", "10.0", PlotGreaterThanOrEqualTo),
|
||||
("request_rate==inf", "request_rate", "inf", PlotEqualTo),
|
||||
("request_rate!='inf'", "request_rate", "inf", PlotNotEqualTo),
|
||||
],
|
||||
)
|
||||
def test_parse_str(self, filter_str, expected_var, expected_target, expected_type):
|
||||
"""Test parsing filter strings."""
|
||||
filter_obj = PlotFilterBase.parse_str(filter_str)
|
||||
assert isinstance(filter_obj, expected_type)
|
||||
assert filter_obj.var == expected_var
|
||||
assert filter_obj.target == expected_target
|
||||
|
||||
def test_parse_str_inf_edge_case(self):
|
||||
"""Test parsing 'inf' string in filter."""
|
||||
filter_obj = PlotFilterBase.parse_str("request_rate==inf")
|
||||
assert isinstance(filter_obj, PlotEqualTo)
|
||||
assert filter_obj.var == "request_rate"
|
||||
assert filter_obj.target == "inf"
|
||||
|
||||
def test_parse_multiple_filters(self):
|
||||
"""Test parsing multiple filters."""
|
||||
filters = PlotFilters.parse_str("request_rate>5.0,value<=40")
|
||||
assert len(filters) == 2
|
||||
assert isinstance(filters[0], PlotGreaterThan)
|
||||
assert isinstance(filters[1], PlotLessThanOrEqualTo)
|
||||
|
||||
def test_parse_empty_filter(self):
|
||||
"""Test parsing empty filter string."""
|
||||
filters = PlotFilters.parse_str("")
|
||||
assert len(filters) == 0
|
||||
@@ -1,486 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
from typing import Any, NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
RandomDataset,
|
||||
RandomMultiModalDataset,
|
||||
SampleRequest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
class Params(NamedTuple):
|
||||
num_requests: int
|
||||
prefix_len: int
|
||||
range_ratio: float
|
||||
input_len: int
|
||||
output_len: int
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def random_dataset_params() -> Params:
|
||||
return Params(
|
||||
num_requests=16, prefix_len=7, range_ratio=0.3, input_len=50, output_len=20
|
||||
)
|
||||
|
||||
|
||||
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
|
||||
"""Project a SampleRequest into a comparable tuple."""
|
||||
assert isinstance(req.prompt, str)
|
||||
return (req.prompt, req.prompt_len, req.expected_output_len)
|
||||
|
||||
|
||||
def _collect_samples(
|
||||
dataset: RandomDataset,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
num_requests: int = 16,
|
||||
prefix_len: int = 7,
|
||||
range_ratio: float = 0.3,
|
||||
input_len: int = 50,
|
||||
output_len: int = 20,
|
||||
) -> list[tuple[str, int, int]]:
|
||||
samples = dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=num_requests,
|
||||
prefix_len=prefix_len,
|
||||
range_ratio=range_ratio,
|
||||
input_len=input_len,
|
||||
output_len=output_len,
|
||||
)
|
||||
return [_fingerprint_sample(s) for s in samples]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_dataset_same_seed(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
|
||||
) -> None:
|
||||
"""Same seed should yield identical outputs, even if global RNGs change.
|
||||
|
||||
This guards against accidental reliance on Python's random or np.random
|
||||
in RandomDataset after moving to numpy.default_rng.
|
||||
"""
|
||||
p = random_dataset_params
|
||||
common_seed = 123
|
||||
dataset_a = RandomDataset(random_seed=common_seed)
|
||||
dataset_b = RandomDataset(random_seed=common_seed)
|
||||
a = _collect_samples(
|
||||
dataset_a,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
|
||||
# Perturb global RNG state to ensure isolation
|
||||
random.seed(999)
|
||||
_ = [random.random() for _ in range(100)]
|
||||
np.random.seed(888)
|
||||
_ = [np.random.random() for _ in range(100)]
|
||||
|
||||
b = _collect_samples(
|
||||
dataset_b,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
assert a == b
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_dataset_different_seeds(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
|
||||
) -> None:
|
||||
"""Different seeds should change outputs with overwhelming likelihood."""
|
||||
p = random_dataset_params
|
||||
seed_a = 0
|
||||
dataset_a = RandomDataset(random_seed=seed_a)
|
||||
a = _collect_samples(
|
||||
dataset_a,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
|
||||
seed_b = 999
|
||||
dataset_b = RandomDataset(random_seed=seed_b)
|
||||
# Perturb global RNG with same seed as dataset_a to ensure isolation
|
||||
random.seed(seed_a)
|
||||
np.random.seed(seed_a)
|
||||
b = _collect_samples(
|
||||
dataset_b,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
assert a != b
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RandomMultiModalDataset tests
|
||||
# -----------------------------
|
||||
|
||||
|
||||
def _mm_fingerprint_sample(
|
||||
req: SampleRequest,
|
||||
) -> tuple[str, int, int, int, list[str]]:
|
||||
"""Create a compact fingerprint for multimodal samples.
|
||||
|
||||
Includes:
|
||||
- prompt string
|
||||
- prompt_len
|
||||
- expected_output_len
|
||||
- count of multimodal items
|
||||
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
|
||||
"""
|
||||
assert isinstance(req.prompt, str)
|
||||
items = req.multi_modal_data or []
|
||||
item_prefixes: list[str] = []
|
||||
for it in items:
|
||||
if isinstance(it, dict) and it.get("type") == "image_url":
|
||||
url = it.get("image_url", {}).get("url", "")
|
||||
# Only keep a short identifying prefix to avoid huge strings
|
||||
item_prefixes.append(f"image:{url[:22]}")
|
||||
elif isinstance(it, dict) and it.get("type") == "video_url":
|
||||
url = it.get("video_url", {}).get("url", "")
|
||||
item_prefixes.append(f"video:{url[:22]}")
|
||||
else:
|
||||
item_prefixes.append("unknown:")
|
||||
return (
|
||||
req.prompt,
|
||||
req.prompt_len,
|
||||
req.expected_output_len,
|
||||
len(items),
|
||||
item_prefixes,
|
||||
)
|
||||
|
||||
|
||||
def _collect_mm_samples(
|
||||
dataset: RandomMultiModalDataset,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
*,
|
||||
num_requests: int = 8,
|
||||
prefix_len: int = 3,
|
||||
range_ratio: float = 0.0,
|
||||
input_len: int = 20,
|
||||
output_len: int = 5,
|
||||
base_items_per_request: int = 2,
|
||||
num_mm_items_range_ratio: float = 0.0,
|
||||
limit_mm_per_prompt: dict[str, int] | None = None,
|
||||
bucket_config: dict[tuple[int, int, int], float] | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
) -> list[SampleRequest]:
|
||||
if limit_mm_per_prompt is None:
|
||||
limit_mm_per_prompt = {"image": 5, "video": 0}
|
||||
if bucket_config is None:
|
||||
bucket_config = {(32, 32, 1): 0.5, (52, 64, 1): 0.5}
|
||||
return dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=num_requests,
|
||||
prefix_len=prefix_len,
|
||||
range_ratio=range_ratio,
|
||||
input_len=input_len,
|
||||
output_len=output_len,
|
||||
base_items_per_request=base_items_per_request,
|
||||
num_mm_items_range_ratio=num_mm_items_range_ratio,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
enable_multimodal_chat=enable_multimodal_chat,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_same_seed(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
seed = 42
|
||||
ds_a = RandomMultiModalDataset(random_seed=seed)
|
||||
ds_b = RandomMultiModalDataset(random_seed=seed)
|
||||
a = _collect_mm_samples(ds_a, hf_tokenizer)
|
||||
b = _collect_mm_samples(ds_b, hf_tokenizer)
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa == fb
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_different_seeds(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds_a = RandomMultiModalDataset(random_seed=0)
|
||||
ds_b = RandomMultiModalDataset(random_seed=999)
|
||||
a = _collect_mm_samples(ds_a, hf_tokenizer)
|
||||
b = _collect_mm_samples(ds_b, hf_tokenizer)
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa != fb
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_respects_limits(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Requesting 3 items with a per-prompt limit of 1 should error per current
|
||||
# design (dataset refuses to silently clamp below the requested baseline).
|
||||
with pytest.raises(ValueError):
|
||||
_collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=12,
|
||||
base_items_per_request=3,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_zero_prob_entries_are_removed(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Second bucket has zero probability and should be ignored after
|
||||
# normalization
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=6,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 10, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0, (52, 64, 1): 0.0},
|
||||
)
|
||||
for s in samples:
|
||||
assert isinstance(s.multi_modal_data, list)
|
||||
typed_mm = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
for it in typed_mm:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_zero_items(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=0,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 5, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
for s in samples:
|
||||
assert s.multi_modal_data == []
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_num_items_per_prompt(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Fixed number of images per prompt
|
||||
# set num_mm_items_range_ratio to 0.0
|
||||
# TODO: modify video values when video sampling is implemented
|
||||
samples_fixed_items = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=3,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 3, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
# Must have 5 requests each with 3 mm items per prompt
|
||||
assert len(samples_fixed_items) == 5
|
||||
for s in samples_fixed_items:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 3
|
||||
for it in mm_data:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_bucket_config_not_mutated(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# This bucket config is not normalized to sum to 1
|
||||
# and has more buckets than requested images
|
||||
original = {(32, 32, 1): 0.2, (52, 64, 1): 6, (25, 64, 1): 3}
|
||||
# Keep a snapshot to compare after sampling
|
||||
snapshot = dict(original)
|
||||
|
||||
_ = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=4,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
bucket_config=original,
|
||||
)
|
||||
|
||||
# Ensure the original dict content is unchanged
|
||||
assert original == snapshot
|
||||
|
||||
# Vary number of mm items per prompt
|
||||
# set num_mm_items_range_ratio to 0.5
|
||||
samples_varying_items = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.5,
|
||||
limit_mm_per_prompt={"image": 4, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
# Must have 5 requests each with less than 4 mm items per prompt
|
||||
# but at least 1 mm item per prompt
|
||||
assert len(samples_varying_items) == 5
|
||||
for s in samples_varying_items:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) <= 4
|
||||
assert len(mm_data) >= 1
|
||||
for it in mm_data:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
"""Test video sampling functionality in RandomMultiModalDataset."""
|
||||
ds = RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
# Test with video bucket configuration
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
assert len(samples) == 5
|
||||
|
||||
# Check that we have both images and videos
|
||||
video_count = 0
|
||||
image_count = 0
|
||||
|
||||
for s in samples:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
if item.get("type") == "video_url":
|
||||
video_count += 1
|
||||
# Verify video URL format
|
||||
url = item.get("video_url", {}).get("url", "")
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
elif item.get("type") == "image_url":
|
||||
image_count += 1
|
||||
# Verify image URL format
|
||||
url = item.get("image_url", {}).get("url", "")
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
# Should have some videos due to 0.7 probability
|
||||
assert video_count > 0
|
||||
assert image_count > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
"""Test sampling with only video buckets."""
|
||||
ds = RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for s in samples:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
assert item.get("type") == "video_url"
|
||||
url = item.get("video_url", {}).get("url", "")
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_deterministic_sampling(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Test that video sampling is deterministic with same seed."""
|
||||
seed = 123
|
||||
ds_a = RandomMultiModalDataset(random_seed=seed)
|
||||
ds_b = RandomMultiModalDataset(random_seed=seed)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
a = _collect_mm_samples(
|
||||
ds_a,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
b = _collect_mm_samples(
|
||||
ds_b,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa == fb
|
||||
@@ -1,434 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import cv2
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
"""Use a small, commonly available tokenizer."""
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_dataset() -> RandomMultiModalDataset:
|
||||
"""Create a RandomMultiModalDataset instance for testing."""
|
||||
return RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_synthetic_video_different_seeds():
|
||||
"""Test that different seeds produce different videos."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=456)
|
||||
|
||||
width, height, num_frames = 64, 48, 8
|
||||
|
||||
video1 = dataset1.generate_synthetic_video(width, height, num_frames)
|
||||
video2 = dataset2.generate_synthetic_video(width, height, num_frames)
|
||||
|
||||
# Videos should be different due to different seeds
|
||||
assert video1["bytes"] != video2["bytes"]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
|
||||
"""Test modality mapping for different configurations."""
|
||||
# Test image configuration (num_frames = 1)
|
||||
assert video_dataset.map_config_to_modality((256, 256, 1)) == "image"
|
||||
assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image"
|
||||
|
||||
# Test video configurations (num_frames > 1)
|
||||
assert video_dataset.map_config_to_modality((256, 256, 8)) == "video"
|
||||
assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video"
|
||||
assert video_dataset.map_config_to_modality((64, 64, 32)) == "video"
|
||||
|
||||
# Test invalid configurations
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.map_config_to_modality((256, 256, 0))
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.map_config_to_modality((256, 256, -1))
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_video(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Generate a decodable video without leaving temporary files behind."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
# Test video item generation
|
||||
video_config = (64, 48, 8) # height, width, num_frames
|
||||
result = video_dataset.generate_mm_item(video_config)
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
# Check the result structure matches OpenAI API format
|
||||
assert isinstance(result, dict)
|
||||
assert result["type"] == "video_url"
|
||||
assert "video_url" in result
|
||||
assert "url" in result["video_url"]
|
||||
|
||||
# Check that the URL is a data URL with base64 encoded video
|
||||
url = result["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
# Decode and verify the video content
|
||||
base64_data = url.split(",")[1]
|
||||
video_bytes = base64.b64decode(base64_data)
|
||||
assert len(video_bytes) > 0
|
||||
|
||||
# Verify the video can be decoded
|
||||
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
temp_file.write(video_bytes)
|
||||
|
||||
try:
|
||||
cap = cv2.VideoCapture(temp_path)
|
||||
assert cap.isOpened()
|
||||
|
||||
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
|
||||
assert frame_count == 8
|
||||
assert frame_width == 48
|
||||
assert frame_height == 64
|
||||
|
||||
cap.release()
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
|
||||
def test_generate_synthetic_video_cleans_up_on_error(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
writer_opened: bool,
|
||||
):
|
||||
"""Release the writer and remove temporary storage when encoding fails."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
writer = Mock(spec=cv2.VideoWriter)
|
||||
writer.isOpened.return_value = writer_opened
|
||||
writer.write.side_effect = RuntimeError("Failed to write video frame")
|
||||
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
|
||||
|
||||
error = (
|
||||
"Failed to write video frame"
|
||||
if writer_opened
|
||||
else "Failed to create video writer"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
video_dataset.generate_synthetic_video(48, 64, 8)
|
||||
|
||||
assert not list(tmp_path.iterdir())
|
||||
writer.release.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
|
||||
"""Test generating multimodal items for image configurations."""
|
||||
# Test image item generation
|
||||
image_config = (64, 48, 1) # height, width, num_frames=1
|
||||
result = video_dataset.generate_mm_item(image_config)
|
||||
|
||||
# Check the result structure matches OpenAI API format
|
||||
assert isinstance(result, dict)
|
||||
assert result["type"] == "image_url"
|
||||
assert "image_url" in result
|
||||
assert "url" in result["image_url"]
|
||||
|
||||
# Check that the URL is a data URL with base64 encoded image
|
||||
url = result["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset):
|
||||
"""Test error handling for invalid configurations."""
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.generate_mm_item((256, 256, 0))
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_with_video_buckets(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with video bucket configurations."""
|
||||
# Configure bucket with video probability > 0
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 5, "video": 3}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 5
|
||||
|
||||
# Check that samples contain both images and videos
|
||||
video_count = 0
|
||||
image_count = 0
|
||||
|
||||
for sample in samples:
|
||||
assert isinstance(sample, SampleRequest)
|
||||
assert sample.multi_modal_data is not None
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) == 2 # base_items_per_request
|
||||
|
||||
for item in mm_data:
|
||||
if item["type"] == "video_url":
|
||||
video_count += 1
|
||||
# Verify video URL format
|
||||
url = item["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
elif item["type"] == "image_url":
|
||||
image_count += 1
|
||||
# Verify image URL format
|
||||
url = item["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
# Should have some videos due to 0.7 probability
|
||||
assert video_count > 0
|
||||
assert image_count > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_video_only_buckets(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with only video buckets."""
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 2}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for sample in samples:
|
||||
assert isinstance(sample, SampleRequest)
|
||||
assert sample.multi_modal_data is not None
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
assert item["type"] == "video_url"
|
||||
url = item["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_respects_video_limits(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test that sampling respects video limits per prompt."""
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
# Set very low video limit
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for sample in samples:
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) <= 1 # Should respect video limit
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_mixed_buckets_with_zero_probability(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with mixed buckets including zero probability entries."""
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.5, # Images
|
||||
(64, 64, 8): 0.5, # Videos
|
||||
(128, 128, 16): 0.0, # Zero probability videos (should be ignored)
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=4,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 4
|
||||
|
||||
# Should only see 64x64 videos, not 128x128 videos
|
||||
for sample in samples:
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
for item in mm_data:
|
||||
if item["type"] == "video_url":
|
||||
# Decode video to verify dimensions
|
||||
url = item["video_url"]["url"]
|
||||
base64_data = url.split(",")[1]
|
||||
video_bytes = base64.b64decode(base64_data)
|
||||
|
||||
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa
|
||||
temp_path = temp_file.name
|
||||
temp_file.write(video_bytes)
|
||||
|
||||
try:
|
||||
cap = cv2.VideoCapture(temp_path)
|
||||
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
cap.release()
|
||||
|
||||
# Should be 64x64, not 128x128
|
||||
assert frame_width == 64
|
||||
assert frame_height == 64
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase):
|
||||
"""Test that sampling with videos is deterministic with same seed."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=123)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples1 = dataset1.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
samples2 = dataset2.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples1) == len(samples2)
|
||||
|
||||
# Compare multimodal data
|
||||
for s1, s2 in zip(samples1, samples2):
|
||||
assert s1.multi_modal_data == s2.multi_modal_data
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_different_seeds_produce_different_videos(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
"""Test that different seeds produce different video content."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=456)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples1 = dataset1.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=2,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
samples2 = dataset2.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=2,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
# Video content should be different
|
||||
for s1, s2 in zip(samples1, samples2):
|
||||
mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data)
|
||||
mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data)
|
||||
|
||||
assert len(mm_data1) == len(mm_data2) == 1
|
||||
|
||||
url1 = mm_data1[0]["video_url"]["url"]
|
||||
url2 = mm_data2[0]["video_url"]["url"]
|
||||
|
||||
assert url1 != url2 # Different video content
|
||||
@@ -1,453 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
|
||||
|
||||
The Responses stream differs from Chat Completions in ways the benchmark client
|
||||
has to get right: every SSE message is labelled with an ``event:`` line,
|
||||
reasoning and output text arrive as separate token-bearing events, and a
|
||||
request ends with ``response.completed`` rather than with its last token. These
|
||||
tests drive the real request function against a local SSE server so the
|
||||
resulting latency and usage numbers are covered end to end.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestServer
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
ASYNC_REQUEST_FUNCS,
|
||||
OPENAI_COMPATIBLE_BACKENDS,
|
||||
RequestFuncInput,
|
||||
RequestFuncOutput,
|
||||
async_request_openai_responses,
|
||||
)
|
||||
|
||||
# These tests never touch torch, so skip the GPU cleanup fixture.
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
MODEL = "openai/gpt-oss-20b"
|
||||
|
||||
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
|
||||
# that a time to first token measured at the wrong event is unambiguous.
|
||||
GAP_PREFILL = 0.15
|
||||
GAP_TOKEN = 0.5
|
||||
GAP_FINALIZE = 0.3
|
||||
|
||||
|
||||
def sse(event_type: str, payload: dict) -> bytes:
|
||||
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
|
||||
|
||||
``ensure_ascii=False`` matches the server, whose pydantic
|
||||
``model_dump_json`` emits non-ASCII characters unescaped.
|
||||
"""
|
||||
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
|
||||
return f"event: {event_type}\ndata: {data}\n\n".encode()
|
||||
|
||||
|
||||
def usage_event(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
event_type: str = "response.completed",
|
||||
status: str = "completed",
|
||||
):
|
||||
return sse(
|
||||
event_type,
|
||||
{
|
||||
"response": {
|
||||
"status": status,
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def text_delta(delta: str) -> bytes:
|
||||
return sse("response.output_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
def reasoning_delta(delta: str) -> bytes:
|
||||
return sse("response.reasoning_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
# The metadata events the server sends before any token. None of these may
|
||||
# start the TTFT clock or contribute an ITL sample.
|
||||
PREAMBLE = (
|
||||
sse("response.created", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.output_item.added", {"output_index": 0})
|
||||
+ sse("response.reasoning_part.added", {"output_index": 0})
|
||||
+ sse("response.content_part.added", {"output_index": 0})
|
||||
)
|
||||
|
||||
# The `.done` family the server sends after the last delta. These repeat the
|
||||
# accumulated text, so timing them would add bogus ITL samples and counting
|
||||
# their payload would duplicate the output.
|
||||
DONE_EVENTS = (
|
||||
sse("response.reasoning_text.done", {"text": "The user asks "})
|
||||
+ sse("response.reasoning_part.done", {"output_index": 0})
|
||||
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
|
||||
+ sse("response.content_part.done", {"output_index": 0})
|
||||
+ sse("response.output_item.done", {"output_index": 0})
|
||||
)
|
||||
|
||||
|
||||
async def _serve(
|
||||
script: Sequence[bytes | float],
|
||||
*,
|
||||
status: int = 200,
|
||||
chunk_size: int | None = None,
|
||||
seen: dict | None = None,
|
||||
**request_kwargs,
|
||||
) -> RequestFuncOutput:
|
||||
"""Run the request function against a server replaying ``script``.
|
||||
|
||||
Args:
|
||||
script: Byte strings to write, interleaved with floats to sleep for.
|
||||
status: HTTP status to respond with. Non-200 skips the script.
|
||||
chunk_size: If set, split every write into chunks of this many bytes to
|
||||
emulate fragmented transport.
|
||||
seen: If given, populated with the request payload.
|
||||
request_kwargs: Extra fields for the ``RequestFuncInput``.
|
||||
|
||||
"""
|
||||
|
||||
async def handler(request: web.Request) -> web.StreamResponse:
|
||||
if seen is not None:
|
||||
seen["payload"] = await request.json()
|
||||
seen["headers"] = dict(request.headers)
|
||||
if status != 200:
|
||||
return web.Response(status=status, reason="Bad Request")
|
||||
|
||||
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
|
||||
await response.prepare(request)
|
||||
for item in script:
|
||||
if isinstance(item, float):
|
||||
await asyncio.sleep(item)
|
||||
elif chunk_size is None:
|
||||
await response.write(item)
|
||||
else:
|
||||
for i in range(0, len(item), chunk_size):
|
||||
await response.write(item[i : i + chunk_size])
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_post("/v1/responses", handler)
|
||||
server = TestServer(app)
|
||||
await server.start_server()
|
||||
try:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="Why is the sky blue?",
|
||||
api_url=str(server.make_url("/v1/responses")),
|
||||
prompt_len=5,
|
||||
output_len=16,
|
||||
model=MODEL,
|
||||
**request_kwargs,
|
||||
)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
return await async_request_openai_responses(request_input, session)
|
||||
finally:
|
||||
await server.close()
|
||||
|
||||
|
||||
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
|
||||
return asyncio.run(_serve(script, **kwargs))
|
||||
|
||||
|
||||
def test_backend_is_registered_as_openai_compatible():
|
||||
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
|
||||
# Membership enables --ignore-eos on random datasets and the sampling
|
||||
# parameter flags, all of which ResponsesRequest accepts.
|
||||
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
|
||||
|
||||
|
||||
def test_request_payload_is_accepted_by_the_server_schema():
|
||||
"""Every field sent must be a real ResponsesRequest field.
|
||||
|
||||
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
|
||||
the flags that membership enables have to reach the model, and the server
|
||||
drops unknown fields without complaining.
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
sampling = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"top_k": 20,
|
||||
"min_p": 0.05,
|
||||
"frequency_penalty": 0.2,
|
||||
"presence_penalty": 0.1,
|
||||
"repetition_penalty": 1.05,
|
||||
}
|
||||
seen: dict = {}
|
||||
output = run(
|
||||
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
|
||||
seen=seen,
|
||||
extra_body=sampling,
|
||||
ignore_eos=True,
|
||||
model_name="served-alias",
|
||||
request_id="bench-7",
|
||||
extra_headers={"x-team": "perf"},
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# --served-model-name wins over --model, as on the other backends.
|
||||
assert seen["payload"]["model"] == "served-alias"
|
||||
# The route is behind validate_json_request, which 415s a missing type.
|
||||
assert seen["headers"]["Content-Type"] == "application/json"
|
||||
assert seen["headers"]["x-request-id"] == "bench-7"
|
||||
assert seen["headers"]["x-team"] == "perf"
|
||||
assert seen["payload"] == {
|
||||
"model": "served-alias",
|
||||
"input": "Why is the sky blue?",
|
||||
"max_output_tokens": 16,
|
||||
"stream": True,
|
||||
"store": False,
|
||||
"ignore_eos": True,
|
||||
**sampling,
|
||||
}
|
||||
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
|
||||
ResponsesRequest.model_validate(seen["payload"])
|
||||
|
||||
|
||||
def test_latency_accounting_matches_the_other_endpoints():
|
||||
"""`latency - ttft` must equal `sum(itl)`.
|
||||
|
||||
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
|
||||
means average inter-token latency when that identity holds. #55508 fixed the
|
||||
same accounting on the chat and audio backends; the trailing events after
|
||||
the last token must not extend the request.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
GAP_PREFILL / 2,
|
||||
# A keep-alive comment sent during prefill is not a token.
|
||||
b": keep-alive\n\n",
|
||||
GAP_PREFILL / 2,
|
||||
reasoning_delta("The user asks "),
|
||||
GAP_TOKEN,
|
||||
text_delta("Rayleigh"),
|
||||
text_delta(" scattering."),
|
||||
DONE_EVENTS,
|
||||
GAP_FINALIZE,
|
||||
usage_event(5, 8),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# Reasoning is timed but not returned, matching how `openai-chat` treats
|
||||
# `DeltaMessage.reasoning`.
|
||||
assert output.generated_text == "Rayleigh scattering."
|
||||
|
||||
# Neither the response.created/in_progress/output_item.added preamble nor
|
||||
# the keep-alive comment may start the clock, and the reasoning delta must,
|
||||
# so TTFT lands on the prefill gap rather than near zero or out at the
|
||||
# first output text delta.
|
||||
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
|
||||
|
||||
# One ITL per token event after the first: the gap before "Rayleigh" and
|
||||
# the back-to-back delta that follows it.
|
||||
assert len(output.itl) == 2
|
||||
assert output.itl[0] >= GAP_TOKEN * 0.5
|
||||
assert output.itl[1] < GAP_TOKEN * 0.5
|
||||
|
||||
# The identity is exact, so no wall-clock margin is involved. The .done
|
||||
# family and the GAP_FINALIZE wait before response.completed sit outside
|
||||
# the request, exactly as the trailing usage chunk does on chat.
|
||||
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
|
||||
|
||||
assert output.start_time > 0.0
|
||||
|
||||
|
||||
def test_usage_is_taken_from_the_terminal_event():
|
||||
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
|
||||
|
||||
assert output.success
|
||||
assert output.prompt_len == 17
|
||||
assert output.output_tokens == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
|
||||
def test_incomplete_response_is_a_successful_request(event_type: str):
|
||||
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
|
||||
|
||||
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
|
||||
The spec's dedicated `response.incomplete` event is accepted as well.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("truncated"),
|
||||
usage_event(17, 16, event_type=event_type, status="incomplete"),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "truncated"
|
||||
assert output.output_tokens == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
|
||||
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
|
||||
"""Splits inside the event line, the JSON, or a multibyte character."""
|
||||
text = " leading 中 trailing "
|
||||
output = run(
|
||||
[PREAMBLE, text_delta(text), usage_event(5, 6)],
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == text
|
||||
assert output.output_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
|
||||
)
|
||||
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
|
||||
"""The server emits these unescaped and they are legal inside a JSON string.
|
||||
|
||||
`str.splitlines()` treats them as line breaks, which would cut the `data:`
|
||||
line short and fail the request on valid model output.
|
||||
"""
|
||||
text = f"before{char}after"
|
||||
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == text
|
||||
|
||||
|
||||
def test_empty_data_line_is_skipped():
|
||||
"""A bare `data:` line carries no JSON and must not fail the request."""
|
||||
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_prompt_len_falls_back_when_usage_is_absent():
|
||||
"""A terminal event without usage keeps the locally computed prompt length."""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("hi"),
|
||||
sse("response.completed", {"response": {"status": "completed"}}),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.prompt_len == 5
|
||||
|
||||
|
||||
def test_multi_line_data_payload_is_joined_with_newlines():
|
||||
"""The SSE spec allows one payload to span several `data:` lines."""
|
||||
script: list[bytes | float] = [
|
||||
PREAMBLE,
|
||||
b"event: response.output_text.delta\n"
|
||||
b'data: {"type": "response.output_text.delta",\n'
|
||||
b'data: "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_done_sentinel_is_skipped():
|
||||
"""Other Responses servers send `[DONE]`, but vLLM does not."""
|
||||
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_error_before_any_token_reports_the_server_error():
|
||||
"""The server's message must win over the generic no-TTFT message."""
|
||||
output = run(
|
||||
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_http_error_is_not_reported_as_success():
|
||||
output = run([], status=400)
|
||||
|
||||
assert not output.success
|
||||
assert output.error
|
||||
|
||||
|
||||
def test_stream_without_tokens_is_not_reported_as_success():
|
||||
"""A stream that finishes without generating anything has no TTFT."""
|
||||
output = run([PREAMBLE, usage_event(5, 0)])
|
||||
|
||||
assert not output.success
|
||||
assert "TTFT" in output.error
|
||||
|
||||
|
||||
def test_truncated_stream_is_not_reported_as_success():
|
||||
"""A stream cut short after a token has no usage and an understated E2EL."""
|
||||
output = run([PREAMBLE, text_delta("partial")])
|
||||
|
||||
assert not output.success
|
||||
assert "terminal event" in output.error
|
||||
|
||||
|
||||
def test_mid_stream_error_is_not_reported_as_success():
|
||||
"""An error payload mid-stream must not be counted as a success.
|
||||
|
||||
vLLM cannot emit this today: its streaming error path fails to validate
|
||||
against the event union, so a generation failure truncates the stream
|
||||
instead. Proxies and other Responses servers do send it.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("partial"),
|
||||
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
|
||||
]
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_data_only_framing_is_still_parsed():
|
||||
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
|
||||
script: list[bytes | float] = [
|
||||
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_endpoint_path_is_validated():
|
||||
request_input = RequestFuncInput(
|
||||
prompt="hi",
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=1,
|
||||
output_len=1,
|
||||
model=MODEL,
|
||||
)
|
||||
|
||||
async def call() -> None:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
await async_request_openai_responses(request_input, session)
|
||||
|
||||
with pytest.raises(ValueError, match="responses"):
|
||||
asyncio.run(call())
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Keep the `vllm bench serve` flag snapshot in sync for the Rust CLI parity test.
|
||||
|
||||
`rust/src/bench/tests/cli_parity.rs` asserts that every flag in the snapshot is
|
||||
either a known `vllm-bench` clap flag/alias or explicitly allowlisted as
|
||||
Python-only, so `VLLM_USE_RUST_BENCH=1` delegation cannot silently reject
|
||||
documented flags. This test keeps the snapshot itself current.
|
||||
|
||||
Run this file directly to regenerate the snapshot:
|
||||
|
||||
.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SNAPSHOT = REPO_ROOT / "rust" / "src" / "bench" / "tests" / "python_serve_flags.txt"
|
||||
HEADER = """\
|
||||
# Long CLI flags of the Python `vllm bench serve` parser
|
||||
# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
|
||||
# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
|
||||
# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
|
||||
# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
|
||||
"""
|
||||
|
||||
|
||||
def _current_flags() -> list[str]:
|
||||
from vllm.benchmarks.serve import add_cli_args
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
return sorted(
|
||||
{
|
||||
opt
|
||||
for action in parser._actions
|
||||
for opt in action.option_strings
|
||||
if opt.startswith("--") and opt != "--help"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_flags() -> list[str]:
|
||||
lines = SNAPSHOT.read_text().splitlines()
|
||||
return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
|
||||
|
||||
|
||||
def test_serve_flag_snapshot_is_current():
|
||||
assert SNAPSHOT.is_file(), f"missing snapshot {SNAPSHOT}"
|
||||
assert _current_flags() == _snapshot_flags(), (
|
||||
"`vllm bench serve` flags changed. Regenerate the snapshot with "
|
||||
"`.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py`, "
|
||||
"then for any added flag either support it in rust/src/bench/src/cli.rs "
|
||||
"or add it to PYTHON_ONLY in rust/src/bench/tests/cli_parity.rs"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
SNAPSHOT.write_text(HEADER + "\n".join(_current_flags()) + "\n")
|
||||
print(f"wrote {SNAPSHOT}")
|
||||
@@ -1,258 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets.utils import get_sampling_params
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
class _FakeTokenizer(TokenizerLike):
|
||||
"""Minimal tokenizer implementing the TokenizerLike protocol
|
||||
for testing get_sampling_params."""
|
||||
|
||||
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
|
||||
self._vocab_size = vocab_size
|
||||
self._num_special_tokens = num_special_tokens
|
||||
|
||||
# -- Properties required by TokenizerLike --
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw):
|
||||
return cls()
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self._vocab_size
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return []
|
||||
|
||||
@property
|
||||
def bos_token_id(self) -> int:
|
||||
return 0
|
||||
|
||||
@property
|
||||
def eos_token_id(self) -> int:
|
||||
return 1
|
||||
|
||||
@property
|
||||
def pad_token_id(self) -> int:
|
||||
return 2
|
||||
|
||||
@property
|
||||
def is_fast(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def max_token_id(self) -> int:
|
||||
return self._vocab_size - 1
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return 4
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
return "right"
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return self._num_special_tokens
|
||||
|
||||
def __call__(self, text, text_pair=None, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def get_added_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text, **kw) -> list[int]: # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def apply_chat_template(self, messages, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_ids(self, tokens):
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_ids_to_tokens(
|
||||
self, ids, skip_special_tokens: bool = False
|
||||
) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TestGetSamplingParams:
|
||||
"""Tests for ``get_sampling_params`` in ``vllm.benchmarks.datasets.shared``."""
|
||||
|
||||
# -- helpers --
|
||||
|
||||
@staticmethod
|
||||
def _tok(vocab_size: int = 1000, num_special: int = 0) -> _FakeTokenizer:
|
||||
return _FakeTokenizer(vocab_size=vocab_size, num_special_tokens=num_special)
|
||||
|
||||
# -- return shape / dtype --
|
||||
|
||||
def test_returns_three_arrays(self):
|
||||
rng = np.random.default_rng(0)
|
||||
result = get_sampling_params(rng, 5, 0.0, 100, 50, self._tok())
|
||||
assert len(result) == 3
|
||||
for arr in result:
|
||||
assert isinstance(arr, np.ndarray)
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 10, 100])
|
||||
def test_output_length_matches_num_requests(self, n: int):
|
||||
rng = np.random.default_rng(42)
|
||||
input_lens, output_lens, offsets = get_sampling_params(
|
||||
rng, n, 0.0, 64, 32, self._tok()
|
||||
)
|
||||
assert input_lens.shape == (n,)
|
||||
assert output_lens.shape == (n,)
|
||||
assert offsets.shape == (n,)
|
||||
|
||||
# -- fixed lengths (range_ratio = 0) --
|
||||
|
||||
def test_zero_range_ratio_gives_constant_lengths(self):
|
||||
rng = np.random.default_rng(7)
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng, 20, 0.0, 128, 64, self._tok()
|
||||
)
|
||||
assert np.all(input_lens == 128)
|
||||
assert np.all(output_lens == 64)
|
||||
|
||||
def test_special_tokens_subtracted_from_input_only(self):
|
||||
rng = np.random.default_rng(7)
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng, 10, 0.0, 100, 50, self._tok(num_special=4)
|
||||
)
|
||||
# real_input_len = 100 - 4 = 96, range_ratio 0 → all 96
|
||||
assert np.all(input_lens == 96)
|
||||
# special tokens are not subtracted from output length
|
||||
assert np.all(output_lens == 50)
|
||||
|
||||
# -- range ratios --
|
||||
|
||||
def test_input_range_bounds(self):
|
||||
rng = np.random.default_rng(0)
|
||||
ratio = 0.5
|
||||
base = 200
|
||||
input_lens, _, _ = get_sampling_params(
|
||||
rng, 500, {"input": ratio, "output": 0.0}, base, 50, self._tok()
|
||||
)
|
||||
lo = int(np.floor(base * (1 - ratio)))
|
||||
hi = int(np.ceil(base * (1 + ratio)))
|
||||
assert np.all(input_lens >= lo)
|
||||
assert np.all(input_lens <= hi)
|
||||
|
||||
def test_output_range_bounds(self):
|
||||
rng = np.random.default_rng(0)
|
||||
ratio = 0.3
|
||||
base = 100
|
||||
_, output_lens, _ = get_sampling_params(
|
||||
rng, 500, {"input": 0.0, "output": ratio}, 50, base, self._tok()
|
||||
)
|
||||
lo = max(1, int(np.floor(base * (1 - ratio))))
|
||||
hi = int(np.ceil(base * (1 + ratio)))
|
||||
assert np.all(output_lens >= lo)
|
||||
assert np.all(output_lens <= hi)
|
||||
|
||||
def test_output_low_clamped_to_one(self):
|
||||
"""Even with a high ratio that would push output_low to 0,
|
||||
the function clamps it to 1."""
|
||||
rng = np.random.default_rng(0)
|
||||
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
|
||||
_, output_lens, _ = get_sampling_params(
|
||||
rng, 50, {"input": 0.0, "output": 0.99}, 100, 1, self._tok()
|
||||
)
|
||||
assert np.all(output_lens >= 1)
|
||||
|
||||
# -- offsets bounded by vocab_size --
|
||||
|
||||
@pytest.mark.parametrize("vocab", [100, 32000, 128256])
|
||||
def test_offsets_within_vocab(self, vocab: int):
|
||||
rng = np.random.default_rng(0)
|
||||
_, _, offsets = get_sampling_params(
|
||||
rng, 200, 0.0, 64, 32, self._tok(vocab_size=vocab)
|
||||
)
|
||||
assert np.all(offsets >= 0)
|
||||
assert np.all(offsets < vocab)
|
||||
|
||||
# -- reproducibility --
|
||||
|
||||
def test_same_seed_same_results(self):
|
||||
tok = self._tok()
|
||||
rr = {"input": 0.3, "output": 0.2}
|
||||
a = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
|
||||
b = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
|
||||
for arr_a, arr_b in zip(a, b):
|
||||
np.testing.assert_array_equal(arr_a, arr_b)
|
||||
|
||||
def test_different_seed_different_results(self):
|
||||
tok = self._tok()
|
||||
rr = {"input": 0.3, "output": 0.2}
|
||||
a = get_sampling_params(np.random.default_rng(0), 50, rr, 256, 64, tok)
|
||||
b = get_sampling_params(np.random.default_rng(1), 50, rr, 256, 64, tok)
|
||||
# Extremely unlikely all three arrays match with different seeds
|
||||
assert not all(np.array_equal(arr_a, arr_b) for arr_a, arr_b in zip(a, b))
|
||||
|
||||
# -- validation / error paths --
|
||||
|
||||
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
|
||||
def test_invalid_input_range_ratio(self, bad_ratio: float):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="input_range_ratio"):
|
||||
get_sampling_params(
|
||||
rng, 10, {"input": bad_ratio, "output": 0.0}, 100, 50, self._tok()
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
|
||||
def test_invalid_output_range_ratio(self, bad_ratio: float):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="output_range_ratio"):
|
||||
get_sampling_params(
|
||||
rng, 10, {"input": 0.0, "output": bad_ratio}, 100, 50, self._tok()
|
||||
)
|
||||
|
||||
def test_invalid_dict_missing_keys(self):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="input.*output"):
|
||||
get_sampling_params(rng, 10, {"input": 0.1}, 100, 50, self._tok())
|
||||
|
||||
def test_input_len_zero_with_special_tokens(self):
|
||||
"""input_len < num_special_tokens → real_input_len = 0, which is fine
|
||||
(range [0, 0])."""
|
||||
rng = np.random.default_rng(0)
|
||||
input_lens, _, _ = get_sampling_params(
|
||||
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
|
||||
)
|
||||
# real_input_len = max(0, 5 - 10) = 0
|
||||
assert np.all(input_lens == 0)
|
||||
|
||||
# -- edge cases --
|
||||
|
||||
def test_single_request(self):
|
||||
rng = np.random.default_rng(0)
|
||||
i, o, off = get_sampling_params(rng, 1, 0.0, 100, 50, self._tok())
|
||||
assert i.shape == (1,)
|
||||
assert o.shape == (1,)
|
||||
assert off.shape == (1,)
|
||||
|
||||
def test_large_num_requests(self):
|
||||
rng = np.random.default_rng(0)
|
||||
i, o, off = get_sampling_params(rng, 10_000, 0.5, 512, 128, self._tok())
|
||||
assert i.shape == (10_000,)
|
||||
assert o.shape == (10_000,)
|
||||
assert off.shape == (10_000,)
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for --skip-tokenizer-init with --dataset-name custom.
|
||||
|
||||
Before the fix (introduced by #39896), running:
|
||||
|
||||
vllm bench serve \
|
||||
--backend vllm-pooling \
|
||||
--dataset-name custom \
|
||||
--dataset-path <path> \
|
||||
--model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \
|
||||
--endpoint /pooling \
|
||||
--skip-tokenizer-init \
|
||||
...
|
||||
|
||||
raised immediately with:
|
||||
|
||||
AssertionError: Tokenizer must be initialized before loading dataset
|
||||
|
||||
even though CustomDataset.sample() already handles tokenizer=None.
|
||||
This test exercises main_async() directly so it catches any regression
|
||||
re-introduced at the serve.py level, not just inside get_samples().
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.benchmarks.serve as serve_module
|
||||
|
||||
# Exact prompt payload from the failing benchmark run against a
|
||||
# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format).
|
||||
_PRITHVI_PROMPT = {
|
||||
"data": {
|
||||
"data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif",
|
||||
"data_format": "url",
|
||||
"out_data_format": "b64_json",
|
||||
"indices": [1, 2, 3, 8, 11, 12],
|
||||
},
|
||||
"priority": 0,
|
||||
"softmax": False,
|
||||
}
|
||||
|
||||
|
||||
def _write_dataset(path: Path) -> None:
|
||||
path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n")
|
||||
|
||||
|
||||
def _args(dataset_path: str) -> argparse.Namespace:
|
||||
"""Reproduce the argparse.Namespace that serve.py builds from the
|
||||
failing command, including skip_tokenizer_init=True."""
|
||||
return argparse.Namespace(
|
||||
# dataset
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=False,
|
||||
num_prompts=1,
|
||||
custom_output_len=256,
|
||||
skip_chat_template=True,
|
||||
chat_template_kwargs=None,
|
||||
no_oversample=False,
|
||||
seed=0,
|
||||
request_id_prefix="bench-",
|
||||
# model / tokenizer
|
||||
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
|
||||
served_model_name=None,
|
||||
tokenizer=None,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=True, # <-- the flag under test
|
||||
# backend / endpoint
|
||||
backend="vllm-pooling",
|
||||
base_url="http://127.0.0.1:8000",
|
||||
host="127.0.0.1",
|
||||
port=8000,
|
||||
endpoint="/pooling",
|
||||
header=None,
|
||||
insecure=False,
|
||||
# traffic
|
||||
request_rate=16.0,
|
||||
burstiness=1.0,
|
||||
max_concurrency=None,
|
||||
probe_request_rate=0.0,
|
||||
# misc serve args that main_async reads before reaching get_samples
|
||||
plot_timeline=False,
|
||||
plot_dataset_stats=False,
|
||||
self_timed=None,
|
||||
metadata=None,
|
||||
label=None,
|
||||
logprobs=None,
|
||||
use_beam_search=False,
|
||||
ignore_eos=False,
|
||||
goodput=None,
|
||||
percentile_metrics="ttft,tpot,itl,e2el",
|
||||
metric_percentiles="25,50,75,99",
|
||||
save_result=False,
|
||||
append_result=False,
|
||||
result_dir=".",
|
||||
result_filename=None,
|
||||
num_warmups=0,
|
||||
profile=False,
|
||||
disable_tqdm=True,
|
||||
lora_modules=None,
|
||||
lora_assignment="random",
|
||||
ramp_up_strategy=None,
|
||||
ramp_up_start_rps=None,
|
||||
ramp_up_end_rps=None,
|
||||
ready_check_timeout_sec=0,
|
||||
extra_body=None,
|
||||
top_p=None,
|
||||
top_k=None,
|
||||
min_p=None,
|
||||
temperature=None,
|
||||
frequency_penalty=None,
|
||||
presence_penalty=None,
|
||||
repetition_penalty=None,
|
||||
save_detailed=False,
|
||||
input_len=None,
|
||||
output_len=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None:
|
||||
"""main_async must not raise AssertionError when skip_tokenizer_init=True
|
||||
and dataset_name='custom'.
|
||||
|
||||
On main (before the fix) this test fails with:
|
||||
AssertionError: Tokenizer must be initialized before loading dataset
|
||||
"""
|
||||
dataset_path = tmp_path / "dataset_url_input_india.jsonl"
|
||||
_write_dataset(dataset_path)
|
||||
|
||||
args = _args(str(dataset_path))
|
||||
|
||||
# Patch benchmark() so we never make real HTTP requests — the regression
|
||||
# triggers before benchmark() is ever called, so this just keeps the test
|
||||
# fast and hermetic.
|
||||
mock_result = {
|
||||
"completed": 1,
|
||||
"failed": 0,
|
||||
"total_input_tokens": 1,
|
||||
"total_output_tokens": 1,
|
||||
"request_throughput": 1.0,
|
||||
"output_throughput": 1.0,
|
||||
"total_token_throughput": 1.0,
|
||||
"input_lens": [],
|
||||
"output_lens": [],
|
||||
"ttfts": [],
|
||||
"itls": [],
|
||||
"generated_texts": [],
|
||||
"errors": [],
|
||||
"duration": 1.0,
|
||||
}
|
||||
with patch.object(
|
||||
serve_module, "benchmark", new=AsyncMock(return_value=mock_result)
|
||||
):
|
||||
# Must NOT raise AssertionError
|
||||
asyncio.run(serve_module.main_async(args))
|
||||
@@ -1,369 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ``vllm bench throughput`` CLI.
|
||||
|
||||
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
|
||||
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
|
||||
(serve has no analogue), keeping the two benchmarks aligned without duplicating
|
||||
serve-side dataset coverage.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from argparse import Namespace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
from vllm.benchmarks.throughput import (
|
||||
_run_vllm_chat_requests,
|
||||
_to_serve_args,
|
||||
add_cli_args,
|
||||
assign_loras,
|
||||
get_requests,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bench_throughput():
|
||||
command = [
|
||||
"vllm",
|
||||
"bench",
|
||||
"throughput",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--input-len",
|
||||
"32",
|
||||
"--output-len",
|
||||
"1",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
print(result.stderr)
|
||||
|
||||
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
|
||||
|
||||
|
||||
def test_bench_throughput_accepts_custom_audio_args():
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"custom_audio",
|
||||
"--dataset-path",
|
||||
"audio.jsonl",
|
||||
"--no-oversample",
|
||||
"--custom-output-len",
|
||||
"32",
|
||||
"--enable-multimodal-chat",
|
||||
]
|
||||
)
|
||||
|
||||
assert args.dataset_name == "custom_audio"
|
||||
assert args.no_oversample
|
||||
assert args.custom_output_len == 32
|
||||
assert args.enable_multimodal_chat
|
||||
|
||||
|
||||
def test_vllm_chat_requests_include_multimodal_content():
|
||||
class FakeLLM:
|
||||
def __init__(self):
|
||||
self.prompts = None
|
||||
|
||||
def chat(self, prompts, sampling_params, use_tqdm):
|
||||
del sampling_params, use_tqdm
|
||||
self.prompts = prompts
|
||||
return []
|
||||
|
||||
llm = FakeLLM()
|
||||
audio_content = {
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "abc", "format": "wav"},
|
||||
}
|
||||
request = SampleRequest(
|
||||
prompt="Transcribe this audio.",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
multi_modal_data=audio_content,
|
||||
)
|
||||
|
||||
_run_vllm_chat_requests(
|
||||
llm,
|
||||
[request],
|
||||
n=1,
|
||||
disable_detokenize=False,
|
||||
do_profile=False,
|
||||
prequeue_requests=False,
|
||||
)
|
||||
|
||||
assert llm.prompts == [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Transcribe this audio."},
|
||||
audio_content,
|
||||
],
|
||||
}
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Small, commonly available tokenizer.
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# LoRA assignment (throughput-only post-processing)
|
||||
# -----------------------------
|
||||
|
||||
|
||||
def _sr(i: int) -> SampleRequest:
|
||||
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
|
||||
|
||||
|
||||
def _lora_args(
|
||||
*,
|
||||
lora_path: str | None,
|
||||
max_loras: int,
|
||||
lora_assignment: str = "random",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
lora_path=lora_path,
|
||||
max_loras=max_loras,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_noop_without_lora_path() -> None:
|
||||
"""No lora_path -> requests returned unchanged, no LoRA attached."""
|
||||
reqs = [_sr(i) for i in range(5)]
|
||||
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
|
||||
assert out is reqs
|
||||
assert all(r.lora_request is None for r in out)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_round_robin() -> None:
|
||||
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
|
||||
reqs = [_sr(i) for i in range(6)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
|
||||
)
|
||||
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_random_in_range() -> None:
|
||||
"""Random assignment stays in [1, max_loras] and covers all IDs."""
|
||||
reqs = [_sr(i) for i in range(200)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
|
||||
)
|
||||
ids = [r.lora_request.lora_int_id for r in out]
|
||||
assert all(1 <= i <= 5 for i in ids)
|
||||
assert len(set(ids)) == 5
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Issue #50838: MMVU via throughput
|
||||
# -----------------------------
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_resolves_mmvu(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
|
||||
|
||||
Previously throughput's validate_args rejected MMVU ("is not supported by
|
||||
hf dataset"). It now resolves through the shared get_samples. The HF
|
||||
download is stubbed so the resolution + dispatch path is exercised without
|
||||
network access.
|
||||
"""
|
||||
import vllm.benchmarks.datasets.datasets as dsmod
|
||||
|
||||
class _StubMMVUDataset:
|
||||
# MMVU is multimodal in content but, like the real class, does not set
|
||||
# IS_MULTIMODAL -- so it is not subject to the backend gate.
|
||||
IS_MULTIMODAL = False
|
||||
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
|
||||
return [
|
||||
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"yale-nlp/MMVU",
|
||||
"--backend",
|
||||
"vllm-chat",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_allows_multimodal_on_plain_vllm_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Regression: --backend vllm must accept multimodal datasets such as ASR.
|
||||
|
||||
_run_vllm_requests forwards multi_modal_data straight to LLM.generate, so
|
||||
the vllm-chat-only gate in get_samples rejected a backend that always
|
||||
worked.
|
||||
"""
|
||||
import vllm.benchmarks.datasets.datasets as dsmod
|
||||
|
||||
class _StubASRDataset:
|
||||
IS_MULTIMODAL = True
|
||||
SUPPORTED_DATASET_PATHS = {"openslr/librispeech_asr"}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
|
||||
return [
|
||||
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(dsmod, "ASRDataset", _StubASRDataset)
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"openslr/librispeech_asr",
|
||||
"--backend",
|
||||
"vllm",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
|
||||
|
||||
def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
"""The adapter must serve callers without throughput's legacy flags.
|
||||
|
||||
``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
|
||||
nor --backend, and leaves --random-prefix-len at its default of 0. The
|
||||
adapter must not read the legacy flags unconditionally, and must keep the
|
||||
explicit 0 instead of coercing it to None (which later crashed
|
||||
RandomMultiModalDataset.get_prefix on ``None <= 0``).
|
||||
"""
|
||||
args = Namespace(
|
||||
random_input_len=32,
|
||||
random_output_len=1,
|
||||
random_prefix_len=0,
|
||||
)
|
||||
serve_args = _to_serve_args(args)
|
||||
|
||||
assert serve_args.random_input_len == 32
|
||||
assert serve_args.random_output_len == 1
|
||||
assert serve_args.random_prefix_len == 0
|
||||
assert serve_args.backend == "vllm-chat"
|
||||
assert serve_args.enable_multimodal_chat
|
||||
assert serve_args.no_oversample is False
|
||||
|
||||
|
||||
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
|
||||
"""Callers that only set the legacy flags still get them mapped."""
|
||||
args = Namespace(
|
||||
input_len=256,
|
||||
output_len=8,
|
||||
prefix_len=16,
|
||||
backend="vllm",
|
||||
)
|
||||
serve_args = _to_serve_args(args)
|
||||
|
||||
assert serve_args.random_input_len == 256
|
||||
assert serve_args.random_output_len == 8
|
||||
assert serve_args.random_prefix_len == 16
|
||||
assert serve_args.backend == "vllm"
|
||||
assert not serve_args.enable_multimodal_chat
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_random_mm_with_mm_processor_args(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Regression: bench mm-processor's random-mm sampling must not crash.
|
||||
|
||||
get_requests feeds mm-processor's parsed args (no legacy length/backend
|
||||
flags) through the shared get_samples dispatch. With the default
|
||||
--random-prefix-len 0 this previously crashed inside
|
||||
RandomMultiModalDataset.get_prefix with a TypeError.
|
||||
"""
|
||||
from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_mm_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"random-mm",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
"--random-input-len",
|
||||
"32",
|
||||
"--random-output-len",
|
||||
"1",
|
||||
"--random-mm-base-items-per-request",
|
||||
"1",
|
||||
"--random-mm-num-mm-items-range-ratio",
|
||||
"0",
|
||||
"--random-mm-limit-mm-per-prompt",
|
||||
'{"image": 1}',
|
||||
"--random-mm-bucket-config",
|
||||
"{(224, 224, 1): 1.0}",
|
||||
]
|
||||
)
|
||||
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
# mm-processor drives llm.chat, so prompts must be chat-formatted with
|
||||
# multimodal content attached.
|
||||
for req in requests:
|
||||
assert isinstance(req.prompt, list)
|
||||
assert req.prompt[0]["role"] == "user"
|
||||
assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import CustomDataset
|
||||
from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices_jsonl
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
text_content = """
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
|
||||
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
|
||||
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
|
||||
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
|
||||
nulla pariatur. Excepteur sint occaecat cupidatat non proident,
|
||||
sunt in culpa qui officia deserunt mollit anim id est laborum.
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_create_txt_slices_jsonl(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""Test that create_txt_slices_jsonl produces valid JSONL for CustomDataset."""
|
||||
txt_path = tmp_path / "input.txt"
|
||||
jsonl_path = tmp_path / "input.txt.jsonl"
|
||||
|
||||
txt_path.write_text(text_content)
|
||||
|
||||
create_txt_slices_jsonl(
|
||||
input_path=str(txt_path),
|
||||
output_path=str(jsonl_path),
|
||||
tokenizer_name="openai-community/gpt2",
|
||||
num_prompts=10,
|
||||
input_len=10,
|
||||
output_len=10,
|
||||
)
|
||||
|
||||
# Verify the JSONL file is valid and has the expected structure
|
||||
records = [json.loads(line) for line in jsonl_path.read_text().splitlines()]
|
||||
|
||||
assert len(records) == 10
|
||||
for record in records:
|
||||
assert "prompt" in record
|
||||
assert "output_tokens" in record
|
||||
assert isinstance(record["prompt"], str)
|
||||
assert record["output_tokens"] == 10
|
||||
|
||||
# Verify the JSONL file can be loaded by CustomDataset
|
||||
dataset = CustomDataset(dataset_path=str(jsonl_path))
|
||||
samples = dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=10,
|
||||
output_len=10,
|
||||
skip_chat_template=True,
|
||||
)
|
||||
|
||||
assert len(samples) == 10
|
||||
assert all(sample.expected_output_len == 10 for sample in samples)
|
||||
Reference in New Issue
Block a user