[MISC][Bench] refactor throughput and reuse serve's get samples (#50981)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
Jared Wen
2026-08-06 10:31:23 +00:00
committed by GitHub
parent 872fd5973e
commit e07532b035
3 changed files with 228 additions and 231 deletions
+128
View File
@@ -1,13 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ``vllm bench throughput`` CLI.
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
(serve has no analogue), keeping the two benchmarks aligned without duplicating
serve-side dataset coverage.
"""
import subprocess
from types import SimpleNamespace
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import SampleRequest
from vllm.benchmarks.throughput import (
_run_vllm_chat_requests,
add_cli_args,
assign_loras,
get_requests,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -102,3 +114,119 @@ def test_vllm_chat_requests_include_multimodal_content():
}
]
]
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Small, commonly available tokenizer.
return AutoTokenizer.from_pretrained("openai-community/gpt2")
# -----------------------------
# LoRA assignment (throughput-only post-processing)
# -----------------------------
def _sr(i: int) -> SampleRequest:
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
def _lora_args(
*,
lora_path: str | None,
max_loras: int,
lora_assignment: str = "random",
) -> SimpleNamespace:
return SimpleNamespace(
lora_path=lora_path,
max_loras=max_loras,
lora_assignment=lora_assignment,
)
@pytest.mark.benchmark
def test_assign_loras_noop_without_lora_path() -> None:
"""No lora_path -> requests returned unchanged, no LoRA attached."""
reqs = [_sr(i) for i in range(5)]
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
assert out is reqs
assert all(r.lora_request is None for r in out)
@pytest.mark.benchmark
def test_assign_loras_round_robin() -> None:
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
reqs = [_sr(i) for i in range(6)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
)
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
@pytest.mark.benchmark
def test_assign_loras_random_in_range() -> None:
"""Random assignment stays in [1, max_loras] and covers all IDs."""
reqs = [_sr(i) for i in range(200)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
)
ids = [r.lora_request.lora_int_id for r in out]
assert all(1 <= i <= 5 for i in ids)
assert len(set(ids)) == 5
# -----------------------------
# Issue #50838: MMVU via throughput
# -----------------------------
@pytest.mark.benchmark
def test_get_requests_resolves_mmvu(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
Previously throughput's validate_args rejected MMVU ("is not supported by
hf dataset"). It now resolves through the shared get_samples. The HF
download is stubbed so the resolution + dispatch path is exercised without
network access.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubMMVUDataset:
# MMVU is multimodal in content but, like the real class, does not set
# IS_MULTIMODAL -- so it is not subject to the backend gate.
IS_MULTIMODAL = False
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"yale-nlp/MMVU",
"--backend",
"vllm-chat",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)