[MISC][Bench] refactor throughput and reuse serve's get samples (#50981)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
Jared Wen
2026-08-06 10:31:23 +00:00
committed by GitHub
parent 872fd5973e
commit e07532b035
3 changed files with 228 additions and 231 deletions
+128
View File
@@ -1,13 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ``vllm bench throughput`` CLI.
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
(serve has no analogue), keeping the two benchmarks aligned without duplicating
serve-side dataset coverage.
"""
import subprocess
from types import SimpleNamespace
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import SampleRequest
from vllm.benchmarks.throughput import (
_run_vllm_chat_requests,
add_cli_args,
assign_loras,
get_requests,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -102,3 +114,119 @@ def test_vllm_chat_requests_include_multimodal_content():
}
]
]
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Small, commonly available tokenizer.
return AutoTokenizer.from_pretrained("openai-community/gpt2")
# -----------------------------
# LoRA assignment (throughput-only post-processing)
# -----------------------------
def _sr(i: int) -> SampleRequest:
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
def _lora_args(
*,
lora_path: str | None,
max_loras: int,
lora_assignment: str = "random",
) -> SimpleNamespace:
return SimpleNamespace(
lora_path=lora_path,
max_loras=max_loras,
lora_assignment=lora_assignment,
)
@pytest.mark.benchmark
def test_assign_loras_noop_without_lora_path() -> None:
"""No lora_path -> requests returned unchanged, no LoRA attached."""
reqs = [_sr(i) for i in range(5)]
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
assert out is reqs
assert all(r.lora_request is None for r in out)
@pytest.mark.benchmark
def test_assign_loras_round_robin() -> None:
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
reqs = [_sr(i) for i in range(6)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
)
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
@pytest.mark.benchmark
def test_assign_loras_random_in_range() -> None:
"""Random assignment stays in [1, max_loras] and covers all IDs."""
reqs = [_sr(i) for i in range(200)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
)
ids = [r.lora_request.lora_int_id for r in out]
assert all(1 <= i <= 5 for i in ids)
assert len(set(ids)) == 5
# -----------------------------
# Issue #50838: MMVU via throughput
# -----------------------------
@pytest.mark.benchmark
def test_get_requests_resolves_mmvu(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
Previously throughput's validate_args rejected MMVU ("is not supported by
hf dataset"). It now resolves through the shared get_samples. The HF
download is stubbed so the resolution + dispatch path is exercised without
network access.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubMMVUDataset:
# MMVU is multimodal in content but, like the real class, does not set
# IS_MULTIMODAL -- so it is not subject to the backend gate.
IS_MULTIMODAL = False
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"yale-nlp/MMVU",
"--backend",
"vllm-chat",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
+28 -16
View File
@@ -164,8 +164,8 @@ class BenchmarkDataset(ABC):
# TODO (jenniferzhao): add support for downloading data
raise NotImplementedError("load_data must be implemented in subclasses.")
@staticmethod
def get_random_lora_request(
self,
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
@@ -197,8 +197,8 @@ class BenchmarkDataset(ABC):
)
return lora_request
@staticmethod
def get_round_robin_lora_request(
self,
index: int,
max_loras: int | None = None,
lora_path: str | None = None,
@@ -232,8 +232,8 @@ class BenchmarkDataset(ABC):
)
return lora_request
@staticmethod
def get_lora_request(
self,
index: int,
max_loras: int | None = None,
lora_path: str | None = None,
@@ -254,10 +254,12 @@ class BenchmarkDataset(ABC):
(or `None` if not applicable).
"""
if lora_assignment == "round-robin":
return self.get_round_robin_lora_request(
return BenchmarkDataset.get_round_robin_lora_request(
index=index, max_loras=max_loras, lora_path=lora_path
)
return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path)
return BenchmarkDataset.get_random_lora_request(
max_loras=max_loras, lora_path=lora_path
)
@abstractmethod
def sample(
@@ -2091,7 +2093,12 @@ def _parse_range_ratio(value: str) -> RangeRatio:
return json.loads(value)
def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
def get_samples(
args,
tokenizer: TokenizerLike | None,
*,
multimodal_backends: tuple[str, ...] = ("openai-chat", "openai-audio"),
) -> list[SampleRequest]:
if not hasattr(args, "request_id_prefix"):
args.request_id_prefix = ""
@@ -2310,15 +2317,17 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
"like to add support for additional dataset formats."
)
if dataset_class.IS_MULTIMODAL and not (
args.backend in ("openai-chat", "openai-audio")
or "embeddings-" in args.backend
if (
dataset_class.IS_MULTIMODAL
and args.backend not in multimodal_backends
and "embeddings-" not in args.backend
):
# multi-modal benchmark is only available on OpenAI Chat
# endpoint-type.
# multi-modal benchmark is only available on chat-style backends;
# which ones are allowed is caller-controlled (serve uses the
# OpenAI endpoints, throughput uses vllm-chat).
raise ValueError(
"Multi-modal content is only supported on 'openai-chat' and "
"'openai-audio' backends."
f"Multi-modal content is not supported on backend "
f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
)
assert tokenizer is not None, (
"Tokenizer must be initialized for the 'hf' dataset."
@@ -2474,10 +2483,13 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
try:
# Enforce endpoint compatibility for multimodal datasets.
if args.dataset_name == "random-mm" and args.backend not in ["openai-chat"]:
if (
args.dataset_name == "random-mm"
and args.backend not in multimodal_backends
):
raise ValueError(
"Multi-modal content (images) is only supported on "
"'openai-chat' backend."
f"Multi-modal content (images) is not supported on backend "
f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
)
input_requests = dataset_mapping[args.dataset_name]()
except KeyError as err:
+72 -215
View File
@@ -16,24 +16,11 @@ from tqdm import tqdm
from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import (
AIMODataset,
ASRDataset,
BenchmarkDataset,
BurstGPTDataset,
ConversationDataset,
CustomAudioDataset,
InstructCoderDataset,
MultiModalConversationDataset,
PrefixRepetitionRandomDataset,
RandomDataset,
RandomDatasetForReranking,
RandomMultiModalDataset,
SampleRequest,
ShareGPTDataset,
SonnetDataset,
VisionArenaDataset,
add_random_dataset_base_args,
add_random_multimodal_dataset_args,
get_samples,
)
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
@@ -536,181 +523,56 @@ def save_to_pytorch_benchmark_format(
write_to_json(pt_file, pt_records)
def get_requests(args, tokenizer):
# Common parameters for all dataset types.
dataset_cls: type[BenchmarkDataset]
common_kwargs = {
"dataset_path": args.dataset_path,
"random_seed": args.seed,
}
sample_kwargs = {
"tokenizer": tokenizer,
"lora_path": args.lora_path,
"max_loras": args.max_loras,
"lora_assignment": getattr(args, "lora_assignment", "random"),
"num_requests": args.num_prompts,
"no_oversample": getattr(args, "no_oversample", False),
}
def _to_serve_args(args) -> argparse.Namespace:
"""Translate throughput args into a namespace the shared get_samples reads.
if args.dataset_name == "random" or (
args.dataset_path is None
and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"}
):
sample_kwargs["range_ratio"] = args.random_range_ratio
# prefer random_* arguments, fall back to regular arguments
random_prefix_len = getattr(args, "random_prefix_len", None)
sample_kwargs["prefix_len"] = (
random_prefix_len if random_prefix_len is not None else args.prefix_len
)
random_input_len = getattr(args, "random_input_len", None)
sample_kwargs["input_len"] = (
random_input_len if random_input_len is not None else args.input_len
)
random_output_len = getattr(args, "random_output_len", None)
sample_kwargs["output_len"] = (
random_output_len if random_output_len is not None else args.output_len
)
dataset_cls = RandomDataset
elif args.dataset_name == "sharegpt":
dataset_cls = ShareGPTDataset
if args.backend == "vllm-chat":
sample_kwargs["enable_multimodal_chat"] = True
if args.output_len is not None:
sample_kwargs["output_len"] = args.output_len
elif args.dataset_name == "sonnet":
assert tokenizer.chat_template or tokenizer.default_chat_template, (
"Tokenizer/model must have chat template for sonnet dataset."
)
dataset_cls = SonnetDataset
sample_kwargs["prefix_len"] = args.prefix_len
sample_kwargs["return_prompt_formatted"] = True
if args.input_len is not None:
sample_kwargs["input_len"] = args.input_len
if args.output_len is not None:
sample_kwargs["output_len"] = args.output_len
elif args.dataset_name == "burstgpt":
dataset_cls = BurstGPTDataset
elif args.dataset_name == "custom_audio":
dataset_cls = CustomAudioDataset
sample_kwargs["enable_multimodal_chat"] = getattr(
args, "enable_multimodal_chat", False
)
custom_output_len = getattr(args, "custom_output_len", None)
if custom_output_len is not None:
sample_kwargs["output_len"] = custom_output_len
elif args.output_len is not None:
sample_kwargs["output_len"] = args.output_len
elif args.dataset_name == "hf":
if args.output_len is not None:
sample_kwargs["output_len"] = args.output_len
common_kwargs["hf_name"] = args.hf_name
if (
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = VisionArenaDataset
common_kwargs["dataset_subset"] = None
common_kwargs["dataset_split"] = "train"
sample_kwargs["enable_multimodal_chat"] = True
elif (
args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = InstructCoderDataset
common_kwargs["dataset_split"] = "train"
elif (
args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = MultiModalConversationDataset
common_kwargs["dataset_subset"] = args.hf_subset
common_kwargs["dataset_split"] = args.hf_split
sample_kwargs["enable_multimodal_chat"] = True
elif (
args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in ConversationDataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = ConversationDataset
common_kwargs["dataset_subset"] = args.hf_subset
common_kwargs["dataset_split"] = args.hf_split
sample_kwargs["enable_multimodal_chat"] = True
elif (
args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS
or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = AIMODataset
common_kwargs["dataset_subset"] = None
common_kwargs["dataset_split"] = "train"
elif (
args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS
):
dataset_cls = ASRDataset
common_kwargs["dataset_subset"] = args.hf_subset
common_kwargs["dataset_split"] = args.hf_split
sample_kwargs["asr_min_audio_len_sec"] = args.asr_min_audio_len_sec
sample_kwargs["asr_max_audio_len_sec"] = args.asr_max_audio_len_sec
elif args.dataset_name == "prefix_repetition":
dataset_cls = PrefixRepetitionRandomDataset
sample_kwargs["prefix_len"] = args.prefix_repetition_prefix_len
sample_kwargs["suffix_len"] = args.prefix_repetition_suffix_len
sample_kwargs["num_prefixes"] = args.prefix_repetition_num_prefixes
sample_kwargs["output_len"] = args.prefix_repetition_output_len
elif args.dataset_name == "random-mm":
dataset_cls = RandomMultiModalDataset
# prefer random_* arguments, fall back to regular arguments
random_input_len = getattr(args, "random_input_len", None)
sample_kwargs["input_len"] = (
random_input_len
if random_input_len is not None
else getattr(args, "input_len", None)
)
random_output_len = getattr(args, "random_output_len", None)
sample_kwargs["output_len"] = (
random_output_len
if random_output_len is not None
else getattr(args, "output_len", None)
)
sample_kwargs["base_items_per_request"] = getattr(
args, "random_mm_base_items_per_request", None
)
sample_kwargs["num_mm_items_range_ratio"] = getattr(
args, "random_mm_num_mm_items_range_ratio", None
)
sample_kwargs["limit_mm_per_prompt"] = getattr(
args, "random_mm_limit_mm_per_prompt", None
)
sample_kwargs["bucket_config"] = getattr(args, "random_mm_bucket_config", None)
sample_kwargs["enable_multimodal_chat"] = True
random_prefix_len = getattr(args, "random_prefix_len", None)
prefix_len = getattr(args, "prefix_len", None)
sample_kwargs["prefix_len"] = (
random_prefix_len if random_prefix_len is not None else prefix_len
)
sample_kwargs["range_ratio"] = args.random_range_ratio
elif args.dataset_name == "random-rerank":
dataset_cls = RandomDatasetForReranking
# prefer random_* arguments, fall back to regular arguments
random_input_len = getattr(args, "random_input_len", None)
sample_kwargs["input_len"] = (
random_input_len
if random_input_len is not None
else getattr(args, "input_len", None)
)
random_output_len = getattr(args, "random_output_len", None)
sample_kwargs["output_len"] = (
random_output_len
if random_output_len is not None
else getattr(args, "output_len", None)
)
sample_kwargs["batchsize"] = getattr(args, "random_batch_size", 1)
sample_kwargs["is_reranker"] = not getattr(args, "no_reranker", False)
sample_kwargs["range_ratio"] = args.random_range_ratio
else:
raise ValueError(f"Unknown dataset name: {args.dataset_name}")
# Remove None values
sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None}
requests = dataset_cls(**common_kwargs).sample(**sample_kwargs)
``get_samples`` (used by ``bench serve``) expects ~45 attributes; the
throughput CLI exposes most of them under the same names, and this adapter
fills the rest while preserving throughput's existing flag names so no
script breaks.
Args:
args: parsed throughput CLI namespace.
Returns:
A namespace satisfying get_samples's attribute reads.
"""
d = vars(args).copy()
# random_*: prefer --random-* over legacy --input/output/prefix-len.
d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len
d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len
d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len
# --output-len maps to the per-dataset output-len entry points get_samples
# reads. None passes through (each dataset applies its own default),
# matching prior throughput behaviour of omitting output_len when unset.
d["hf_output_len"] = args.output_len
d["sharegpt_output_len"] = args.output_len
# sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550
d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150
d["sonnet_prefix_len"] = args.prefix_len
# Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
# multimodal chat backend (preserves today's vllm-chat handling).
d["enable_multimodal_chat"] = bool(
getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat"
)
# serve-only attrs throughput never exposed; keep serve's defaults.
d.setdefault("disable_shuffle", False)
d.setdefault("skip_chat_template", False)
d.setdefault("no_stream", False)
d.setdefault("request_id_prefix", "")
d.setdefault("chat_template_kwargs", None)
return argparse.Namespace(**d)
def get_requests(args, tokenizer):
serve_args = _to_serve_args(args)
# Throughput reuses bench serve's dataset dispatch. vllm-chat is the only
# multimodal-capable throughput backend, so it is the only one allowed past
# the multimodal gates inside get_samples; other backends raise there.
mm_backends = ("vllm-chat",) if args.backend == "vllm-chat" else ()
requests = get_samples(serve_args, tokenizer, multimodal_backends=mm_backends)
requests = assign_loras(requests, args)
requests = filter_requests_for_dp(requests, args.data_parallel_size)
return requests
@@ -732,6 +594,28 @@ def filter_requests_for_dp(requests, data_parallel_size):
]
def assign_loras(requests, args):
"""Attach a LoRA request to each sample (throughput-only post-processing).
The shared ``datasets.get_samples`` path carries no LoRA information, so
LoRA assignment is applied here, uniformly across every dataset type. No-op
when ``--lora-path`` is unset.
"""
lora_path = getattr(args, "lora_path", None)
if not lora_path:
return requests
max_loras = args.max_loras
lora_assignment = getattr(args, "lora_assignment", "random")
for i, req in enumerate(requests):
req.lora_request = BenchmarkDataset.get_lora_request(
index=i,
max_loras=max_loras,
lora_path=lora_path,
lora_assignment=lora_assignment,
)
return requests
def validate_args(args):
"""
Validate command-line arguments.
@@ -785,33 +669,6 @@ def validate_args(args):
since --dataset-name is not 'hf'.",
stacklevel=2,
)
elif args.dataset_name == "hf":
if args.dataset_path in (
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
| ConversationDataset.SUPPORTED_DATASET_PATHS
) or args.hf_name in (
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
| ConversationDataset.SUPPORTED_DATASET_PATHS
):
assert args.backend == "vllm-chat", (
f"{args.dataset_path} needs to use vllm-chat as the backend."
)
elif args.dataset_path in (
InstructCoderDataset.SUPPORTED_DATASET_PATHS
| AIMODataset.SUPPORTED_DATASET_PATHS
| ASRDataset.SUPPORTED_DATASET_PATHS
) or args.hf_name in (
InstructCoderDataset.SUPPORTED_DATASET_PATHS
| AIMODataset.SUPPORTED_DATASET_PATHS
| ASRDataset.SUPPORTED_DATASET_PATHS
):
assert args.backend == "vllm", (
f"{args.dataset_path} needs to use vllm as the backend."
)
else:
raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")
# --random-range-ratio: only used when dataset_name is 'random',
# 'random-mm', or 'random-rerank'