diff --git a/tests/benchmarks/test_throughput_cli.py b/tests/benchmarks/test_throughput_cli.py index 87a8cecd5e..8f65872249 100644 --- a/tests/benchmarks/test_throughput_cli.py +++ b/tests/benchmarks/test_throughput_cli.py @@ -1,13 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ``vllm bench throughput`` CLI. + +Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the +LoRA-assignment and MMVU cases below cover the pieces that are throughput-only +(serve has no analogue), keeping the two benchmarks aligned without duplicating +serve-side dataset coverage. +""" + import subprocess +from types import SimpleNamespace import pytest +from transformers import AutoTokenizer, PreTrainedTokenizerBase from vllm.benchmarks.datasets import SampleRequest from vllm.benchmarks.throughput import ( _run_vllm_chat_requests, add_cli_args, + assign_loras, + get_requests, ) from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -102,3 +114,119 @@ def test_vllm_chat_requests_include_multimodal_content(): } ] ] + + +@pytest.fixture(scope="session") +def hf_tokenizer() -> PreTrainedTokenizerBase: + # Small, commonly available tokenizer. + return AutoTokenizer.from_pretrained("openai-community/gpt2") + + +# ----------------------------- +# LoRA assignment (throughput-only post-processing) +# ----------------------------- + + +def _sr(i: int) -> SampleRequest: + return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2) + + +def _lora_args( + *, + lora_path: str | None, + max_loras: int, + lora_assignment: str = "random", +) -> SimpleNamespace: + return SimpleNamespace( + lora_path=lora_path, + max_loras=max_loras, + lora_assignment=lora_assignment, + ) + + +@pytest.mark.benchmark +def test_assign_loras_noop_without_lora_path() -> None: + """No lora_path -> requests returned unchanged, no LoRA attached.""" + reqs = [_sr(i) for i in range(5)] + out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4)) + assert out is reqs + assert all(r.lora_request is None for r in out) + + +@pytest.mark.benchmark +def test_assign_loras_round_robin() -> None: + """Round-robin assigns deterministic index % max_loras + 1 IDs.""" + reqs = [_sr(i) for i in range(6)] + out = assign_loras( + reqs, + _lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"), + ) + assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3] + + +@pytest.mark.benchmark +def test_assign_loras_random_in_range() -> None: + """Random assignment stays in [1, max_loras] and covers all IDs.""" + reqs = [_sr(i) for i in range(200)] + out = assign_loras( + reqs, + _lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"), + ) + ids = [r.lora_request.lora_int_id for r in out] + assert all(1 <= i <= 5 for i in ids) + assert len(set(ids)) == 5 + + +# ----------------------------- +# Issue #50838: MMVU via throughput +# ----------------------------- + + +@pytest.mark.benchmark +def test_get_requests_resolves_mmvu( + monkeypatch: pytest.MonkeyPatch, + hf_tokenizer: PreTrainedTokenizerBase, +) -> None: + """Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput. + + Previously throughput's validate_args rejected MMVU ("is not supported by + hf dataset"). It now resolves through the shared get_samples. The HF + download is stubbed so the resolution + dispatch path is exercised without + network access. + """ + import vllm.benchmarks.datasets.datasets as dsmod + + class _StubMMVUDataset: + # MMVU is multimodal in content but, like the real class, does not set + # IS_MULTIMODAL -- so it is not subject to the backend gate. + IS_MULTIMODAL = False + SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]} + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002 + return [ + SampleRequest(prompt="q", prompt_len=1, expected_output_len=1) + for _ in range(num_requests) + ] + + monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset) + + parser = FlexibleArgumentParser() + add_cli_args(parser) + args = parser.parse_args( + [ + "--dataset-name", + "hf", + "--dataset-path", + "yale-nlp/MMVU", + "--backend", + "vllm-chat", + "--num-prompts", + "3", + ] + ) + requests = get_requests(args, hf_tokenizer) + assert len(requests) == 3 + assert all(isinstance(r, SampleRequest) for r in requests) diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index de98ab135a..2db092580a 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -164,8 +164,8 @@ class BenchmarkDataset(ABC): # TODO (jenniferzhao): add support for downloading data raise NotImplementedError("load_data must be implemented in subclasses.") + @staticmethod def get_random_lora_request( - self, max_loras: int | None = None, lora_path: str | None = None, ) -> LoRARequest | None: @@ -197,8 +197,8 @@ class BenchmarkDataset(ABC): ) return lora_request + @staticmethod def get_round_robin_lora_request( - self, index: int, max_loras: int | None = None, lora_path: str | None = None, @@ -232,8 +232,8 @@ class BenchmarkDataset(ABC): ) return lora_request + @staticmethod def get_lora_request( - self, index: int, max_loras: int | None = None, lora_path: str | None = None, @@ -254,10 +254,12 @@ class BenchmarkDataset(ABC): (or `None` if not applicable). """ if lora_assignment == "round-robin": - return self.get_round_robin_lora_request( + return BenchmarkDataset.get_round_robin_lora_request( index=index, max_loras=max_loras, lora_path=lora_path ) - return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path) + return BenchmarkDataset.get_random_lora_request( + max_loras=max_loras, lora_path=lora_path + ) @abstractmethod def sample( @@ -2091,7 +2093,12 @@ def _parse_range_ratio(value: str) -> RangeRatio: return json.loads(value) -def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]: +def get_samples( + args, + tokenizer: TokenizerLike | None, + *, + multimodal_backends: tuple[str, ...] = ("openai-chat", "openai-audio"), +) -> list[SampleRequest]: if not hasattr(args, "request_id_prefix"): args.request_id_prefix = "" @@ -2310,15 +2317,17 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]: "like to add support for additional dataset formats." ) - if dataset_class.IS_MULTIMODAL and not ( - args.backend in ("openai-chat", "openai-audio") - or "embeddings-" in args.backend + if ( + dataset_class.IS_MULTIMODAL + and args.backend not in multimodal_backends + and "embeddings-" not in args.backend ): - # multi-modal benchmark is only available on OpenAI Chat - # endpoint-type. + # multi-modal benchmark is only available on chat-style backends; + # which ones are allowed is caller-controlled (serve uses the + # OpenAI endpoints, throughput uses vllm-chat). raise ValueError( - "Multi-modal content is only supported on 'openai-chat' and " - "'openai-audio' backends." + f"Multi-modal content is not supported on backend " + f"{args.backend!r}; use one of {sorted(multimodal_backends)}." ) assert tokenizer is not None, ( "Tokenizer must be initialized for the 'hf' dataset." @@ -2474,10 +2483,13 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]: try: # Enforce endpoint compatibility for multimodal datasets. - if args.dataset_name == "random-mm" and args.backend not in ["openai-chat"]: + if ( + args.dataset_name == "random-mm" + and args.backend not in multimodal_backends + ): raise ValueError( - "Multi-modal content (images) is only supported on " - "'openai-chat' backend." + f"Multi-modal content (images) is not supported on backend " + f"{args.backend!r}; use one of {sorted(multimodal_backends)}." ) input_requests = dataset_mapping[args.dataset_name]() except KeyError as err: diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py index f6264f955f..894334a7cf 100644 --- a/vllm/benchmarks/throughput.py +++ b/vllm/benchmarks/throughput.py @@ -16,24 +16,11 @@ from tqdm import tqdm from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase from vllm.benchmarks.datasets import ( - AIMODataset, - ASRDataset, BenchmarkDataset, - BurstGPTDataset, - ConversationDataset, - CustomAudioDataset, - InstructCoderDataset, - MultiModalConversationDataset, - PrefixRepetitionRandomDataset, - RandomDataset, - RandomDatasetForReranking, - RandomMultiModalDataset, SampleRequest, - ShareGPTDataset, - SonnetDataset, - VisionArenaDataset, add_random_dataset_base_args, add_random_multimodal_dataset_args, + get_samples, ) from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs @@ -536,181 +523,56 @@ def save_to_pytorch_benchmark_format( write_to_json(pt_file, pt_records) -def get_requests(args, tokenizer): - # Common parameters for all dataset types. - dataset_cls: type[BenchmarkDataset] - common_kwargs = { - "dataset_path": args.dataset_path, - "random_seed": args.seed, - } - sample_kwargs = { - "tokenizer": tokenizer, - "lora_path": args.lora_path, - "max_loras": args.max_loras, - "lora_assignment": getattr(args, "lora_assignment", "random"), - "num_requests": args.num_prompts, - "no_oversample": getattr(args, "no_oversample", False), - } +def _to_serve_args(args) -> argparse.Namespace: + """Translate throughput args into a namespace the shared get_samples reads. - if args.dataset_name == "random" or ( - args.dataset_path is None - and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"} - ): - sample_kwargs["range_ratio"] = args.random_range_ratio - # prefer random_* arguments, fall back to regular arguments - random_prefix_len = getattr(args, "random_prefix_len", None) - sample_kwargs["prefix_len"] = ( - random_prefix_len if random_prefix_len is not None else args.prefix_len - ) - random_input_len = getattr(args, "random_input_len", None) - sample_kwargs["input_len"] = ( - random_input_len if random_input_len is not None else args.input_len - ) - random_output_len = getattr(args, "random_output_len", None) - sample_kwargs["output_len"] = ( - random_output_len if random_output_len is not None else args.output_len - ) - dataset_cls = RandomDataset - elif args.dataset_name == "sharegpt": - dataset_cls = ShareGPTDataset - if args.backend == "vllm-chat": - sample_kwargs["enable_multimodal_chat"] = True - if args.output_len is not None: - sample_kwargs["output_len"] = args.output_len - elif args.dataset_name == "sonnet": - assert tokenizer.chat_template or tokenizer.default_chat_template, ( - "Tokenizer/model must have chat template for sonnet dataset." - ) - dataset_cls = SonnetDataset - sample_kwargs["prefix_len"] = args.prefix_len - sample_kwargs["return_prompt_formatted"] = True - if args.input_len is not None: - sample_kwargs["input_len"] = args.input_len - if args.output_len is not None: - sample_kwargs["output_len"] = args.output_len - elif args.dataset_name == "burstgpt": - dataset_cls = BurstGPTDataset - elif args.dataset_name == "custom_audio": - dataset_cls = CustomAudioDataset - sample_kwargs["enable_multimodal_chat"] = getattr( - args, "enable_multimodal_chat", False - ) - custom_output_len = getattr(args, "custom_output_len", None) - if custom_output_len is not None: - sample_kwargs["output_len"] = custom_output_len - elif args.output_len is not None: - sample_kwargs["output_len"] = args.output_len - elif args.dataset_name == "hf": - if args.output_len is not None: - sample_kwargs["output_len"] = args.output_len - common_kwargs["hf_name"] = args.hf_name - if ( - args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS - or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = VisionArenaDataset - common_kwargs["dataset_subset"] = None - common_kwargs["dataset_split"] = "train" - sample_kwargs["enable_multimodal_chat"] = True - elif ( - args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS - or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = InstructCoderDataset - common_kwargs["dataset_split"] = "train" - elif ( - args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS - or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = MultiModalConversationDataset - common_kwargs["dataset_subset"] = args.hf_subset - common_kwargs["dataset_split"] = args.hf_split - sample_kwargs["enable_multimodal_chat"] = True - elif ( - args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS - or args.hf_name in ConversationDataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = ConversationDataset - common_kwargs["dataset_subset"] = args.hf_subset - common_kwargs["dataset_split"] = args.hf_split - sample_kwargs["enable_multimodal_chat"] = True - elif ( - args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS - or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = AIMODataset - common_kwargs["dataset_subset"] = None - common_kwargs["dataset_split"] = "train" - elif ( - args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS - or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS - ): - dataset_cls = ASRDataset - common_kwargs["dataset_subset"] = args.hf_subset - common_kwargs["dataset_split"] = args.hf_split - sample_kwargs["asr_min_audio_len_sec"] = args.asr_min_audio_len_sec - sample_kwargs["asr_max_audio_len_sec"] = args.asr_max_audio_len_sec - elif args.dataset_name == "prefix_repetition": - dataset_cls = PrefixRepetitionRandomDataset - sample_kwargs["prefix_len"] = args.prefix_repetition_prefix_len - sample_kwargs["suffix_len"] = args.prefix_repetition_suffix_len - sample_kwargs["num_prefixes"] = args.prefix_repetition_num_prefixes - sample_kwargs["output_len"] = args.prefix_repetition_output_len - elif args.dataset_name == "random-mm": - dataset_cls = RandomMultiModalDataset - # prefer random_* arguments, fall back to regular arguments - random_input_len = getattr(args, "random_input_len", None) - sample_kwargs["input_len"] = ( - random_input_len - if random_input_len is not None - else getattr(args, "input_len", None) - ) - random_output_len = getattr(args, "random_output_len", None) - sample_kwargs["output_len"] = ( - random_output_len - if random_output_len is not None - else getattr(args, "output_len", None) - ) - sample_kwargs["base_items_per_request"] = getattr( - args, "random_mm_base_items_per_request", None - ) - sample_kwargs["num_mm_items_range_ratio"] = getattr( - args, "random_mm_num_mm_items_range_ratio", None - ) - sample_kwargs["limit_mm_per_prompt"] = getattr( - args, "random_mm_limit_mm_per_prompt", None - ) - sample_kwargs["bucket_config"] = getattr(args, "random_mm_bucket_config", None) - sample_kwargs["enable_multimodal_chat"] = True - random_prefix_len = getattr(args, "random_prefix_len", None) - prefix_len = getattr(args, "prefix_len", None) - sample_kwargs["prefix_len"] = ( - random_prefix_len if random_prefix_len is not None else prefix_len - ) - sample_kwargs["range_ratio"] = args.random_range_ratio - elif args.dataset_name == "random-rerank": - dataset_cls = RandomDatasetForReranking - # prefer random_* arguments, fall back to regular arguments - random_input_len = getattr(args, "random_input_len", None) - sample_kwargs["input_len"] = ( - random_input_len - if random_input_len is not None - else getattr(args, "input_len", None) - ) - random_output_len = getattr(args, "random_output_len", None) - sample_kwargs["output_len"] = ( - random_output_len - if random_output_len is not None - else getattr(args, "output_len", None) - ) - sample_kwargs["batchsize"] = getattr(args, "random_batch_size", 1) - sample_kwargs["is_reranker"] = not getattr(args, "no_reranker", False) - sample_kwargs["range_ratio"] = args.random_range_ratio - else: - raise ValueError(f"Unknown dataset name: {args.dataset_name}") - # Remove None values - sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None} - requests = dataset_cls(**common_kwargs).sample(**sample_kwargs) + ``get_samples`` (used by ``bench serve``) expects ~45 attributes; the + throughput CLI exposes most of them under the same names, and this adapter + fills the rest while preserving throughput's existing flag names so no + script breaks. + + Args: + args: parsed throughput CLI namespace. + + Returns: + A namespace satisfying get_samples's attribute reads. + """ + d = vars(args).copy() + # random_*: prefer --random-* over legacy --input/output/prefix-len. + d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len + d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len + d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len + # --output-len maps to the per-dataset output-len entry points get_samples + # reads. None passes through (each dataset applies its own default), + # matching prior throughput behaviour of omitting output_len when unset. + d["hf_output_len"] = args.output_len + d["sharegpt_output_len"] = args.output_len + # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults. + d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550 + d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150 + d["sonnet_prefix_len"] = args.prefix_len + # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the + # multimodal chat backend (preserves today's vllm-chat handling). + d["enable_multimodal_chat"] = bool( + getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat" + ) + # serve-only attrs throughput never exposed; keep serve's defaults. + d.setdefault("disable_shuffle", False) + d.setdefault("skip_chat_template", False) + d.setdefault("no_stream", False) + d.setdefault("request_id_prefix", "") + d.setdefault("chat_template_kwargs", None) + return argparse.Namespace(**d) + + +def get_requests(args, tokenizer): + serve_args = _to_serve_args(args) + # Throughput reuses bench serve's dataset dispatch. vllm-chat is the only + # multimodal-capable throughput backend, so it is the only one allowed past + # the multimodal gates inside get_samples; other backends raise there. + mm_backends = ("vllm-chat",) if args.backend == "vllm-chat" else () + requests = get_samples(serve_args, tokenizer, multimodal_backends=mm_backends) + requests = assign_loras(requests, args) requests = filter_requests_for_dp(requests, args.data_parallel_size) return requests @@ -732,6 +594,28 @@ def filter_requests_for_dp(requests, data_parallel_size): ] +def assign_loras(requests, args): + """Attach a LoRA request to each sample (throughput-only post-processing). + + The shared ``datasets.get_samples`` path carries no LoRA information, so + LoRA assignment is applied here, uniformly across every dataset type. No-op + when ``--lora-path`` is unset. + """ + lora_path = getattr(args, "lora_path", None) + if not lora_path: + return requests + max_loras = args.max_loras + lora_assignment = getattr(args, "lora_assignment", "random") + for i, req in enumerate(requests): + req.lora_request = BenchmarkDataset.get_lora_request( + index=i, + max_loras=max_loras, + lora_path=lora_path, + lora_assignment=lora_assignment, + ) + return requests + + def validate_args(args): """ Validate command-line arguments. @@ -785,33 +669,6 @@ def validate_args(args): since --dataset-name is not 'hf'.", stacklevel=2, ) - elif args.dataset_name == "hf": - if args.dataset_path in ( - VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys() - | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS - | ConversationDataset.SUPPORTED_DATASET_PATHS - ) or args.hf_name in ( - VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys() - | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS - | ConversationDataset.SUPPORTED_DATASET_PATHS - ): - assert args.backend == "vllm-chat", ( - f"{args.dataset_path} needs to use vllm-chat as the backend." - ) - elif args.dataset_path in ( - InstructCoderDataset.SUPPORTED_DATASET_PATHS - | AIMODataset.SUPPORTED_DATASET_PATHS - | ASRDataset.SUPPORTED_DATASET_PATHS - ) or args.hf_name in ( - InstructCoderDataset.SUPPORTED_DATASET_PATHS - | AIMODataset.SUPPORTED_DATASET_PATHS - | ASRDataset.SUPPORTED_DATASET_PATHS - ): - assert args.backend == "vllm", ( - f"{args.dataset_path} needs to use vllm as the backend." - ) - else: - raise ValueError(f"{args.dataset_path} is not supported by hf dataset.") # --random-range-ratio: only used when dataset_name is 'random', # 'random-mm', or 'random-rerank'