[MISC][Bench] refactor throughput and reuse serve's get samples (#50981)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
@@ -1,13 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ``vllm bench throughput`` CLI.
|
||||
|
||||
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
|
||||
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
|
||||
(serve has no analogue), keeping the two benchmarks aligned without duplicating
|
||||
serve-side dataset coverage.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
from vllm.benchmarks.throughput import (
|
||||
_run_vllm_chat_requests,
|
||||
add_cli_args,
|
||||
assign_loras,
|
||||
get_requests,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
@@ -102,3 +114,119 @@ def test_vllm_chat_requests_include_multimodal_content():
|
||||
}
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Small, commonly available tokenizer.
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# LoRA assignment (throughput-only post-processing)
|
||||
# -----------------------------
|
||||
|
||||
|
||||
def _sr(i: int) -> SampleRequest:
|
||||
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
|
||||
|
||||
|
||||
def _lora_args(
|
||||
*,
|
||||
lora_path: str | None,
|
||||
max_loras: int,
|
||||
lora_assignment: str = "random",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
lora_path=lora_path,
|
||||
max_loras=max_loras,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_noop_without_lora_path() -> None:
|
||||
"""No lora_path -> requests returned unchanged, no LoRA attached."""
|
||||
reqs = [_sr(i) for i in range(5)]
|
||||
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
|
||||
assert out is reqs
|
||||
assert all(r.lora_request is None for r in out)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_round_robin() -> None:
|
||||
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
|
||||
reqs = [_sr(i) for i in range(6)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
|
||||
)
|
||||
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_random_in_range() -> None:
|
||||
"""Random assignment stays in [1, max_loras] and covers all IDs."""
|
||||
reqs = [_sr(i) for i in range(200)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
|
||||
)
|
||||
ids = [r.lora_request.lora_int_id for r in out]
|
||||
assert all(1 <= i <= 5 for i in ids)
|
||||
assert len(set(ids)) == 5
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Issue #50838: MMVU via throughput
|
||||
# -----------------------------
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_resolves_mmvu(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
|
||||
|
||||
Previously throughput's validate_args rejected MMVU ("is not supported by
|
||||
hf dataset"). It now resolves through the shared get_samples. The HF
|
||||
download is stubbed so the resolution + dispatch path is exercised without
|
||||
network access.
|
||||
"""
|
||||
import vllm.benchmarks.datasets.datasets as dsmod
|
||||
|
||||
class _StubMMVUDataset:
|
||||
# MMVU is multimodal in content but, like the real class, does not set
|
||||
# IS_MULTIMODAL -- so it is not subject to the backend gate.
|
||||
IS_MULTIMODAL = False
|
||||
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
|
||||
return [
|
||||
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"yale-nlp/MMVU",
|
||||
"--backend",
|
||||
"vllm-chat",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
|
||||
@@ -164,8 +164,8 @@ class BenchmarkDataset(ABC):
|
||||
# TODO (jenniferzhao): add support for downloading data
|
||||
raise NotImplementedError("load_data must be implemented in subclasses.")
|
||||
|
||||
@staticmethod
|
||||
def get_random_lora_request(
|
||||
self,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
) -> LoRARequest | None:
|
||||
@@ -197,8 +197,8 @@ class BenchmarkDataset(ABC):
|
||||
)
|
||||
return lora_request
|
||||
|
||||
@staticmethod
|
||||
def get_round_robin_lora_request(
|
||||
self,
|
||||
index: int,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
@@ -232,8 +232,8 @@ class BenchmarkDataset(ABC):
|
||||
)
|
||||
return lora_request
|
||||
|
||||
@staticmethod
|
||||
def get_lora_request(
|
||||
self,
|
||||
index: int,
|
||||
max_loras: int | None = None,
|
||||
lora_path: str | None = None,
|
||||
@@ -254,10 +254,12 @@ class BenchmarkDataset(ABC):
|
||||
(or `None` if not applicable).
|
||||
"""
|
||||
if lora_assignment == "round-robin":
|
||||
return self.get_round_robin_lora_request(
|
||||
return BenchmarkDataset.get_round_robin_lora_request(
|
||||
index=index, max_loras=max_loras, lora_path=lora_path
|
||||
)
|
||||
return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path)
|
||||
return BenchmarkDataset.get_random_lora_request(
|
||||
max_loras=max_loras, lora_path=lora_path
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def sample(
|
||||
@@ -2091,7 +2093,12 @@ def _parse_range_ratio(value: str) -> RangeRatio:
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
|
||||
def get_samples(
|
||||
args,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
multimodal_backends: tuple[str, ...] = ("openai-chat", "openai-audio"),
|
||||
) -> list[SampleRequest]:
|
||||
if not hasattr(args, "request_id_prefix"):
|
||||
args.request_id_prefix = ""
|
||||
|
||||
@@ -2310,15 +2317,17 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
|
||||
"like to add support for additional dataset formats."
|
||||
)
|
||||
|
||||
if dataset_class.IS_MULTIMODAL and not (
|
||||
args.backend in ("openai-chat", "openai-audio")
|
||||
or "embeddings-" in args.backend
|
||||
if (
|
||||
dataset_class.IS_MULTIMODAL
|
||||
and args.backend not in multimodal_backends
|
||||
and "embeddings-" not in args.backend
|
||||
):
|
||||
# multi-modal benchmark is only available on OpenAI Chat
|
||||
# endpoint-type.
|
||||
# multi-modal benchmark is only available on chat-style backends;
|
||||
# which ones are allowed is caller-controlled (serve uses the
|
||||
# OpenAI endpoints, throughput uses vllm-chat).
|
||||
raise ValueError(
|
||||
"Multi-modal content is only supported on 'openai-chat' and "
|
||||
"'openai-audio' backends."
|
||||
f"Multi-modal content is not supported on backend "
|
||||
f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
|
||||
)
|
||||
assert tokenizer is not None, (
|
||||
"Tokenizer must be initialized for the 'hf' dataset."
|
||||
@@ -2474,10 +2483,13 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
|
||||
|
||||
try:
|
||||
# Enforce endpoint compatibility for multimodal datasets.
|
||||
if args.dataset_name == "random-mm" and args.backend not in ["openai-chat"]:
|
||||
if (
|
||||
args.dataset_name == "random-mm"
|
||||
and args.backend not in multimodal_backends
|
||||
):
|
||||
raise ValueError(
|
||||
"Multi-modal content (images) is only supported on "
|
||||
"'openai-chat' backend."
|
||||
f"Multi-modal content (images) is not supported on backend "
|
||||
f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
|
||||
)
|
||||
input_requests = dataset_mapping[args.dataset_name]()
|
||||
except KeyError as err:
|
||||
|
||||
+72
-215
@@ -16,24 +16,11 @@ from tqdm import tqdm
|
||||
from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
AIMODataset,
|
||||
ASRDataset,
|
||||
BenchmarkDataset,
|
||||
BurstGPTDataset,
|
||||
ConversationDataset,
|
||||
CustomAudioDataset,
|
||||
InstructCoderDataset,
|
||||
MultiModalConversationDataset,
|
||||
PrefixRepetitionRandomDataset,
|
||||
RandomDataset,
|
||||
RandomDatasetForReranking,
|
||||
RandomMultiModalDataset,
|
||||
SampleRequest,
|
||||
ShareGPTDataset,
|
||||
SonnetDataset,
|
||||
VisionArenaDataset,
|
||||
add_random_dataset_base_args,
|
||||
add_random_multimodal_dataset_args,
|
||||
get_samples,
|
||||
)
|
||||
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
@@ -536,181 +523,56 @@ def save_to_pytorch_benchmark_format(
|
||||
write_to_json(pt_file, pt_records)
|
||||
|
||||
|
||||
def get_requests(args, tokenizer):
|
||||
# Common parameters for all dataset types.
|
||||
dataset_cls: type[BenchmarkDataset]
|
||||
common_kwargs = {
|
||||
"dataset_path": args.dataset_path,
|
||||
"random_seed": args.seed,
|
||||
}
|
||||
sample_kwargs = {
|
||||
"tokenizer": tokenizer,
|
||||
"lora_path": args.lora_path,
|
||||
"max_loras": args.max_loras,
|
||||
"lora_assignment": getattr(args, "lora_assignment", "random"),
|
||||
"num_requests": args.num_prompts,
|
||||
"no_oversample": getattr(args, "no_oversample", False),
|
||||
}
|
||||
def _to_serve_args(args) -> argparse.Namespace:
|
||||
"""Translate throughput args into a namespace the shared get_samples reads.
|
||||
|
||||
if args.dataset_name == "random" or (
|
||||
args.dataset_path is None
|
||||
and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"}
|
||||
):
|
||||
sample_kwargs["range_ratio"] = args.random_range_ratio
|
||||
# prefer random_* arguments, fall back to regular arguments
|
||||
random_prefix_len = getattr(args, "random_prefix_len", None)
|
||||
sample_kwargs["prefix_len"] = (
|
||||
random_prefix_len if random_prefix_len is not None else args.prefix_len
|
||||
)
|
||||
random_input_len = getattr(args, "random_input_len", None)
|
||||
sample_kwargs["input_len"] = (
|
||||
random_input_len if random_input_len is not None else args.input_len
|
||||
)
|
||||
random_output_len = getattr(args, "random_output_len", None)
|
||||
sample_kwargs["output_len"] = (
|
||||
random_output_len if random_output_len is not None else args.output_len
|
||||
)
|
||||
dataset_cls = RandomDataset
|
||||
elif args.dataset_name == "sharegpt":
|
||||
dataset_cls = ShareGPTDataset
|
||||
if args.backend == "vllm-chat":
|
||||
sample_kwargs["enable_multimodal_chat"] = True
|
||||
if args.output_len is not None:
|
||||
sample_kwargs["output_len"] = args.output_len
|
||||
elif args.dataset_name == "sonnet":
|
||||
assert tokenizer.chat_template or tokenizer.default_chat_template, (
|
||||
"Tokenizer/model must have chat template for sonnet dataset."
|
||||
)
|
||||
dataset_cls = SonnetDataset
|
||||
sample_kwargs["prefix_len"] = args.prefix_len
|
||||
sample_kwargs["return_prompt_formatted"] = True
|
||||
if args.input_len is not None:
|
||||
sample_kwargs["input_len"] = args.input_len
|
||||
if args.output_len is not None:
|
||||
sample_kwargs["output_len"] = args.output_len
|
||||
elif args.dataset_name == "burstgpt":
|
||||
dataset_cls = BurstGPTDataset
|
||||
elif args.dataset_name == "custom_audio":
|
||||
dataset_cls = CustomAudioDataset
|
||||
sample_kwargs["enable_multimodal_chat"] = getattr(
|
||||
args, "enable_multimodal_chat", False
|
||||
)
|
||||
custom_output_len = getattr(args, "custom_output_len", None)
|
||||
if custom_output_len is not None:
|
||||
sample_kwargs["output_len"] = custom_output_len
|
||||
elif args.output_len is not None:
|
||||
sample_kwargs["output_len"] = args.output_len
|
||||
elif args.dataset_name == "hf":
|
||||
if args.output_len is not None:
|
||||
sample_kwargs["output_len"] = args.output_len
|
||||
common_kwargs["hf_name"] = args.hf_name
|
||||
if (
|
||||
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = VisionArenaDataset
|
||||
common_kwargs["dataset_subset"] = None
|
||||
common_kwargs["dataset_split"] = "train"
|
||||
sample_kwargs["enable_multimodal_chat"] = True
|
||||
elif (
|
||||
args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = InstructCoderDataset
|
||||
common_kwargs["dataset_split"] = "train"
|
||||
elif (
|
||||
args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = MultiModalConversationDataset
|
||||
common_kwargs["dataset_subset"] = args.hf_subset
|
||||
common_kwargs["dataset_split"] = args.hf_split
|
||||
sample_kwargs["enable_multimodal_chat"] = True
|
||||
elif (
|
||||
args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in ConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = ConversationDataset
|
||||
common_kwargs["dataset_subset"] = args.hf_subset
|
||||
common_kwargs["dataset_split"] = args.hf_split
|
||||
sample_kwargs["enable_multimodal_chat"] = True
|
||||
elif (
|
||||
args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = AIMODataset
|
||||
common_kwargs["dataset_subset"] = None
|
||||
common_kwargs["dataset_split"] = "train"
|
||||
elif (
|
||||
args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS
|
||||
or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
dataset_cls = ASRDataset
|
||||
common_kwargs["dataset_subset"] = args.hf_subset
|
||||
common_kwargs["dataset_split"] = args.hf_split
|
||||
sample_kwargs["asr_min_audio_len_sec"] = args.asr_min_audio_len_sec
|
||||
sample_kwargs["asr_max_audio_len_sec"] = args.asr_max_audio_len_sec
|
||||
elif args.dataset_name == "prefix_repetition":
|
||||
dataset_cls = PrefixRepetitionRandomDataset
|
||||
sample_kwargs["prefix_len"] = args.prefix_repetition_prefix_len
|
||||
sample_kwargs["suffix_len"] = args.prefix_repetition_suffix_len
|
||||
sample_kwargs["num_prefixes"] = args.prefix_repetition_num_prefixes
|
||||
sample_kwargs["output_len"] = args.prefix_repetition_output_len
|
||||
elif args.dataset_name == "random-mm":
|
||||
dataset_cls = RandomMultiModalDataset
|
||||
# prefer random_* arguments, fall back to regular arguments
|
||||
random_input_len = getattr(args, "random_input_len", None)
|
||||
sample_kwargs["input_len"] = (
|
||||
random_input_len
|
||||
if random_input_len is not None
|
||||
else getattr(args, "input_len", None)
|
||||
)
|
||||
random_output_len = getattr(args, "random_output_len", None)
|
||||
sample_kwargs["output_len"] = (
|
||||
random_output_len
|
||||
if random_output_len is not None
|
||||
else getattr(args, "output_len", None)
|
||||
)
|
||||
sample_kwargs["base_items_per_request"] = getattr(
|
||||
args, "random_mm_base_items_per_request", None
|
||||
)
|
||||
sample_kwargs["num_mm_items_range_ratio"] = getattr(
|
||||
args, "random_mm_num_mm_items_range_ratio", None
|
||||
)
|
||||
sample_kwargs["limit_mm_per_prompt"] = getattr(
|
||||
args, "random_mm_limit_mm_per_prompt", None
|
||||
)
|
||||
sample_kwargs["bucket_config"] = getattr(args, "random_mm_bucket_config", None)
|
||||
sample_kwargs["enable_multimodal_chat"] = True
|
||||
random_prefix_len = getattr(args, "random_prefix_len", None)
|
||||
prefix_len = getattr(args, "prefix_len", None)
|
||||
sample_kwargs["prefix_len"] = (
|
||||
random_prefix_len if random_prefix_len is not None else prefix_len
|
||||
)
|
||||
sample_kwargs["range_ratio"] = args.random_range_ratio
|
||||
elif args.dataset_name == "random-rerank":
|
||||
dataset_cls = RandomDatasetForReranking
|
||||
# prefer random_* arguments, fall back to regular arguments
|
||||
random_input_len = getattr(args, "random_input_len", None)
|
||||
sample_kwargs["input_len"] = (
|
||||
random_input_len
|
||||
if random_input_len is not None
|
||||
else getattr(args, "input_len", None)
|
||||
)
|
||||
random_output_len = getattr(args, "random_output_len", None)
|
||||
sample_kwargs["output_len"] = (
|
||||
random_output_len
|
||||
if random_output_len is not None
|
||||
else getattr(args, "output_len", None)
|
||||
)
|
||||
sample_kwargs["batchsize"] = getattr(args, "random_batch_size", 1)
|
||||
sample_kwargs["is_reranker"] = not getattr(args, "no_reranker", False)
|
||||
sample_kwargs["range_ratio"] = args.random_range_ratio
|
||||
else:
|
||||
raise ValueError(f"Unknown dataset name: {args.dataset_name}")
|
||||
# Remove None values
|
||||
sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None}
|
||||
requests = dataset_cls(**common_kwargs).sample(**sample_kwargs)
|
||||
``get_samples`` (used by ``bench serve``) expects ~45 attributes; the
|
||||
throughput CLI exposes most of them under the same names, and this adapter
|
||||
fills the rest while preserving throughput's existing flag names so no
|
||||
script breaks.
|
||||
|
||||
Args:
|
||||
args: parsed throughput CLI namespace.
|
||||
|
||||
Returns:
|
||||
A namespace satisfying get_samples's attribute reads.
|
||||
"""
|
||||
d = vars(args).copy()
|
||||
# random_*: prefer --random-* over legacy --input/output/prefix-len.
|
||||
d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len
|
||||
d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len
|
||||
d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len
|
||||
# --output-len maps to the per-dataset output-len entry points get_samples
|
||||
# reads. None passes through (each dataset applies its own default),
|
||||
# matching prior throughput behaviour of omitting output_len when unset.
|
||||
d["hf_output_len"] = args.output_len
|
||||
d["sharegpt_output_len"] = args.output_len
|
||||
# sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
|
||||
d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550
|
||||
d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150
|
||||
d["sonnet_prefix_len"] = args.prefix_len
|
||||
# Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
|
||||
# multimodal chat backend (preserves today's vllm-chat handling).
|
||||
d["enable_multimodal_chat"] = bool(
|
||||
getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat"
|
||||
)
|
||||
# serve-only attrs throughput never exposed; keep serve's defaults.
|
||||
d.setdefault("disable_shuffle", False)
|
||||
d.setdefault("skip_chat_template", False)
|
||||
d.setdefault("no_stream", False)
|
||||
d.setdefault("request_id_prefix", "")
|
||||
d.setdefault("chat_template_kwargs", None)
|
||||
return argparse.Namespace(**d)
|
||||
|
||||
|
||||
def get_requests(args, tokenizer):
|
||||
serve_args = _to_serve_args(args)
|
||||
# Throughput reuses bench serve's dataset dispatch. vllm-chat is the only
|
||||
# multimodal-capable throughput backend, so it is the only one allowed past
|
||||
# the multimodal gates inside get_samples; other backends raise there.
|
||||
mm_backends = ("vllm-chat",) if args.backend == "vllm-chat" else ()
|
||||
requests = get_samples(serve_args, tokenizer, multimodal_backends=mm_backends)
|
||||
requests = assign_loras(requests, args)
|
||||
requests = filter_requests_for_dp(requests, args.data_parallel_size)
|
||||
return requests
|
||||
|
||||
@@ -732,6 +594,28 @@ def filter_requests_for_dp(requests, data_parallel_size):
|
||||
]
|
||||
|
||||
|
||||
def assign_loras(requests, args):
|
||||
"""Attach a LoRA request to each sample (throughput-only post-processing).
|
||||
|
||||
The shared ``datasets.get_samples`` path carries no LoRA information, so
|
||||
LoRA assignment is applied here, uniformly across every dataset type. No-op
|
||||
when ``--lora-path`` is unset.
|
||||
"""
|
||||
lora_path = getattr(args, "lora_path", None)
|
||||
if not lora_path:
|
||||
return requests
|
||||
max_loras = args.max_loras
|
||||
lora_assignment = getattr(args, "lora_assignment", "random")
|
||||
for i, req in enumerate(requests):
|
||||
req.lora_request = BenchmarkDataset.get_lora_request(
|
||||
index=i,
|
||||
max_loras=max_loras,
|
||||
lora_path=lora_path,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
return requests
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
"""
|
||||
Validate command-line arguments.
|
||||
@@ -785,33 +669,6 @@ def validate_args(args):
|
||||
since --dataset-name is not 'hf'.",
|
||||
stacklevel=2,
|
||||
)
|
||||
elif args.dataset_name == "hf":
|
||||
if args.dataset_path in (
|
||||
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
|
||||
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
| ConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
) or args.hf_name in (
|
||||
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
|
||||
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
| ConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
assert args.backend == "vllm-chat", (
|
||||
f"{args.dataset_path} needs to use vllm-chat as the backend."
|
||||
)
|
||||
elif args.dataset_path in (
|
||||
InstructCoderDataset.SUPPORTED_DATASET_PATHS
|
||||
| AIMODataset.SUPPORTED_DATASET_PATHS
|
||||
| ASRDataset.SUPPORTED_DATASET_PATHS
|
||||
) or args.hf_name in (
|
||||
InstructCoderDataset.SUPPORTED_DATASET_PATHS
|
||||
| AIMODataset.SUPPORTED_DATASET_PATHS
|
||||
| ASRDataset.SUPPORTED_DATASET_PATHS
|
||||
):
|
||||
assert args.backend == "vllm", (
|
||||
f"{args.dataset_path} needs to use vllm as the backend."
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")
|
||||
|
||||
# --random-range-ratio: only used when dataset_name is 'random',
|
||||
# 'random-mm', or 'random-rerank'
|
||||
|
||||
Reference in New Issue
Block a user