iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[MyPy] Fix mypy for vllm/benchmarks (#39896)

Browse Source 
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Martin Hickey
2026-06-23 15:22:29 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 547d2c40d7
commit 275b43183c
24 changed files with 297 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/pre_commit/mypy.py
-2
View File
@@ -31,8 +31,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [
"vllm/model_executor/models",
"vllm/model_executor/layers/fla/ops",
# TODO: Remove these entries after fixing mypy errors.
"vllm/benchmarks",
]
vllm/benchmarks/datasets/create_txt_slices_dataset.py
+2 -2
View File
@@ -30,7 +30,6 @@ The resulting JSONL file can then be used with the serving benchmark::
from __future__ import annotations
import argparse
import json
import logging
import random
@@ -40,6 +39,7 @@ import numpy as np
from transformers import AutoTokenizer
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = logging.getLogger(__name__)
@@ -121,7 +121,7 @@ def create_txt_slices_jsonl(
def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser(
parser = FlexibleArgumentParser(
description="Convert a plain-text file into a JSONL dataset "
"for CustomDataset (--dataset-name custom).",
)
vllm/benchmarks/datasets/datasets.py
+141 -64
View File
@@ -81,7 +81,7 @@ class SampleRequest:
prompt: str | list[str] | list[dict]
prompt_len: int
expected_output_len: int | None
expected_output_len: int = 0
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
lora_request: LoRARequest | None = None
request_id: str | None = None
@@ -634,7 +634,7 @@ class RandomDataset(BenchmarkDataset):
# Generate prefix once
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
requests = []
requests: list[SampleRequest] = []
token_mismatch_total = 0
for i in range(num_requests):
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
@@ -665,15 +665,14 @@ class RandomDataset(BenchmarkDataset):
)
# only used for embeddings benchmark.
if batchsize > 1:
batch_requests = []
batch_requests: list[SampleRequest] = []
# Create batched requests
for i in range(0, num_requests, batchsize):
batch = requests[i : i + batchsize]
batch_requests.append(
SampleRequest(
prompt=[req.prompt for req in batch],
prompt=[req.prompt for req in batch], # type: ignore[arg-type]
prompt_len=sum(req.prompt_len for req in batch),
expected_output_len=0,
request_id=request_id_prefix + str(i // batchsize),
)
)
@@ -797,6 +796,9 @@ class RandomDatasetForReranking(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
is_reranker: bool = True,
**kwargs,
) -> list[SampleRequest]:
@@ -1207,6 +1209,9 @@ class RandomMultiModalDataset(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
@@ -1251,6 +1256,7 @@ class RandomMultiModalDataset(RandomDataset):
# We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it
# may break prompt replacement logic.
assert hasattr(tokenizer, "added_tokens_decoder")
prohibited_tokens = list(
tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items()
@@ -1376,6 +1382,7 @@ class ShareGPTDataset(BenchmarkDataset):
lora_assignment: str = "random",
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
samples: list[SampleRequest] = []
ind = 0
for entry in self.data:
@@ -1454,7 +1461,7 @@ class TimedTrace(BenchmarkDataset):
f'label_output_length: "{self.label_output_length}", '
f'label_hash_ids: "{self.label_hash_ids}"'
)
self._expanded_generated_prompts = {}
self._expanded_generated_prompts: dict[str, Any] = {}
self.load_data()
def load_data(self) -> None:
@@ -1529,10 +1536,12 @@ class TimedTrace(BenchmarkDataset):
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list:
samples: list = []
) -> list[SampleRequest]:
samples: list[SampleRequest] = []
assert tokenizer is not None, "Tokenizer must be provided, now is Null"
assert self.data is not None, "Data must be loaded before sampling"
for ind, entry in enumerate(self.data):
if len(samples) >= num_requests:
@@ -2139,12 +2148,12 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
)
elif args.dataset_name == "sonnet":
dataset = SonnetDataset(
sonnet_dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
if args.backend == "openai-chat":
input_requests = dataset.sample(
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
@@ -2155,10 +2164,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample,
)
else:
assert tokenizer.chat_template or tokenizer.default_chat_template, (
"Tokenizer/model must have chat template for sonnet dataset."
)
input_requests = dataset.sample(
assert (
hasattr(tokenizer, "chat_template") and tokenizer.chat_template
) or (
hasattr(tokenizer, "default_chat_template")
and tokenizer.default_chat_template
), "Tokenizer/model must have chat template for sonnet dataset."
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
@@ -2173,6 +2185,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
# all following datasets are implemented from the
# HuggingFaceDataset base class
hf_kwargs = {}
dataset_class: type[BenchmarkDataset]
if (
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
@@ -2548,23 +2561,23 @@ class CustomDataset(BenchmarkDataset):
if tokenizer is None:
new_output_len = 1
elif output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
else:
new_output_len = output_len
if output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
if tokenizer is None:
prompt_len = 1
@@ -2797,11 +2810,15 @@ class CustomImageDataset(CustomDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
ensure_client_side_data: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
ensure_client_side_data: bool = False,
**kwargs,
) -> list[SampleRequest]:
# load all data if needed
@@ -2814,7 +2831,7 @@ class CustomImageDataset(CustomDataset):
num_requests,
)
sampled_requests = []
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -2837,7 +2854,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
expected_output_len=output_len or 0,
multi_modal_data=None,
request_id=request_id_prefix + str(i),
)
@@ -2863,7 +2880,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
expected_output_len=output_len or 0,
multi_modal_data=mm_content,
request_id=request_id_prefix + str(i),
)
@@ -2891,17 +2908,20 @@ class CustomAudioDataset(CustomDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
skip_chat_template: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs,
) -> list[SampleRequest]:
self.num_available_samples = len(self.data)
if num_requests <= 0:
num_requests = self.num_available_samples
sampled_requests = []
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -2950,7 +2970,6 @@ class CustomAudioDataset(CustomDataset):
prompt_len = (
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
)
new_output_len = output_len
if output_len is None or output_len == -1:
if "output_tokens" not in item:
raise ValueError(
@@ -2958,6 +2977,8 @@ class CustomAudioDataset(CustomDataset):
"custom dataset must contain an 'output_tokens' field."
)
new_output_len = int(item["output_tokens"])
else:
new_output_len = output_len
sampled_requests.append(
SampleRequest(
prompt=prompt,
@@ -3015,10 +3036,30 @@ class SpecBench(CustomDataset):
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs,
) -> list[SampleRequest]:
# leverage CustomDataset sample
return super().sample(
tokenizer=tokenizer,
num_requests=num_requests,
request_id_prefix=request_id_prefix,
no_oversample=no_oversample,
lora_path=lora_path,
max_loras=max_loras,
output_len=output_len,
enable_multimodal_chat=enable_multimodal_chat,
skip_chat_template=skip_chat_template,
chat_template_kwargs=chat_template_kwargs,
**kwargs,
)
@@ -3067,16 +3108,21 @@ class SonnetDataset(BenchmarkDataset):
return_prompt_formatted: bool = False,
**kwargs,
) -> list[SampleRequest]:
poem_lines = self.data
assert poem_lines is not None
# Calculate average token length for a poem line.
tokenized_lines = [tokenizer(line).input_ids for line in self.data]
tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
# Build the base prompt.
base_prompt = "Pick as many lines as you can from these poem lines:\n"
base_msg = [{"role": "user", "content": base_prompt}]
base_fmt = tokenizer.apply_chat_template(
base_msg, add_generation_prompt=True, tokenize=False
base_msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(base_fmt, str)
base_offset = len(tokenizer(base_fmt).input_ids)
if input_len <= base_offset:
raise ValueError(
@@ -3085,26 +3131,31 @@ class SonnetDataset(BenchmarkDataset):
)
# Determine how many poem lines to use.
num_input_lines = round((input_len - base_offset) / avg_len)
num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
prefix_lines = self.data[:num_prefix_lines]
prefix_lines = poem_lines[:num_prefix_lines]
samples: list[SampleRequest] = []
ind = 0
while len(samples) < num_requests:
extra_lines = random.choices(
self.data, k=num_input_lines - num_prefix_lines
poem_lines, k=num_input_lines - num_prefix_lines
)
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
msg = [{"role": "user", "content": prompt}]
prompt_formatted = tokenizer.apply_chat_template(
msg, add_generation_prompt=True, tokenize=False
msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids)
if prompt_len <= input_len:
samples.append(
SampleRequest(
prompt=prompt_formatted if return_prompt_formatted else prompt,
prompt=(
prompt_formatted if return_prompt_formatted else prompt
),
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(ind),
@@ -3145,6 +3196,7 @@ class BurstGPTDataset(BenchmarkDataset):
self.data = gpt4_df
def _sample_loaded_data(self, num_requests: int) -> list:
assert self.data is not None, "Dataset must be loaded before sampling"
if num_requests <= len(self.data):
data = self.data.sample(n=num_requests, random_state=self.random_seed)
else:
@@ -3258,6 +3310,7 @@ class ConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = []
@@ -3318,6 +3371,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = []
@@ -3393,7 +3447,8 @@ class VisionArenaDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3465,7 +3520,8 @@ class MMVUDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3531,18 +3587,21 @@ class InstructCoderDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests: list[SampleRequest] = []
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
prompt_text = prompt
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
prompt_text_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_text_result, str)
prompt_text = prompt_text_result
prompt_len = len(tokenizer(prompt).input_ids)
prompt_len = len(tokenizer(prompt_text).input_ids)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt=prompt_text,
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(i),
@@ -3554,6 +3613,7 @@ class InstructCoderDataset(HuggingFaceDataset):
return sampled_requests
def sample_prompts(self, n: int) -> Iterator[str]:
assert self.data is not None, "Dataset must be loaded before sampling"
for item in self.data.take(n):
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
@@ -3594,6 +3654,7 @@ class MTBenchDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
@@ -3654,7 +3715,8 @@ class HumanEvalDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
@@ -3714,7 +3776,8 @@ class GSM8KDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
@@ -3772,17 +3835,18 @@ class BlazeditDataset(HuggingFaceDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
skip_chat_template: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
output_len: int | None = None,
skip_chat_template: bool = False,
min_distance: float = 0.0,
max_distance: float = 1.0,
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Dataset must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3810,11 +3874,13 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
prompt_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_result, str)
prompt = prompt_result
prompt_len = len(tokenizer(prompt).input_ids)
@@ -3858,6 +3924,7 @@ class AIMODataset(HuggingFaceDataset):
output_len: int | None = None,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
dynamic_output = output_len is None
@@ -3970,6 +4037,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
if formatting_prompt_func is None:
raise ValueError(f"Unsupported dataset path: {self.hf_name}")
assert self.data is not None, "Dataset must be loaded before sampling"
samples = []
for i, sample in enumerate(self.data):
sample = formatting_prompt_func(sample)
@@ -4061,12 +4129,14 @@ class ASRDataset(HuggingFaceDataset):
def _disable_audio_decode(self) -> None:
from datasets import Audio
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None:
local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
)
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.map(
lambda item: {
"audio": str(local_path_root / item["url"]),
@@ -4090,15 +4160,19 @@ class ASRDataset(HuggingFaceDataset):
else:
prompt = ""
prompt_len = len(tokenizer(prompt).input_ids)
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
skipped = 0
asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec")
asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec")
asr_min_audio_len_sec: float = float(kwargs.get("asr_min_audio_len_sec") or 0.0)
asr_max_audio_len_sec: float = float(
kwargs.get("asr_max_audio_len_sec") or float("inf")
)
durations = []
for item in self.data:
if len(sampled_requests) >= num_requests:
break
mm_content: dict[str, Any]
audio = item["audio"]
if (
isinstance(audio, dict)
@@ -4203,8 +4277,7 @@ class MLPerfDataset(HuggingFaceDataset):
output_len: int | None = None,
**kwargs,
) -> list[SampleRequest]:
# Force dynamic output length based on reference completion.
dynamic_output = output_len is None
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
@@ -4222,15 +4295,18 @@ class MLPerfDataset(HuggingFaceDataset):
{"role": "user", "content": question},
]
prompt_formatted = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, tokenize=False
messages, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids)
# Determine output length from reference answer tokens.
ref_out_len = len(
tokenizer(reference_answer, add_special_tokens=False).input_ids
)
expected_output_len = ref_out_len if dynamic_output else output_len
expected_output_len = ref_out_len if output_len is None else output_len
# Validate sequence lengths.
if not is_valid_sequence(prompt_len, expected_output_len):
@@ -4371,6 +4447,7 @@ class MMStarDataset(HuggingFaceDataset):
) -> list[SampleRequest]:
# If --hf-output-len is not set, use the default output length.
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for ind, item in enumerate(self.data):
@@ -4603,7 +4680,7 @@ class BFCLDataset(HuggingFaceDataset):
exc_info=True,
)
rendered = None
if rendered is not None:
if rendered is not None and isinstance(rendered, str):
prompt_len = len(tokenizer(rendered).input_ids)
else:
text = "\n".join(m.get("content", "") for m in messages)
vllm/benchmarks/latency.py
+6 -4
View File
@@ -13,8 +13,9 @@ from tqdm import tqdm
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.engine.arg_utils import EngineArgs
from vllm.inputs import PromptType
from vllm.inputs import TextPrompt, TokensPrompt
from vllm.sampling_params import BeamSearchParams
from vllm.utils.argparse_utils import FlexibleArgumentParser
def save_to_pytorch_benchmark_format(
@@ -30,7 +31,7 @@ def save_to_pytorch_benchmark_format(
write_to_json(pt_file, pt_records)
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument("--input-len", type=int, default=32)
parser.add_argument("--output-len", type=int, default=128)
parser.add_argument("--batch-size", type=int, default=8)
@@ -103,8 +104,9 @@ def main(args: argparse.Namespace):
dummy_prompt_token_ids = np.random.randint(
10000, size=(args.batch_size, args.input_len)
)
dummy_prompts: list[PromptType] = [
{"prompt_token_ids": batch} for batch in dummy_prompt_token_ids.tolist()
dummy_prompts: list[TokensPrompt | TextPrompt] = [
TokensPrompt(prompt_token_ids=batch)
for batch in dummy_prompt_token_ids.tolist()
]
def llm_generate():
vllm/benchmarks/lib/endpoint_request_func.py
+24 -20
View File
@@ -75,7 +75,7 @@ class RequestFuncInput:
logprobs: int | None = None
extra_headers: dict | None = None
extra_body: dict | None = None
multi_modal_content: dict | list[dict] | None = None
multi_modal_content: dict[str, Any] | list[dict[str, Any]] | None = None
ignore_eos: bool = False
language: str | None = None
request_id: str | None = None
@@ -272,13 +272,13 @@ def _get_chat_content(
request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]:
mm_contents = []
mm_contents: list[dict[str, Any]] = []
if request_func_input.multi_modal_content:
mm_content = request_func_input.multi_modal_content
if isinstance(mm_content, list):
mm_contents.extend(request_func_input.multi_modal_content)
mm_contents.extend(mm_content)
elif isinstance(mm_content, dict):
mm_contents.append(request_func_input.multi_modal_content)
mm_contents.append(mm_content)
else:
raise TypeError(
"multi_modal_content must be a dict or list[dict] for openai-chat"
@@ -293,10 +293,11 @@ def _get_chat_content(
for item in prompt
)
):
prompt_dicts: list[dict[str, Any]] = prompt # type: ignore[assignment]
if mm_position == "first":
return mm_contents + prompt
return mm_contents + prompt_dicts
return prompt + mm_contents
return prompt_dicts + mm_contents
text_contents = [{"type": "text", "text": prompt}]
@@ -307,15 +308,15 @@ def _get_chat_content(
def _is_chat_messages(prompt: Any) -> bool:
return (
isinstance(prompt, list)
and prompt
and all(
isinstance(item, dict)
and isinstance(item.get("role"), str)
and isinstance(item.get("content"), (str, list))
for item in prompt
)
if not isinstance(prompt, list):
return False
if not prompt:
return False
return all(
isinstance(item, dict)
and isinstance(item.get("role"), str)
and isinstance(item.get("content"), (str, list))
for item in prompt
)
@@ -325,7 +326,7 @@ def _get_chat_messages(
) -> list[dict[str, Any]]:
prompt = request_func_input.prompt
if _is_chat_messages(prompt):
return prompt
return prompt # type: ignore[return-value]
return [
{
@@ -385,8 +386,8 @@ async def async_request_openai_chat_completions(
if not chunk_bytes:
continue
messages = handler.add_chunk(chunk_bytes)
for message in messages:
message_strings = handler.add_chunk(chunk_bytes)
for message in message_strings:
# NOTE: SSE comments (often used as pings) start with
# a colon. These are not JSON data payload and should
# be skipped.
@@ -790,7 +791,7 @@ async def async_request_infinity_embeddings(
api_url = request_func_input.api_url
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
payload = {
payload: dict[str, Any] = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
@@ -849,7 +850,10 @@ async def async_request_vllm_pooling(
"truncate_prompt_tokens": -1,
}
payload = payload | request_func_input.prompt
if isinstance(request_func_input.prompt, dict):
payload = payload | request_func_input.prompt
else:
payload["input"] = request_func_input.prompt
_update_payload_common(payload, request_func_input)
vllm/benchmarks/lib/utils.py
+2 -2
View File
@@ -15,7 +15,7 @@ def extract_field(
if field_name in extra_info:
return extra_info[field_name]
v = args
v: Any = args
# For example, args.compilation_config.mode
for nested_field in field_name.split("."):
if not hasattr(v, nested_field):
@@ -43,7 +43,7 @@ def convert_to_pytorch_benchmark_format(
on metric per record
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
"""
records = []
records: list[Any] = []
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
return records
vllm/benchmarks/mm_processor.py
+3 -2
View File
@@ -28,6 +28,7 @@ from vllm.benchmarks.datasets import (
)
from vllm.benchmarks.throughput import get_requests
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.gc_utils import freeze_gc_heap
from vllm.utils.import_utils import PlaceholderModule
@@ -369,7 +370,7 @@ def benchmark_multimodal_processor(
return benchmark_result
def add_cli_args(parser: argparse.ArgumentParser) -> None:
def add_cli_args(parser: FlexibleArgumentParser) -> None:
"""Add CLI arguments for the multimodal processor benchmark."""
from vllm.engine.arg_utils import EngineArgs
@@ -532,7 +533,7 @@ def main(args: argparse.Namespace) -> None:
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Benchmark mm processor latency")
parser = FlexibleArgumentParser(description="Benchmark mm processor latency")
add_cli_args(parser)
args = parser.parse_args()
main(args)
vllm/benchmarks/serve.py
+37 -14
View File
@@ -30,7 +30,7 @@ import ssl
import time
import uuid
import warnings
from collections.abc import AsyncGenerator, Iterable
from collections.abc import AsyncGenerator, Iterable, Iterator
from dataclasses import dataclass, replace
from datetime import datetime
from enum import Enum
@@ -52,6 +52,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.gc_utils import freeze_gc_heap
from vllm.utils.network_utils import join_host_port
@@ -129,6 +130,7 @@ async def _align_prompts_to_server_tokenizer(
)
async def _fix_one(req: SampleRequest) -> SampleRequest:
assert isinstance(req.prompt, str)
tokens = await _tokenize(req.prompt)
if len(tokens) <= req.prompt_len:
return req
@@ -360,7 +362,7 @@ class EmbedBenchmarkMetrics:
mean_e2el_ms: float
std_e2el_ms: float
median_e2el_ms: float
percentiles_e2el_ms: float
percentiles_e2el_ms: list[tuple[float, float]]
def _get_current_request_rate(
@@ -433,8 +435,8 @@ async def get_request(
assert total_requests > 0, "No requests provided."
# Precompute delays among requests to minimize request send laggings
request_rates = []
delay_ts = []
request_rates: list[float] = []
delay_ts: list[float] = []
# if the traces have timing info then:
if not self_timed:
@@ -484,7 +486,10 @@ async def get_request(
else:
for request_index, request in enumerate(input_requests):
# this is cumulative running ts, from which sleep is calculated later
delay_ts.append(request.timestamp)
if request.timestamp is not None:
delay_ts.append(request.timestamp)
else:
delay_ts.append(0.0)
# TODO: there is no notion of RPS here, may be we can calculate
# from the trace.
request_rates.append(0.0)
@@ -599,7 +604,7 @@ def calculate_metrics(
)
actual_output_lens.append(output_len)
total_input += outputs[i].prompt_len
tpot = 0
tpot = 0.0
if output_len > 1:
latency_minus_ttft = outputs[i].latency - outputs[i].ttft
tpot = latency_minus_ttft / (output_len - 1)
@@ -894,11 +899,12 @@ async def benchmark(
print("Starting main benchmark run...")
lora_modules_iter: Iterator[str] | None = None
if lora_modules:
lora_modules_list = list(lora_modules)
if lora_assignment == "round-robin":
# Deterministic round-robin assignment across requests.
lora_modules = iter(
lora_modules_iter = iter(
[
lora_modules_list[i % len(lora_modules_list)]
for i in range(len(input_requests))
@@ -906,7 +912,7 @@ async def benchmark(
)
else:
# For each input request, choose a LoRA module at random.
lora_modules = iter(
lora_modules_iter = iter(
[random.choice(lora_modules_list) for _ in range(len(input_requests))]
)
@@ -1004,19 +1010,23 @@ async def benchmark(
)
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
req_model_id, req_model_name = model_id, model_name
if lora_modules:
req_lora_module = next(lora_modules)
if lora_modules_iter:
req_lora_module = next(lora_modules_iter)
req_model_id, req_model_name = req_lora_module, req_lora_module
mm_content_typed: dict[str, Any] | list[dict[str, Any]] | None = None
if isinstance(mm_content, (dict, list)):
mm_content_typed = mm_content
request_func_input = RequestFuncInput(
model=req_model_id,
model_name=req_model_name,
prompt=prompt,
api_url=api_url,
prompt_len=prompt_len,
output_len=output_len,
output_len=output_len or 0,
logprobs=logprobs,
multi_modal_content=mm_content,
multi_modal_content=mm_content_typed,
ignore_eos=ignore_eos,
extra_headers=extra_headers,
extra_body=per_request_extra_body,
@@ -1107,6 +1117,8 @@ async def benchmark(
"committed_per_step": delta_committed / denoising_steps,
}
metrics: BenchmarkMetrics | EmbedBenchmarkMetrics
actual_output_lens: list[int] | int
if task_type == TaskType.GENERATION:
metrics, actual_output_lens = calculate_metrics(
input_requests=input_requests,
@@ -1140,7 +1152,7 @@ async def benchmark(
"Request throughput (req/s):", metrics.request_throughput
)
)
if goodput_config_dict:
if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
print(
"{:<40} {:<10.2f}".format(
"Request goodput (req/s):", metrics.request_goodput
@@ -1177,6 +1189,7 @@ async def benchmark(
)
)
result: dict[str, Any]
if isinstance(metrics, BenchmarkMetrics):
result = {
"duration": benchmark_duration,
@@ -1462,7 +1475,7 @@ def compute_result_filename(
return file_name
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
add_dataset_parser(parser)
parser.add_argument(
"--label",
@@ -2023,6 +2036,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
args.self_timed = False
# Load the dataset.
assert tokenizer is not None, "Tokenizer must be initialized before loading dataset"
input_requests = get_samples(args, tokenizer)
if args.dataset_name in ("random", "prefix_repetition"):
@@ -2154,6 +2168,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Generate timeline plot if requested
if args.plot_timeline:
assert file_name is not None, (
"file_name must be set when plot_timeline is enabled"
)
try:
from vllm.benchmarks.plot import generate_timeline_plot
@@ -2200,6 +2217,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Generate dataset statistics plot if requested
if args.plot_dataset_stats:
assert file_name is not None, (
"file_name must be set when plot_dataset_stats is enabled"
)
try:
from vllm.benchmarks.plot import generate_dataset_stats_plot
@@ -2249,6 +2269,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Save to file
if args.save_result or args.append_result:
assert file_name is not None, (
"file_name must be set when save_result or append_result is enabled"
)
with open(
file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
) as outfile:
vllm/benchmarks/startup.py
+3 -2
View File
@@ -26,6 +26,7 @@ from vllm.benchmarks.lib.utils import (
write_to_json,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
PERCENTAGES = [10, 25, 50, 75, 90, 99]
@@ -190,7 +191,7 @@ def save_to_pytorch_benchmark_format(
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument(
"--num-iters-cold",
type=int,
@@ -234,7 +235,7 @@ def main(args: argparse.Namespace):
"""
# Create a queue for inter-process communication
result_queue = multiprocessing.Queue()
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
process = multiprocessing.Process(
target=run_startup_in_subprocess,
args=(
vllm/benchmarks/sweep/cli.py
+2 -1
View File
@@ -3,6 +3,7 @@
import argparse
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.utils.argparse_utils import FlexibleArgumentParser
from .plot import SweepPlotArgs
from .plot import main as plot_main
@@ -24,7 +25,7 @@ SUBCOMMANDS = (
)
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
subparsers = parser.add_subparsers(required=True, dest="sweep_type")
for cmd, entrypoint in SUBCOMMANDS:
vllm/benchmarks/sweep/param_sweep.py
+2 -3
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
from typing import Any
class ParameterSweep(list["ParameterSweepItem"]):
@@ -62,8 +61,8 @@ class ParameterSweepItem(dict[str, object]):
return cls(record)
def __or__(self, other: dict[str, Any]):
return type(self)(super().__or__(other))
def __or__(self, other: dict[str, object], /) -> "ParameterSweepItem": # type: ignore[override]
return ParameterSweepItem(super().__or__(other))
@property
def name(self) -> str:
vllm/benchmarks/sweep/plot.py
+6 -3
View File
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, ClassVar
from typing_extensions import Self, override
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule
@@ -51,6 +52,7 @@ class PlotFilterBase(ABC):
class PlotEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try:
target = float(self.target)
except ValueError:
@@ -63,6 +65,7 @@ class PlotEqualTo(PlotFilterBase):
class PlotNotEqualTo(PlotFilterBase):
@override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try:
target = float(self.target)
except ValueError:
@@ -182,7 +185,7 @@ def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, ob
"""
converted_data = []
for record in data:
converted_record = {}
converted_record: dict[str, object] = {}
for key, value in record.items():
if isinstance(value, str):
if value in ["inf", "-inf", "nan"]:
@@ -531,7 +534,7 @@ class SweepPlotArgs:
)
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"EXPERIMENT_DIR",
type=str,
@@ -682,7 +685,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepPlotArgs.parser_help)
parser = FlexibleArgumentParser(description=SweepPlotArgs.parser_help)
SweepPlotArgs.add_cli_args(parser)
main(parser.parse_args())
vllm/benchmarks/sweep/plot_pareto.py
+3 -2
View File
@@ -8,6 +8,7 @@ from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule
@@ -342,7 +343,7 @@ class SweepPlotParetoArgs:
)
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser):
def add_cli_args(cls, parser: FlexibleArgumentParser):
parser.add_argument(
"EXPERIMENT_DIR",
type=str,
@@ -394,7 +395,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepPlotParetoArgs.parser_help)
parser = FlexibleArgumentParser(description=SweepPlotParetoArgs.parser_help)
SweepPlotParetoArgs.add_cli_args(parser)
main(parser.parse_args())
vllm/benchmarks/sweep/serve.py
+3 -2
View File
@@ -10,6 +10,7 @@ from datetime import datetime
from pathlib import Path
from typing import ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -368,7 +369,7 @@ class SweepServeArgs:
)
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--serve-cmd",
type=str,
@@ -531,7 +532,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepServeArgs.parser_help)
parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help)
SweepServeArgs.add_cli_args(parser)
main(parser.parse_args())
vllm/benchmarks/sweep/serve_workload.py
+3 -2
View File
@@ -10,6 +10,7 @@ import numpy as np
from typing_extensions import assert_never
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -273,7 +274,7 @@ class SweepServeWorkloadArgs(SweepServeArgs):
)
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser = super().add_cli_args(parser)
workload_group = parser.add_argument_group("workload options")
@@ -322,7 +323,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepServeWorkloadArgs.parser_help)
parser = FlexibleArgumentParser(description=SweepServeWorkloadArgs.parser_help)
SweepServeWorkloadArgs.add_cli_args(parser)
main(parser.parse_args())
vllm/benchmarks/sweep/startup.py
+2 -2
View File
@@ -317,7 +317,7 @@ class SweepStartupArgs:
)
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--startup-cmd",
type=str,
@@ -438,6 +438,6 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepStartupArgs.parser_help)
parser = FlexibleArgumentParser(description=SweepStartupArgs.parser_help)
SweepStartupArgs.add_cli_args(parser)
main(parser.parse_args())
vllm/benchmarks/throughput.py
+44 -17
View File
@@ -18,6 +18,7 @@ from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import (
AIMODataset,
ASRDataset,
BenchmarkDataset,
BurstGPTDataset,
ConversationDataset,
InstructCoderDataset,
@@ -41,6 +42,7 @@ from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.async_utils import merge_async_iterators
@@ -102,13 +104,15 @@ def _run_vllm_requests(
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
)
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
prompt_token_ids = request.prompt["prompt_token_ids"]
assert isinstance(prompt_token_ids, list)
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
prompt["multi_modal_data"] = request.multi_modal_data
@@ -159,7 +163,20 @@ def _run_vllm_requests(
end = time.perf_counter()
else:
assert lora_requests is None, "BeamSearch API does not support LoRA"
beam_prompts = [request.prompt for request in requests]
beam_prompts: list[TextPrompt | TokensPrompt] = []
for request in requests:
if isinstance(request.prompt, str):
beam_prompts.append(TextPrompt(prompt=request.prompt))
elif (
isinstance(request.prompt, dict)
and "prompt_token_ids" in request.prompt
):
token_ids = request.prompt["prompt_token_ids"]
assert isinstance(token_ids, list)
beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids))
else:
# Fallback: convert to string
beam_prompts.append(TextPrompt(prompt=str(request.prompt)))
# output_len should be the same for all requests.
output_len = requests[0].expected_output_len
for request in requests:
@@ -268,7 +285,7 @@ def _run_vllm_chat_requests(
llm.wake_up(tags=["scheduling"])
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
else:
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type]
if do_profile:
llm.stop_profile()
@@ -339,11 +356,13 @@ async def _run_vllm_async_requests(
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
)
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
prompt_token_ids = request.prompt["prompt_token_ids"]
assert isinstance(prompt_token_ids, list)
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
@@ -366,9 +385,11 @@ async def _run_vllm_async_requests(
start = time.perf_counter()
if do_profile:
await llm.start_profile()
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
for i, (prompt_item, sp, lr) in enumerate(
zip(prompts, sampling_params, lora_requests)
):
generator = llm.generate(
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
)
generators.append(generator)
all_gens = merge_async_iterators(*generators)
@@ -440,6 +461,7 @@ def _run_hf_requests(
prompt_len = requests[i].prompt_len
output_len = requests[i].expected_output_len
# Add the prompt to the batch.
assert isinstance(prompt, str), "Prompt must be a string for HF backend"
batch.append(prompt)
max_prompt_len = max(max_prompt_len, prompt_len)
max_output_len = max(max_output_len, output_len)
@@ -500,6 +522,7 @@ def save_to_pytorch_benchmark_format(
def get_requests(args, tokenizer):
# Common parameters for all dataset types.
dataset_cls: type[BenchmarkDataset]
common_kwargs = {
"dataset_path": args.dataset_path,
"random_seed": args.seed,
@@ -875,7 +898,7 @@ def validate_args(args):
)
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument(
"--backend",
type=str,
@@ -1178,7 +1201,11 @@ def main(args: argparse.Namespace):
total_prompt_tokens += (
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
)
total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o)
total_output_tokens += sum(
len(o.token_ids)
for o in ro.outputs
if o is not None and o.token_ids is not None
)
total_num_tokens = total_prompt_tokens + total_output_tokens
else:
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)
vllm/entrypoints/cli/benchmark/base.py
+2 -1
View File
@@ -3,6 +3,7 @@
import argparse
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSubcommandBase(CLISubcommand):
@@ -11,7 +12,7 @@ class BenchmarkSubcommandBase(CLISubcommand):
help: str
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser."""
raise NotImplementedError
vllm/entrypoints/cli/benchmark/latency.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.latency import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
help = "Benchmark the latency of a single batch of requests."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
vllm/entrypoints/cli/benchmark/mm_processor.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.mm_processor import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
help = "Benchmark multimodal processor latency across different configurations."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
vllm/entrypoints/cli/benchmark/serve.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.serve import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
help = "Benchmark the online serving throughput."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
vllm/entrypoints/cli/benchmark/startup.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.startup import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
help = "Benchmark the startup time of vLLM models."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
vllm/entrypoints/cli/benchmark/sweep.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.sweep.cli import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
help = "Benchmark for a parameter sweep."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
vllm/entrypoints/cli/benchmark/throughput.py
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.throughput import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
help = "Benchmark offline inference throughput."
@classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod