[MyPy] Fix mypy for vllm/benchmarks (#39896)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Martin Hickey
2026-06-23 15:22:29 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 547d2c40d7
commit 275b43183c
24 changed files with 297 additions and 151 deletions
-2
View File
@@ -31,8 +31,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [ EXCLUDE = [
"vllm/model_executor/models", "vllm/model_executor/models",
"vllm/model_executor/layers/fla/ops", "vllm/model_executor/layers/fla/ops",
# TODO: Remove these entries after fixing mypy errors.
"vllm/benchmarks",
] ]
@@ -30,7 +30,6 @@ The resulting JSONL file can then be used with the serving benchmark::
from __future__ import annotations from __future__ import annotations
import argparse
import json import json
import logging import logging
import random import random
@@ -40,6 +39,7 @@ import numpy as np
from transformers import AutoTokenizer from transformers import AutoTokenizer
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -121,7 +121,7 @@ def create_txt_slices_jsonl(
def main(argv: list[str] | None = None) -> None: def main(argv: list[str] | None = None) -> None:
parser = argparse.ArgumentParser( parser = FlexibleArgumentParser(
description="Convert a plain-text file into a JSONL dataset " description="Convert a plain-text file into a JSONL dataset "
"for CustomDataset (--dataset-name custom).", "for CustomDataset (--dataset-name custom).",
) )
+141 -64
View File
@@ -81,7 +81,7 @@ class SampleRequest:
prompt: str | list[str] | list[dict] prompt: str | list[str] | list[dict]
prompt_len: int prompt_len: int
expected_output_len: int | None expected_output_len: int = 0
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
lora_request: LoRARequest | None = None lora_request: LoRARequest | None = None
request_id: str | None = None request_id: str | None = None
@@ -634,7 +634,7 @@ class RandomDataset(BenchmarkDataset):
# Generate prefix once # Generate prefix once
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len) prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
requests = [] requests: list[SampleRequest] = []
token_mismatch_total = 0 token_mismatch_total = 0
for i in range(num_requests): for i in range(num_requests):
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501 prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
@@ -665,15 +665,14 @@ class RandomDataset(BenchmarkDataset):
) )
# only used for embeddings benchmark. # only used for embeddings benchmark.
if batchsize > 1: if batchsize > 1:
batch_requests = [] batch_requests: list[SampleRequest] = []
# Create batched requests # Create batched requests
for i in range(0, num_requests, batchsize): for i in range(0, num_requests, batchsize):
batch = requests[i : i + batchsize] batch = requests[i : i + batchsize]
batch_requests.append( batch_requests.append(
SampleRequest( SampleRequest(
prompt=[req.prompt for req in batch], prompt=[req.prompt for req in batch], # type: ignore[arg-type]
prompt_len=sum(req.prompt_len for req in batch), prompt_len=sum(req.prompt_len for req in batch),
expected_output_len=0,
request_id=request_id_prefix + str(i // batchsize), request_id=request_id_prefix + str(i // batchsize),
) )
) )
@@ -797,6 +796,9 @@ class RandomDatasetForReranking(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN, input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN, output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1, batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
is_reranker: bool = True, is_reranker: bool = True,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
@@ -1207,6 +1209,9 @@ class RandomMultiModalDataset(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN, input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN, output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1, batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT, limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST, base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO, num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
@@ -1251,6 +1256,7 @@ class RandomMultiModalDataset(RandomDataset):
# We want to exclude placeholder tokens and all # We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it # tokens that indicate start/end of image as it
# may break prompt replacement logic. # may break prompt replacement logic.
assert hasattr(tokenizer, "added_tokens_decoder")
prohibited_tokens = list( prohibited_tokens = list(
tok_id tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items() for tok_id, token in tokenizer.added_tokens_decoder.items()
@@ -1376,6 +1382,7 @@ class ShareGPTDataset(BenchmarkDataset):
lora_assignment: str = "random", lora_assignment: str = "random",
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
samples: list[SampleRequest] = [] samples: list[SampleRequest] = []
ind = 0 ind = 0
for entry in self.data: for entry in self.data:
@@ -1454,7 +1461,7 @@ class TimedTrace(BenchmarkDataset):
f'label_output_length: "{self.label_output_length}", ' f'label_output_length: "{self.label_output_length}", '
f'label_hash_ids: "{self.label_hash_ids}"' f'label_hash_ids: "{self.label_hash_ids}"'
) )
self._expanded_generated_prompts = {} self._expanded_generated_prompts: dict[str, Any] = {}
self.load_data() self.load_data()
def load_data(self) -> None: def load_data(self) -> None:
@@ -1529,10 +1536,12 @@ class TimedTrace(BenchmarkDataset):
tokenizer: TokenizerLike, tokenizer: TokenizerLike,
num_requests: int, num_requests: int,
request_id_prefix: str = "", request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs, **kwargs,
) -> list: ) -> list[SampleRequest]:
samples: list = [] samples: list[SampleRequest] = []
assert tokenizer is not None, "Tokenizer must be provided, now is Null" assert tokenizer is not None, "Tokenizer must be provided, now is Null"
assert self.data is not None, "Data must be loaded before sampling"
for ind, entry in enumerate(self.data): for ind, entry in enumerate(self.data):
if len(samples) >= num_requests: if len(samples) >= num_requests:
@@ -2139,12 +2148,12 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
) )
elif args.dataset_name == "sonnet": elif args.dataset_name == "sonnet":
dataset = SonnetDataset( sonnet_dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
) )
# For the "sonnet" dataset, formatting depends on the backend. # For the "sonnet" dataset, formatting depends on the backend.
if args.backend == "openai-chat": if args.backend == "openai-chat":
input_requests = dataset.sample( input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts, num_requests=args.num_prompts,
input_len=args.sonnet_input_len, input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len, output_len=args.sonnet_output_len,
@@ -2155,10 +2164,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample, no_oversample=args.no_oversample,
) )
else: else:
assert tokenizer.chat_template or tokenizer.default_chat_template, ( assert (
"Tokenizer/model must have chat template for sonnet dataset." hasattr(tokenizer, "chat_template") and tokenizer.chat_template
) ) or (
input_requests = dataset.sample( hasattr(tokenizer, "default_chat_template")
and tokenizer.default_chat_template
), "Tokenizer/model must have chat template for sonnet dataset."
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts, num_requests=args.num_prompts,
input_len=args.sonnet_input_len, input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len, output_len=args.sonnet_output_len,
@@ -2173,6 +2185,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
# all following datasets are implemented from the # all following datasets are implemented from the
# HuggingFaceDataset base class # HuggingFaceDataset base class
hf_kwargs = {} hf_kwargs = {}
dataset_class: type[BenchmarkDataset]
if ( if (
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
@@ -2548,23 +2561,23 @@ class CustomDataset(BenchmarkDataset):
if tokenizer is None: if tokenizer is None:
new_output_len = 1 new_output_len = 1
elif output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
else: else:
new_output_len = output_len new_output_len = output_len
if output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
if tokenizer is None: if tokenizer is None:
prompt_len = 1 prompt_len = 1
@@ -2797,11 +2810,15 @@ class CustomImageDataset(CustomDataset):
self, self,
tokenizer: TokenizerLike, tokenizer: TokenizerLike,
num_requests: int, num_requests: int,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
ensure_client_side_data: bool = False,
request_id_prefix: str = "", request_id_prefix: str = "",
no_oversample: bool = False, no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
ensure_client_side_data: bool = False,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
# load all data if needed # load all data if needed
@@ -2814,7 +2831,7 @@ class CustomImageDataset(CustomDataset):
num_requests, num_requests,
) )
sampled_requests = [] sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
@@ -2837,7 +2854,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest( SampleRequest(
prompt=prompt, prompt=prompt,
prompt_len=prompt_len, prompt_len=prompt_len,
expected_output_len=output_len, expected_output_len=output_len or 0,
multi_modal_data=None, multi_modal_data=None,
request_id=request_id_prefix + str(i), request_id=request_id_prefix + str(i),
) )
@@ -2863,7 +2880,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest( SampleRequest(
prompt=prompt, prompt=prompt,
prompt_len=prompt_len, prompt_len=prompt_len,
expected_output_len=output_len, expected_output_len=output_len or 0,
multi_modal_data=mm_content, multi_modal_data=mm_content,
request_id=request_id_prefix + str(i), request_id=request_id_prefix + str(i),
) )
@@ -2891,17 +2908,20 @@ class CustomAudioDataset(CustomDataset):
self, self,
tokenizer: TokenizerLike, tokenizer: TokenizerLike,
num_requests: int, num_requests: int,
output_len: int | None = None,
request_id_prefix: str = "", request_id_prefix: str = "",
no_oversample: bool = False, no_oversample: bool = False,
skip_chat_template: bool = False, lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False, enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
self.num_available_samples = len(self.data) self.num_available_samples = len(self.data)
if num_requests <= 0: if num_requests <= 0:
num_requests = self.num_available_samples num_requests = self.num_available_samples
sampled_requests = [] sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
@@ -2950,7 +2970,6 @@ class CustomAudioDataset(CustomDataset):
prompt_len = ( prompt_len = (
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1 len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
) )
new_output_len = output_len
if output_len is None or output_len == -1: if output_len is None or output_len == -1:
if "output_tokens" not in item: if "output_tokens" not in item:
raise ValueError( raise ValueError(
@@ -2958,6 +2977,8 @@ class CustomAudioDataset(CustomDataset):
"custom dataset must contain an 'output_tokens' field." "custom dataset must contain an 'output_tokens' field."
) )
new_output_len = int(item["output_tokens"]) new_output_len = int(item["output_tokens"])
else:
new_output_len = output_len
sampled_requests.append( sampled_requests.append(
SampleRequest( SampleRequest(
prompt=prompt, prompt=prompt,
@@ -3015,10 +3036,30 @@ class SpecBench(CustomDataset):
def sample( def sample(
self, self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
# leverage CustomDataset sample # leverage CustomDataset sample
return super().sample( return super().sample(
tokenizer=tokenizer,
num_requests=num_requests,
request_id_prefix=request_id_prefix,
no_oversample=no_oversample,
lora_path=lora_path,
max_loras=max_loras,
output_len=output_len,
enable_multimodal_chat=enable_multimodal_chat,
skip_chat_template=skip_chat_template,
chat_template_kwargs=chat_template_kwargs,
**kwargs, **kwargs,
) )
@@ -3067,16 +3108,21 @@ class SonnetDataset(BenchmarkDataset):
return_prompt_formatted: bool = False, return_prompt_formatted: bool = False,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
poem_lines = self.data
assert poem_lines is not None
# Calculate average token length for a poem line. # Calculate average token length for a poem line.
tokenized_lines = [tokenizer(line).input_ids for line in self.data] tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines) avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
# Build the base prompt. # Build the base prompt.
base_prompt = "Pick as many lines as you can from these poem lines:\n" base_prompt = "Pick as many lines as you can from these poem lines:\n"
base_msg = [{"role": "user", "content": base_prompt}] base_msg = [{"role": "user", "content": base_prompt}]
base_fmt = tokenizer.apply_chat_template( base_fmt = tokenizer.apply_chat_template(
base_msg, add_generation_prompt=True, tokenize=False base_msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
) )
assert isinstance(base_fmt, str)
base_offset = len(tokenizer(base_fmt).input_ids) base_offset = len(tokenizer(base_fmt).input_ids)
if input_len <= base_offset: if input_len <= base_offset:
raise ValueError( raise ValueError(
@@ -3085,26 +3131,31 @@ class SonnetDataset(BenchmarkDataset):
) )
# Determine how many poem lines to use. # Determine how many poem lines to use.
num_input_lines = round((input_len - base_offset) / avg_len) num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0) num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
prefix_lines = self.data[:num_prefix_lines] prefix_lines = poem_lines[:num_prefix_lines]
samples: list[SampleRequest] = [] samples: list[SampleRequest] = []
ind = 0 ind = 0
while len(samples) < num_requests: while len(samples) < num_requests:
extra_lines = random.choices( extra_lines = random.choices(
self.data, k=num_input_lines - num_prefix_lines poem_lines, k=num_input_lines - num_prefix_lines
) )
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}" prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
msg = [{"role": "user", "content": prompt}] msg = [{"role": "user", "content": prompt}]
prompt_formatted = tokenizer.apply_chat_template( prompt_formatted = tokenizer.apply_chat_template(
msg, add_generation_prompt=True, tokenize=False msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
) )
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids) prompt_len = len(tokenizer(prompt_formatted).input_ids)
if prompt_len <= input_len: if prompt_len <= input_len:
samples.append( samples.append(
SampleRequest( SampleRequest(
prompt=prompt_formatted if return_prompt_formatted else prompt, prompt=(
prompt_formatted if return_prompt_formatted else prompt
),
prompt_len=prompt_len, prompt_len=prompt_len,
expected_output_len=output_len, expected_output_len=output_len,
request_id=request_id_prefix + str(ind), request_id=request_id_prefix + str(ind),
@@ -3145,6 +3196,7 @@ class BurstGPTDataset(BenchmarkDataset):
self.data = gpt4_df self.data = gpt4_df
def _sample_loaded_data(self, num_requests: int) -> list: def _sample_loaded_data(self, num_requests: int) -> list:
assert self.data is not None, "Dataset must be loaded before sampling"
if num_requests <= len(self.data): if num_requests <= len(self.data):
data = self.data.sample(n=num_requests, random_state=self.random_seed) data = self.data.sample(n=num_requests, random_state=self.random_seed)
else: else:
@@ -3258,6 +3310,7 @@ class ConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False, enable_multimodal_chat: bool = False,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations # Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2) filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
@@ -3318,6 +3371,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False, enable_multimodal_chat: bool = False,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations # Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2) filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
@@ -3393,7 +3447,8 @@ class VisionArenaDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = [] assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
@@ -3465,7 +3520,8 @@ class MMVUDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = [] assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
@@ -3531,18 +3587,21 @@ class InstructCoderDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
for i, prompt in enumerate(self.sample_prompts(n=num_requests)): for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
prompt_text = prompt
# apply template # apply template
if not skip_chat_template: if not skip_chat_template:
prompt = tokenizer.apply_chat_template( prompt_text_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}], [{"role": "user", "content": prompt}],
add_generation_prompt=True, add_generation_prompt=True,
tokenize=False, tokenize=False,
) )
assert isinstance(prompt_text_result, str)
prompt_text = prompt_text_result
prompt_len = len(tokenizer(prompt).input_ids) prompt_len = len(tokenizer(prompt_text).input_ids)
sampled_requests.append( sampled_requests.append(
SampleRequest( SampleRequest(
prompt=prompt, prompt=prompt_text,
prompt_len=prompt_len, prompt_len=prompt_len,
expected_output_len=output_len, expected_output_len=output_len,
request_id=request_id_prefix + str(i), request_id=request_id_prefix + str(i),
@@ -3554,6 +3613,7 @@ class InstructCoderDataset(HuggingFaceDataset):
return sampled_requests return sampled_requests
def sample_prompts(self, n: int) -> Iterator[str]: def sample_prompts(self, n: int) -> Iterator[str]:
assert self.data is not None, "Dataset must be loaded before sampling"
for item in self.data.take(n): for item in self.data.take(n):
prompt = ( prompt = (
f"{item['input']}\n\n{item['instruction']} Just output " f"{item['input']}\n\n{item['instruction']} Just output "
@@ -3594,6 +3654,7 @@ class MTBenchDataset(HuggingFaceDataset):
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
@@ -3654,7 +3715,8 @@ class HumanEvalDataset(HuggingFaceDataset):
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = [] sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
@@ -3714,7 +3776,8 @@ class GSM8KDataset(HuggingFaceDataset):
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = [] sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
@@ -3772,17 +3835,18 @@ class BlazeditDataset(HuggingFaceDataset):
self, self,
tokenizer: TokenizerLike, tokenizer: TokenizerLike,
num_requests: int, num_requests: int,
output_len: int | None = None,
skip_chat_template: bool = False,
request_id_prefix: str = "", request_id_prefix: str = "",
no_oversample: bool = False, no_oversample: bool = False,
output_len: int | None = None,
skip_chat_template: bool = False,
min_distance: float = 0.0, min_distance: float = 0.0,
max_distance: float = 1.0, max_distance: float = 1.0,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = [] sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Dataset must be loaded before sampling"
for i, item in enumerate(self.data): for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
@@ -3810,11 +3874,13 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
# apply template # apply template
if not skip_chat_template: if not skip_chat_template:
prompt = tokenizer.apply_chat_template( prompt_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}], [{"role": "user", "content": prompt}],
add_generation_prompt=True, add_generation_prompt=True,
tokenize=False, tokenize=False,
) )
assert isinstance(prompt_result, str)
prompt = prompt_result
prompt_len = len(tokenizer(prompt).input_ids) prompt_len = len(tokenizer(prompt).input_ids)
@@ -3858,6 +3924,7 @@ class AIMODataset(HuggingFaceDataset):
output_len: int | None = None, output_len: int | None = None,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
ind = 0 ind = 0
dynamic_output = output_len is None dynamic_output = output_len is None
@@ -3970,6 +4037,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name) formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
if formatting_prompt_func is None: if formatting_prompt_func is None:
raise ValueError(f"Unsupported dataset path: {self.hf_name}") raise ValueError(f"Unsupported dataset path: {self.hf_name}")
assert self.data is not None, "Dataset must be loaded before sampling"
samples = [] samples = []
for i, sample in enumerate(self.data): for i, sample in enumerate(self.data):
sample = formatting_prompt_func(sample) sample = formatting_prompt_func(sample)
@@ -4061,12 +4129,14 @@ class ASRDataset(HuggingFaceDataset):
def _disable_audio_decode(self) -> None: def _disable_audio_decode(self) -> None:
from datasets import Audio from datasets import Audio
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.cast_column("audio", Audio(decode=False)) self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None: def _materialize_local_audio_column(self) -> None:
local_path_root = Path( local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset") hf_api().snapshot_download(self.hf_name, repo_type="dataset")
) )
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.map( self.data = self.data.map(
lambda item: { lambda item: {
"audio": str(local_path_root / item["url"]), "audio": str(local_path_root / item["url"]),
@@ -4090,15 +4160,19 @@ class ASRDataset(HuggingFaceDataset):
else: else:
prompt = "" prompt = ""
prompt_len = len(tokenizer(prompt).input_ids) prompt_len = len(tokenizer(prompt).input_ids)
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
ind = 0 ind = 0
skipped = 0 skipped = 0
asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec") asr_min_audio_len_sec: float = float(kwargs.get("asr_min_audio_len_sec") or 0.0)
asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec") asr_max_audio_len_sec: float = float(
kwargs.get("asr_max_audio_len_sec") or float("inf")
)
durations = [] durations = []
for item in self.data: for item in self.data:
if len(sampled_requests) >= num_requests: if len(sampled_requests) >= num_requests:
break break
mm_content: dict[str, Any]
audio = item["audio"] audio = item["audio"]
if ( if (
isinstance(audio, dict) isinstance(audio, dict)
@@ -4203,8 +4277,7 @@ class MLPerfDataset(HuggingFaceDataset):
output_len: int | None = None, output_len: int | None = None,
**kwargs, **kwargs,
) -> list[SampleRequest]: ) -> list[SampleRequest]:
# Force dynamic output length based on reference completion. assert self.data is not None, "Dataset must be loaded before sampling"
dynamic_output = output_len is None
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
ind = 0 ind = 0
@@ -4222,15 +4295,18 @@ class MLPerfDataset(HuggingFaceDataset):
{"role": "user", "content": question}, {"role": "user", "content": question},
] ]
prompt_formatted = tokenizer.apply_chat_template( prompt_formatted = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, tokenize=False messages, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
) )
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids) prompt_len = len(tokenizer(prompt_formatted).input_ids)
# Determine output length from reference answer tokens. # Determine output length from reference answer tokens.
ref_out_len = len( ref_out_len = len(
tokenizer(reference_answer, add_special_tokens=False).input_ids tokenizer(reference_answer, add_special_tokens=False).input_ids
) )
expected_output_len = ref_out_len if dynamic_output else output_len expected_output_len = ref_out_len if output_len is None else output_len
# Validate sequence lengths. # Validate sequence lengths.
if not is_valid_sequence(prompt_len, expected_output_len): if not is_valid_sequence(prompt_len, expected_output_len):
@@ -4371,6 +4447,7 @@ class MMStarDataset(HuggingFaceDataset):
) -> list[SampleRequest]: ) -> list[SampleRequest]:
# If --hf-output-len is not set, use the default output length. # If --hf-output-len is not set, use the default output length.
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = [] sampled_requests: list[SampleRequest] = []
for ind, item in enumerate(self.data): for ind, item in enumerate(self.data):
@@ -4603,7 +4680,7 @@ class BFCLDataset(HuggingFaceDataset):
exc_info=True, exc_info=True,
) )
rendered = None rendered = None
if rendered is not None: if rendered is not None and isinstance(rendered, str):
prompt_len = len(tokenizer(rendered).input_ids) prompt_len = len(tokenizer(rendered).input_ids)
else: else:
text = "\n".join(m.get("content", "") for m in messages) text = "\n".join(m.get("content", "") for m in messages)
+6 -4
View File
@@ -13,8 +13,9 @@ from tqdm import tqdm
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
from vllm.inputs import PromptType from vllm.inputs import TextPrompt, TokensPrompt
from vllm.sampling_params import BeamSearchParams from vllm.sampling_params import BeamSearchParams
from vllm.utils.argparse_utils import FlexibleArgumentParser
def save_to_pytorch_benchmark_format( def save_to_pytorch_benchmark_format(
@@ -30,7 +31,7 @@ def save_to_pytorch_benchmark_format(
write_to_json(pt_file, pt_records) write_to_json(pt_file, pt_records)
def add_cli_args(parser: argparse.ArgumentParser): def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument("--input-len", type=int, default=32) parser.add_argument("--input-len", type=int, default=32)
parser.add_argument("--output-len", type=int, default=128) parser.add_argument("--output-len", type=int, default=128)
parser.add_argument("--batch-size", type=int, default=8) parser.add_argument("--batch-size", type=int, default=8)
@@ -103,8 +104,9 @@ def main(args: argparse.Namespace):
dummy_prompt_token_ids = np.random.randint( dummy_prompt_token_ids = np.random.randint(
10000, size=(args.batch_size, args.input_len) 10000, size=(args.batch_size, args.input_len)
) )
dummy_prompts: list[PromptType] = [ dummy_prompts: list[TokensPrompt | TextPrompt] = [
{"prompt_token_ids": batch} for batch in dummy_prompt_token_ids.tolist() TokensPrompt(prompt_token_ids=batch)
for batch in dummy_prompt_token_ids.tolist()
] ]
def llm_generate(): def llm_generate():
+24 -20
View File
@@ -75,7 +75,7 @@ class RequestFuncInput:
logprobs: int | None = None logprobs: int | None = None
extra_headers: dict | None = None extra_headers: dict | None = None
extra_body: dict | None = None extra_body: dict | None = None
multi_modal_content: dict | list[dict] | None = None multi_modal_content: dict[str, Any] | list[dict[str, Any]] | None = None
ignore_eos: bool = False ignore_eos: bool = False
language: str | None = None language: str | None = None
request_id: str | None = None request_id: str | None = None
@@ -272,13 +272,13 @@ def _get_chat_content(
request_func_input: RequestFuncInput, request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last", mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
mm_contents = [] mm_contents: list[dict[str, Any]] = []
if request_func_input.multi_modal_content: if request_func_input.multi_modal_content:
mm_content = request_func_input.multi_modal_content mm_content = request_func_input.multi_modal_content
if isinstance(mm_content, list): if isinstance(mm_content, list):
mm_contents.extend(request_func_input.multi_modal_content) mm_contents.extend(mm_content)
elif isinstance(mm_content, dict): elif isinstance(mm_content, dict):
mm_contents.append(request_func_input.multi_modal_content) mm_contents.append(mm_content)
else: else:
raise TypeError( raise TypeError(
"multi_modal_content must be a dict or list[dict] for openai-chat" "multi_modal_content must be a dict or list[dict] for openai-chat"
@@ -293,10 +293,11 @@ def _get_chat_content(
for item in prompt for item in prompt
) )
): ):
prompt_dicts: list[dict[str, Any]] = prompt # type: ignore[assignment]
if mm_position == "first": if mm_position == "first":
return mm_contents + prompt return mm_contents + prompt_dicts
return prompt + mm_contents return prompt_dicts + mm_contents
text_contents = [{"type": "text", "text": prompt}] text_contents = [{"type": "text", "text": prompt}]
@@ -307,15 +308,15 @@ def _get_chat_content(
def _is_chat_messages(prompt: Any) -> bool: def _is_chat_messages(prompt: Any) -> bool:
return ( if not isinstance(prompt, list):
isinstance(prompt, list) return False
and prompt if not prompt:
and all( return False
isinstance(item, dict) return all(
and isinstance(item.get("role"), str) isinstance(item, dict)
and isinstance(item.get("content"), (str, list)) and isinstance(item.get("role"), str)
for item in prompt and isinstance(item.get("content"), (str, list))
) for item in prompt
) )
@@ -325,7 +326,7 @@ def _get_chat_messages(
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
prompt = request_func_input.prompt prompt = request_func_input.prompt
if _is_chat_messages(prompt): if _is_chat_messages(prompt):
return prompt return prompt # type: ignore[return-value]
return [ return [
{ {
@@ -385,8 +386,8 @@ async def async_request_openai_chat_completions(
if not chunk_bytes: if not chunk_bytes:
continue continue
messages = handler.add_chunk(chunk_bytes) message_strings = handler.add_chunk(chunk_bytes)
for message in messages: for message in message_strings:
# NOTE: SSE comments (often used as pings) start with # NOTE: SSE comments (often used as pings) start with
# a colon. These are not JSON data payload and should # a colon. These are not JSON data payload and should
# be skipped. # be skipped.
@@ -790,7 +791,7 @@ async def async_request_infinity_embeddings(
api_url = request_func_input.api_url api_url = request_func_input.api_url
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings") _validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
payload = { payload: dict[str, Any] = {
"model": request_func_input.model_name "model": request_func_input.model_name
if request_func_input.model_name if request_func_input.model_name
else request_func_input.model, else request_func_input.model,
@@ -849,7 +850,10 @@ async def async_request_vllm_pooling(
"truncate_prompt_tokens": -1, "truncate_prompt_tokens": -1,
} }
payload = payload | request_func_input.prompt if isinstance(request_func_input.prompt, dict):
payload = payload | request_func_input.prompt
else:
payload["input"] = request_func_input.prompt
_update_payload_common(payload, request_func_input) _update_payload_common(payload, request_func_input)
+2 -2
View File
@@ -15,7 +15,7 @@ def extract_field(
if field_name in extra_info: if field_name in extra_info:
return extra_info[field_name] return extra_info[field_name]
v = args v: Any = args
# For example, args.compilation_config.mode # For example, args.compilation_config.mode
for nested_field in field_name.split("."): for nested_field in field_name.split("."):
if not hasattr(v, nested_field): if not hasattr(v, nested_field):
@@ -43,7 +43,7 @@ def convert_to_pytorch_benchmark_format(
on metric per record on metric per record
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
""" """
records = [] records: list[Any] = []
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False): if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
return records return records
+3 -2
View File
@@ -28,6 +28,7 @@ from vllm.benchmarks.datasets import (
) )
from vllm.benchmarks.throughput import get_requests from vllm.benchmarks.throughput import get_requests
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.gc_utils import freeze_gc_heap from vllm.utils.gc_utils import freeze_gc_heap
from vllm.utils.import_utils import PlaceholderModule from vllm.utils.import_utils import PlaceholderModule
@@ -369,7 +370,7 @@ def benchmark_multimodal_processor(
return benchmark_result return benchmark_result
def add_cli_args(parser: argparse.ArgumentParser) -> None: def add_cli_args(parser: FlexibleArgumentParser) -> None:
"""Add CLI arguments for the multimodal processor benchmark.""" """Add CLI arguments for the multimodal processor benchmark."""
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
@@ -532,7 +533,7 @@ def main(args: argparse.Namespace) -> None:
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Benchmark mm processor latency") parser = FlexibleArgumentParser(description="Benchmark mm processor latency")
add_cli_args(parser) add_cli_args(parser)
args = parser.parse_args() args = parser.parse_args()
main(args) main(args)
+37 -14
View File
@@ -30,7 +30,7 @@ import ssl
import time import time
import uuid import uuid
import warnings import warnings
from collections.abc import AsyncGenerator, Iterable from collections.abc import AsyncGenerator, Iterable, Iterator
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from datetime import datetime from datetime import datetime
from enum import Enum from enum import Enum
@@ -52,6 +52,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.gc_utils import freeze_gc_heap from vllm.utils.gc_utils import freeze_gc_heap
from vllm.utils.network_utils import join_host_port from vllm.utils.network_utils import join_host_port
@@ -129,6 +130,7 @@ async def _align_prompts_to_server_tokenizer(
) )
async def _fix_one(req: SampleRequest) -> SampleRequest: async def _fix_one(req: SampleRequest) -> SampleRequest:
assert isinstance(req.prompt, str)
tokens = await _tokenize(req.prompt) tokens = await _tokenize(req.prompt)
if len(tokens) <= req.prompt_len: if len(tokens) <= req.prompt_len:
return req return req
@@ -360,7 +362,7 @@ class EmbedBenchmarkMetrics:
mean_e2el_ms: float mean_e2el_ms: float
std_e2el_ms: float std_e2el_ms: float
median_e2el_ms: float median_e2el_ms: float
percentiles_e2el_ms: float percentiles_e2el_ms: list[tuple[float, float]]
def _get_current_request_rate( def _get_current_request_rate(
@@ -433,8 +435,8 @@ async def get_request(
assert total_requests > 0, "No requests provided." assert total_requests > 0, "No requests provided."
# Precompute delays among requests to minimize request send laggings # Precompute delays among requests to minimize request send laggings
request_rates = [] request_rates: list[float] = []
delay_ts = [] delay_ts: list[float] = []
# if the traces have timing info then: # if the traces have timing info then:
if not self_timed: if not self_timed:
@@ -484,7 +486,10 @@ async def get_request(
else: else:
for request_index, request in enumerate(input_requests): for request_index, request in enumerate(input_requests):
# this is cumulative running ts, from which sleep is calculated later # this is cumulative running ts, from which sleep is calculated later
delay_ts.append(request.timestamp) if request.timestamp is not None:
delay_ts.append(request.timestamp)
else:
delay_ts.append(0.0)
# TODO: there is no notion of RPS here, may be we can calculate # TODO: there is no notion of RPS here, may be we can calculate
# from the trace. # from the trace.
request_rates.append(0.0) request_rates.append(0.0)
@@ -599,7 +604,7 @@ def calculate_metrics(
) )
actual_output_lens.append(output_len) actual_output_lens.append(output_len)
total_input += outputs[i].prompt_len total_input += outputs[i].prompt_len
tpot = 0 tpot = 0.0
if output_len > 1: if output_len > 1:
latency_minus_ttft = outputs[i].latency - outputs[i].ttft latency_minus_ttft = outputs[i].latency - outputs[i].ttft
tpot = latency_minus_ttft / (output_len - 1) tpot = latency_minus_ttft / (output_len - 1)
@@ -894,11 +899,12 @@ async def benchmark(
print("Starting main benchmark run...") print("Starting main benchmark run...")
lora_modules_iter: Iterator[str] | None = None
if lora_modules: if lora_modules:
lora_modules_list = list(lora_modules) lora_modules_list = list(lora_modules)
if lora_assignment == "round-robin": if lora_assignment == "round-robin":
# Deterministic round-robin assignment across requests. # Deterministic round-robin assignment across requests.
lora_modules = iter( lora_modules_iter = iter(
[ [
lora_modules_list[i % len(lora_modules_list)] lora_modules_list[i % len(lora_modules_list)]
for i in range(len(input_requests)) for i in range(len(input_requests))
@@ -906,7 +912,7 @@ async def benchmark(
) )
else: else:
# For each input request, choose a LoRA module at random. # For each input request, choose a LoRA module at random.
lora_modules = iter( lora_modules_iter = iter(
[random.choice(lora_modules_list) for _ in range(len(input_requests))] [random.choice(lora_modules_list) for _ in range(len(input_requests))]
) )
@@ -1004,19 +1010,23 @@ async def benchmark(
) )
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides) per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
req_model_id, req_model_name = model_id, model_name req_model_id, req_model_name = model_id, model_name
if lora_modules: if lora_modules_iter:
req_lora_module = next(lora_modules) req_lora_module = next(lora_modules_iter)
req_model_id, req_model_name = req_lora_module, req_lora_module req_model_id, req_model_name = req_lora_module, req_lora_module
mm_content_typed: dict[str, Any] | list[dict[str, Any]] | None = None
if isinstance(mm_content, (dict, list)):
mm_content_typed = mm_content
request_func_input = RequestFuncInput( request_func_input = RequestFuncInput(
model=req_model_id, model=req_model_id,
model_name=req_model_name, model_name=req_model_name,
prompt=prompt, prompt=prompt,
api_url=api_url, api_url=api_url,
prompt_len=prompt_len, prompt_len=prompt_len,
output_len=output_len, output_len=output_len or 0,
logprobs=logprobs, logprobs=logprobs,
multi_modal_content=mm_content, multi_modal_content=mm_content_typed,
ignore_eos=ignore_eos, ignore_eos=ignore_eos,
extra_headers=extra_headers, extra_headers=extra_headers,
extra_body=per_request_extra_body, extra_body=per_request_extra_body,
@@ -1107,6 +1117,8 @@ async def benchmark(
"committed_per_step": delta_committed / denoising_steps, "committed_per_step": delta_committed / denoising_steps,
} }
metrics: BenchmarkMetrics | EmbedBenchmarkMetrics
actual_output_lens: list[int] | int
if task_type == TaskType.GENERATION: if task_type == TaskType.GENERATION:
metrics, actual_output_lens = calculate_metrics( metrics, actual_output_lens = calculate_metrics(
input_requests=input_requests, input_requests=input_requests,
@@ -1140,7 +1152,7 @@ async def benchmark(
"Request throughput (req/s):", metrics.request_throughput "Request throughput (req/s):", metrics.request_throughput
) )
) )
if goodput_config_dict: if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
print( print(
"{:<40} {:<10.2f}".format( "{:<40} {:<10.2f}".format(
"Request goodput (req/s):", metrics.request_goodput "Request goodput (req/s):", metrics.request_goodput
@@ -1177,6 +1189,7 @@ async def benchmark(
) )
) )
result: dict[str, Any]
if isinstance(metrics, BenchmarkMetrics): if isinstance(metrics, BenchmarkMetrics):
result = { result = {
"duration": benchmark_duration, "duration": benchmark_duration,
@@ -1462,7 +1475,7 @@ def compute_result_filename(
return file_name return file_name
def add_cli_args(parser: argparse.ArgumentParser): def add_cli_args(parser: FlexibleArgumentParser):
add_dataset_parser(parser) add_dataset_parser(parser)
parser.add_argument( parser.add_argument(
"--label", "--label",
@@ -2023,6 +2036,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
args.self_timed = False args.self_timed = False
# Load the dataset. # Load the dataset.
assert tokenizer is not None, "Tokenizer must be initialized before loading dataset"
input_requests = get_samples(args, tokenizer) input_requests = get_samples(args, tokenizer)
if args.dataset_name in ("random", "prefix_repetition"): if args.dataset_name in ("random", "prefix_repetition"):
@@ -2154,6 +2168,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Generate timeline plot if requested # Generate timeline plot if requested
if args.plot_timeline: if args.plot_timeline:
assert file_name is not None, (
"file_name must be set when plot_timeline is enabled"
)
try: try:
from vllm.benchmarks.plot import generate_timeline_plot from vllm.benchmarks.plot import generate_timeline_plot
@@ -2200,6 +2217,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Generate dataset statistics plot if requested # Generate dataset statistics plot if requested
if args.plot_dataset_stats: if args.plot_dataset_stats:
assert file_name is not None, (
"file_name must be set when plot_dataset_stats is enabled"
)
try: try:
from vllm.benchmarks.plot import generate_dataset_stats_plot from vllm.benchmarks.plot import generate_dataset_stats_plot
@@ -2249,6 +2269,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
# Save to file # Save to file
if args.save_result or args.append_result: if args.save_result or args.append_result:
assert file_name is not None, (
"file_name must be set when save_result or append_result is enabled"
)
with open( with open(
file_name, mode="a+" if args.append_result else "w", encoding="utf-8" file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
) as outfile: ) as outfile:
+3 -2
View File
@@ -26,6 +26,7 @@ from vllm.benchmarks.lib.utils import (
write_to_json, write_to_json,
) )
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
PERCENTAGES = [10, 25, 50, 75, 90, 99] PERCENTAGES = [10, 25, 50, 75, 90, 99]
@@ -190,7 +191,7 @@ def save_to_pytorch_benchmark_format(
write_to_json(f"{base_name}.{m.key}.pytorch.json", records) write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
def add_cli_args(parser: argparse.ArgumentParser): def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument( parser.add_argument(
"--num-iters-cold", "--num-iters-cold",
type=int, type=int,
@@ -234,7 +235,7 @@ def main(args: argparse.Namespace):
""" """
# Create a queue for inter-process communication # Create a queue for inter-process communication
result_queue = multiprocessing.Queue() result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
process = multiprocessing.Process( process = multiprocessing.Process(
target=run_startup_in_subprocess, target=run_startup_in_subprocess,
args=( args=(
+2 -1
View File
@@ -3,6 +3,7 @@
import argparse import argparse
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.utils.argparse_utils import FlexibleArgumentParser
from .plot import SweepPlotArgs from .plot import SweepPlotArgs
from .plot import main as plot_main from .plot import main as plot_main
@@ -24,7 +25,7 @@ SUBCOMMANDS = (
) )
def add_cli_args(parser: argparse.ArgumentParser): def add_cli_args(parser: FlexibleArgumentParser):
subparsers = parser.add_subparsers(required=True, dest="sweep_type") subparsers = parser.add_subparsers(required=True, dest="sweep_type")
for cmd, entrypoint in SUBCOMMANDS: for cmd, entrypoint in SUBCOMMANDS:
+2 -3
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json import json
import os import os
from typing import Any
class ParameterSweep(list["ParameterSweepItem"]): class ParameterSweep(list["ParameterSweepItem"]):
@@ -62,8 +61,8 @@ class ParameterSweepItem(dict[str, object]):
return cls(record) return cls(record)
def __or__(self, other: dict[str, Any]): def __or__(self, other: dict[str, object], /) -> "ParameterSweepItem": # type: ignore[override]
return type(self)(super().__or__(other)) return ParameterSweepItem(super().__or__(other))
@property @property
def name(self) -> str: def name(self) -> str:
+6 -3
View File
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, ClassVar
from typing_extensions import Self, override from typing_extensions import Self, override
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule from vllm.utils.import_utils import PlaceholderModule
@@ -51,6 +52,7 @@ class PlotFilterBase(ABC):
class PlotEqualTo(PlotFilterBase): class PlotEqualTo(PlotFilterBase):
@override @override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try: try:
target = float(self.target) target = float(self.target)
except ValueError: except ValueError:
@@ -63,6 +65,7 @@ class PlotEqualTo(PlotFilterBase):
class PlotNotEqualTo(PlotFilterBase): class PlotNotEqualTo(PlotFilterBase):
@override @override
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame": def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
target: float | str
try: try:
target = float(self.target) target = float(self.target)
except ValueError: except ValueError:
@@ -182,7 +185,7 @@ def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, ob
""" """
converted_data = [] converted_data = []
for record in data: for record in data:
converted_record = {} converted_record: dict[str, object] = {}
for key, value in record.items(): for key, value in record.items():
if isinstance(value, str): if isinstance(value, str):
if value in ["inf", "-inf", "nan"]: if value in ["inf", "-inf", "nan"]:
@@ -531,7 +534,7 @@ class SweepPlotArgs:
) )
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument( parser.add_argument(
"EXPERIMENT_DIR", "EXPERIMENT_DIR",
type=str, type=str,
@@ -682,7 +685,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepPlotArgs.parser_help) parser = FlexibleArgumentParser(description=SweepPlotArgs.parser_help)
SweepPlotArgs.add_cli_args(parser) SweepPlotArgs.add_cli_args(parser)
main(parser.parse_args()) main(parser.parse_args())
+3 -2
View File
@@ -8,6 +8,7 @@ from functools import partial
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, ClassVar from typing import TYPE_CHECKING, ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.collection_utils import full_groupby from vllm.utils.collection_utils import full_groupby
from vllm.utils.import_utils import PlaceholderModule from vllm.utils.import_utils import PlaceholderModule
@@ -342,7 +343,7 @@ class SweepPlotParetoArgs:
) )
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser): def add_cli_args(cls, parser: FlexibleArgumentParser):
parser.add_argument( parser.add_argument(
"EXPERIMENT_DIR", "EXPERIMENT_DIR",
type=str, type=str,
@@ -394,7 +395,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepPlotParetoArgs.parser_help) parser = FlexibleArgumentParser(description=SweepPlotParetoArgs.parser_help)
SweepPlotParetoArgs.add_cli_args(parser) SweepPlotParetoArgs.add_cli_args(parser)
main(parser.parse_args()) main(parser.parse_args())
+3 -2
View File
@@ -10,6 +10,7 @@ from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import ClassVar from typing import ClassVar
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -368,7 +369,7 @@ class SweepServeArgs:
) )
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument( parser.add_argument(
"--serve-cmd", "--serve-cmd",
type=str, type=str,
@@ -531,7 +532,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepServeArgs.parser_help) parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help)
SweepServeArgs.add_cli_args(parser) SweepServeArgs.add_cli_args(parser)
main(parser.parse_args()) main(parser.parse_args())
+3 -2
View File
@@ -10,6 +10,7 @@ import numpy as np
from typing_extensions import assert_never from typing_extensions import assert_never
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -273,7 +274,7 @@ class SweepServeWorkloadArgs(SweepServeArgs):
) )
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser = super().add_cli_args(parser) parser = super().add_cli_args(parser)
workload_group = parser.add_argument_group("workload options") workload_group = parser.add_argument_group("workload options")
@@ -322,7 +323,7 @@ def main(args: argparse.Namespace):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepServeWorkloadArgs.parser_help) parser = FlexibleArgumentParser(description=SweepServeWorkloadArgs.parser_help)
SweepServeWorkloadArgs.add_cli_args(parser) SweepServeWorkloadArgs.add_cli_args(parser)
main(parser.parse_args()) main(parser.parse_args())
+2 -2
View File
@@ -317,7 +317,7 @@ class SweepStartupArgs:
) )
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument( parser.add_argument(
"--startup-cmd", "--startup-cmd",
type=str, type=str,
@@ -438,6 +438,6 @@ def main(args: argparse.Namespace):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description=SweepStartupArgs.parser_help) parser = FlexibleArgumentParser(description=SweepStartupArgs.parser_help)
SweepStartupArgs.add_cli_args(parser) SweepStartupArgs.add_cli_args(parser)
main(parser.parse_args()) main(parser.parse_args())
+44 -17
View File
@@ -18,6 +18,7 @@ from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import ( from vllm.benchmarks.datasets import (
AIMODataset, AIMODataset,
ASRDataset, ASRDataset,
BenchmarkDataset,
BurstGPTDataset, BurstGPTDataset,
ConversationDataset, ConversationDataset,
InstructCoderDataset, InstructCoderDataset,
@@ -41,6 +42,7 @@ from vllm.outputs import RequestOutput
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.async_utils import merge_async_iterators from vllm.utils.async_utils import merge_async_iterators
@@ -102,13 +104,15 @@ def _run_vllm_requests(
prompts: list[TextPrompt | TokensPrompt] = [] prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = [] sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest] | None = [] if enable_lora else None lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None
for request in requests: for request in requests:
prompt = ( if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"]) prompt_token_ids = request.prompt["prompt_token_ids"]
if "prompt_token_ids" in request.prompt assert isinstance(prompt_token_ids, list)
else TextPrompt(prompt=request.prompt) prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
) else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data: if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict) assert isinstance(request.multi_modal_data, dict)
prompt["multi_modal_data"] = request.multi_modal_data prompt["multi_modal_data"] = request.multi_modal_data
@@ -159,7 +163,20 @@ def _run_vllm_requests(
end = time.perf_counter() end = time.perf_counter()
else: else:
assert lora_requests is None, "BeamSearch API does not support LoRA" assert lora_requests is None, "BeamSearch API does not support LoRA"
beam_prompts = [request.prompt for request in requests] beam_prompts: list[TextPrompt | TokensPrompt] = []
for request in requests:
if isinstance(request.prompt, str):
beam_prompts.append(TextPrompt(prompt=request.prompt))
elif (
isinstance(request.prompt, dict)
and "prompt_token_ids" in request.prompt
):
token_ids = request.prompt["prompt_token_ids"]
assert isinstance(token_ids, list)
beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids))
else:
# Fallback: convert to string
beam_prompts.append(TextPrompt(prompt=str(request.prompt)))
# output_len should be the same for all requests. # output_len should be the same for all requests.
output_len = requests[0].expected_output_len output_len = requests[0].expected_output_len
for request in requests: for request in requests:
@@ -268,7 +285,7 @@ def _run_vllm_chat_requests(
llm.wake_up(tags=["scheduling"]) llm.wake_up(tags=["scheduling"])
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True) outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
else: else:
outputs = llm.chat(prompts, sampling_params, use_tqdm=True) outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type]
if do_profile: if do_profile:
llm.stop_profile() llm.stop_profile()
@@ -339,11 +356,13 @@ async def _run_vllm_async_requests(
sampling_params: list[SamplingParams] = [] sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest | None] = [] lora_requests: list[LoRARequest | None] = []
for request in requests: for request in requests:
prompt = ( if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"]) prompt_token_ids = request.prompt["prompt_token_ids"]
if "prompt_token_ids" in request.prompt assert isinstance(prompt_token_ids, list)
else TextPrompt(prompt=request.prompt) prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
) else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data: if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict) assert isinstance(request.multi_modal_data, dict)
@@ -366,9 +385,11 @@ async def _run_vllm_async_requests(
start = time.perf_counter() start = time.perf_counter()
if do_profile: if do_profile:
await llm.start_profile() await llm.start_profile()
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)): for i, (prompt_item, sp, lr) in enumerate(
zip(prompts, sampling_params, lora_requests)
):
generator = llm.generate( generator = llm.generate(
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}" prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
) )
generators.append(generator) generators.append(generator)
all_gens = merge_async_iterators(*generators) all_gens = merge_async_iterators(*generators)
@@ -440,6 +461,7 @@ def _run_hf_requests(
prompt_len = requests[i].prompt_len prompt_len = requests[i].prompt_len
output_len = requests[i].expected_output_len output_len = requests[i].expected_output_len
# Add the prompt to the batch. # Add the prompt to the batch.
assert isinstance(prompt, str), "Prompt must be a string for HF backend"
batch.append(prompt) batch.append(prompt)
max_prompt_len = max(max_prompt_len, prompt_len) max_prompt_len = max(max_prompt_len, prompt_len)
max_output_len = max(max_output_len, output_len) max_output_len = max(max_output_len, output_len)
@@ -500,6 +522,7 @@ def save_to_pytorch_benchmark_format(
def get_requests(args, tokenizer): def get_requests(args, tokenizer):
# Common parameters for all dataset types. # Common parameters for all dataset types.
dataset_cls: type[BenchmarkDataset]
common_kwargs = { common_kwargs = {
"dataset_path": args.dataset_path, "dataset_path": args.dataset_path,
"random_seed": args.seed, "random_seed": args.seed,
@@ -875,7 +898,7 @@ def validate_args(args):
) )
def add_cli_args(parser: argparse.ArgumentParser): def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument( parser.add_argument(
"--backend", "--backend",
type=str, type=str,
@@ -1178,7 +1201,11 @@ def main(args: argparse.Namespace):
total_prompt_tokens += ( total_prompt_tokens += (
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0 len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
) )
total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o) total_output_tokens += sum(
len(o.token_ids)
for o in ro.outputs
if o is not None and o.token_ids is not None
)
total_num_tokens = total_prompt_tokens + total_output_tokens total_num_tokens = total_prompt_tokens + total_output_tokens
else: else:
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests) total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)
+2 -1
View File
@@ -3,6 +3,7 @@
import argparse import argparse
from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.cli.types import CLISubcommand
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSubcommandBase(CLISubcommand): class BenchmarkSubcommandBase(CLISubcommand):
@@ -11,7 +12,7 @@ class BenchmarkSubcommandBase(CLISubcommand):
help: str help: str
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser.""" """Add the CLI arguments to the parser."""
raise NotImplementedError raise NotImplementedError
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.latency import add_cli_args, main from vllm.benchmarks.latency import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase): class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
help = "Benchmark the latency of a single batch of requests." help = "Benchmark the latency of a single batch of requests."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.mm_processor import add_cli_args, main from vllm.benchmarks.mm_processor import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase): class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
help = "Benchmark multimodal processor latency across different configurations." help = "Benchmark multimodal processor latency across different configurations."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.serve import add_cli_args, main from vllm.benchmarks.serve import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkServingSubcommand(BenchmarkSubcommandBase): class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
help = "Benchmark the online serving throughput." help = "Benchmark the online serving throughput."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.startup import add_cli_args, main from vllm.benchmarks.startup import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase): class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
help = "Benchmark the startup time of vLLM models." help = "Benchmark the startup time of vLLM models."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.sweep.cli import add_cli_args, main from vllm.benchmarks.sweep.cli import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase): class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
help = "Benchmark for a parameter sweep." help = "Benchmark for a parameter sweep."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod
+2 -1
View File
@@ -4,6 +4,7 @@ import argparse
from vllm.benchmarks.throughput import add_cli_args, main from vllm.benchmarks.throughput import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase): class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
@@ -13,7 +14,7 @@ class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
help = "Benchmark offline inference throughput." help = "Benchmark offline inference throughput."
@classmethod @classmethod
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser) add_cli_args(parser)
@staticmethod @staticmethod