[MyPy] Fix mypy for vllm/benchmarks (#39896)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
547d2c40d7
commit
275b43183c
@@ -31,8 +31,6 @@ SEPARATE_GROUPS = [
|
|||||||
EXCLUDE = [
|
EXCLUDE = [
|
||||||
"vllm/model_executor/models",
|
"vllm/model_executor/models",
|
||||||
"vllm/model_executor/layers/fla/ops",
|
"vllm/model_executor/layers/fla/ops",
|
||||||
# TODO: Remove these entries after fixing mypy errors.
|
|
||||||
"vllm/benchmarks",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ The resulting JSONL file can then be used with the serving benchmark::
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
@@ -40,6 +39,7 @@ import numpy as np
|
|||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
|
from vllm.benchmarks.datasets.utils import RangeRatio, get_sampling_params
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ def create_txt_slices_jsonl(
|
|||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> None:
|
def main(argv: list[str] | None = None) -> None:
|
||||||
parser = argparse.ArgumentParser(
|
parser = FlexibleArgumentParser(
|
||||||
description="Convert a plain-text file into a JSONL dataset "
|
description="Convert a plain-text file into a JSONL dataset "
|
||||||
"for CustomDataset (--dataset-name custom).",
|
"for CustomDataset (--dataset-name custom).",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ class SampleRequest:
|
|||||||
|
|
||||||
prompt: str | list[str] | list[dict]
|
prompt: str | list[str] | list[dict]
|
||||||
prompt_len: int
|
prompt_len: int
|
||||||
expected_output_len: int | None
|
expected_output_len: int = 0
|
||||||
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
|
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
|
||||||
lora_request: LoRARequest | None = None
|
lora_request: LoRARequest | None = None
|
||||||
request_id: str | None = None
|
request_id: str | None = None
|
||||||
@@ -634,7 +634,7 @@ class RandomDataset(BenchmarkDataset):
|
|||||||
# Generate prefix once
|
# Generate prefix once
|
||||||
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
|
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
|
||||||
|
|
||||||
requests = []
|
requests: list[SampleRequest] = []
|
||||||
token_mismatch_total = 0
|
token_mismatch_total = 0
|
||||||
for i in range(num_requests):
|
for i in range(num_requests):
|
||||||
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
|
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
|
||||||
@@ -665,15 +665,14 @@ class RandomDataset(BenchmarkDataset):
|
|||||||
)
|
)
|
||||||
# only used for embeddings benchmark.
|
# only used for embeddings benchmark.
|
||||||
if batchsize > 1:
|
if batchsize > 1:
|
||||||
batch_requests = []
|
batch_requests: list[SampleRequest] = []
|
||||||
# Create batched requests
|
# Create batched requests
|
||||||
for i in range(0, num_requests, batchsize):
|
for i in range(0, num_requests, batchsize):
|
||||||
batch = requests[i : i + batchsize]
|
batch = requests[i : i + batchsize]
|
||||||
batch_requests.append(
|
batch_requests.append(
|
||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=[req.prompt for req in batch],
|
prompt=[req.prompt for req in batch], # type: ignore[arg-type]
|
||||||
prompt_len=sum(req.prompt_len for req in batch),
|
prompt_len=sum(req.prompt_len for req in batch),
|
||||||
expected_output_len=0,
|
|
||||||
request_id=request_id_prefix + str(i // batchsize),
|
request_id=request_id_prefix + str(i // batchsize),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -797,6 +796,9 @@ class RandomDatasetForReranking(RandomDataset):
|
|||||||
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
||||||
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
||||||
batchsize: int = 1,
|
batchsize: int = 1,
|
||||||
|
max_loras: int | None = None,
|
||||||
|
lora_path: str | None = None,
|
||||||
|
lora_assignment: str = "random",
|
||||||
is_reranker: bool = True,
|
is_reranker: bool = True,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
@@ -1207,6 +1209,9 @@ class RandomMultiModalDataset(RandomDataset):
|
|||||||
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
|
||||||
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
|
||||||
batchsize: int = 1,
|
batchsize: int = 1,
|
||||||
|
max_loras: int | None = None,
|
||||||
|
lora_path: str | None = None,
|
||||||
|
lora_assignment: str = "random",
|
||||||
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
|
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
|
||||||
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
|
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
|
||||||
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
|
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
|
||||||
@@ -1251,6 +1256,7 @@ class RandomMultiModalDataset(RandomDataset):
|
|||||||
# We want to exclude placeholder tokens and all
|
# We want to exclude placeholder tokens and all
|
||||||
# tokens that indicate start/end of image as it
|
# tokens that indicate start/end of image as it
|
||||||
# may break prompt replacement logic.
|
# may break prompt replacement logic.
|
||||||
|
assert hasattr(tokenizer, "added_tokens_decoder")
|
||||||
prohibited_tokens = list(
|
prohibited_tokens = list(
|
||||||
tok_id
|
tok_id
|
||||||
for tok_id, token in tokenizer.added_tokens_decoder.items()
|
for tok_id, token in tokenizer.added_tokens_decoder.items()
|
||||||
@@ -1376,6 +1382,7 @@ class ShareGPTDataset(BenchmarkDataset):
|
|||||||
lora_assignment: str = "random",
|
lora_assignment: str = "random",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
samples: list[SampleRequest] = []
|
samples: list[SampleRequest] = []
|
||||||
ind = 0
|
ind = 0
|
||||||
for entry in self.data:
|
for entry in self.data:
|
||||||
@@ -1454,7 +1461,7 @@ class TimedTrace(BenchmarkDataset):
|
|||||||
f'label_output_length: "{self.label_output_length}", '
|
f'label_output_length: "{self.label_output_length}", '
|
||||||
f'label_hash_ids: "{self.label_hash_ids}"'
|
f'label_hash_ids: "{self.label_hash_ids}"'
|
||||||
)
|
)
|
||||||
self._expanded_generated_prompts = {}
|
self._expanded_generated_prompts: dict[str, Any] = {}
|
||||||
self.load_data()
|
self.load_data()
|
||||||
|
|
||||||
def load_data(self) -> None:
|
def load_data(self) -> None:
|
||||||
@@ -1529,10 +1536,12 @@ class TimedTrace(BenchmarkDataset):
|
|||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
num_requests: int,
|
num_requests: int,
|
||||||
request_id_prefix: str = "",
|
request_id_prefix: str = "",
|
||||||
|
no_oversample: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list:
|
) -> list[SampleRequest]:
|
||||||
samples: list = []
|
samples: list[SampleRequest] = []
|
||||||
assert tokenizer is not None, "Tokenizer must be provided, now is Null"
|
assert tokenizer is not None, "Tokenizer must be provided, now is Null"
|
||||||
|
assert self.data is not None, "Data must be loaded before sampling"
|
||||||
|
|
||||||
for ind, entry in enumerate(self.data):
|
for ind, entry in enumerate(self.data):
|
||||||
if len(samples) >= num_requests:
|
if len(samples) >= num_requests:
|
||||||
@@ -2139,12 +2148,12 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
elif args.dataset_name == "sonnet":
|
elif args.dataset_name == "sonnet":
|
||||||
dataset = SonnetDataset(
|
sonnet_dataset = SonnetDataset(
|
||||||
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
|
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
|
||||||
)
|
)
|
||||||
# For the "sonnet" dataset, formatting depends on the backend.
|
# For the "sonnet" dataset, formatting depends on the backend.
|
||||||
if args.backend == "openai-chat":
|
if args.backend == "openai-chat":
|
||||||
input_requests = dataset.sample(
|
input_requests = sonnet_dataset.sample(
|
||||||
num_requests=args.num_prompts,
|
num_requests=args.num_prompts,
|
||||||
input_len=args.sonnet_input_len,
|
input_len=args.sonnet_input_len,
|
||||||
output_len=args.sonnet_output_len,
|
output_len=args.sonnet_output_len,
|
||||||
@@ -2155,10 +2164,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
|||||||
no_oversample=args.no_oversample,
|
no_oversample=args.no_oversample,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
assert tokenizer.chat_template or tokenizer.default_chat_template, (
|
assert (
|
||||||
"Tokenizer/model must have chat template for sonnet dataset."
|
hasattr(tokenizer, "chat_template") and tokenizer.chat_template
|
||||||
)
|
) or (
|
||||||
input_requests = dataset.sample(
|
hasattr(tokenizer, "default_chat_template")
|
||||||
|
and tokenizer.default_chat_template
|
||||||
|
), "Tokenizer/model must have chat template for sonnet dataset."
|
||||||
|
input_requests = sonnet_dataset.sample(
|
||||||
num_requests=args.num_prompts,
|
num_requests=args.num_prompts,
|
||||||
input_len=args.sonnet_input_len,
|
input_len=args.sonnet_input_len,
|
||||||
output_len=args.sonnet_output_len,
|
output_len=args.sonnet_output_len,
|
||||||
@@ -2173,6 +2185,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
|||||||
# all following datasets are implemented from the
|
# all following datasets are implemented from the
|
||||||
# HuggingFaceDataset base class
|
# HuggingFaceDataset base class
|
||||||
hf_kwargs = {}
|
hf_kwargs = {}
|
||||||
|
dataset_class: type[BenchmarkDataset]
|
||||||
if (
|
if (
|
||||||
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||||
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
|
||||||
@@ -2548,23 +2561,23 @@ class CustomDataset(BenchmarkDataset):
|
|||||||
|
|
||||||
if tokenizer is None:
|
if tokenizer is None:
|
||||||
new_output_len = 1
|
new_output_len = 1
|
||||||
|
elif output_len is None or output_len == -1:
|
||||||
|
# check that the request has an 'output_tokens' field
|
||||||
|
if "output_tokens" not in item:
|
||||||
|
raise ValueError(
|
||||||
|
"If no output length is provided the "
|
||||||
|
"custom dataset must contain an 'output_tokens' field."
|
||||||
|
)
|
||||||
|
# Use number of output tokens from the request data
|
||||||
|
try:
|
||||||
|
new_output_len = int(item["output_tokens"])
|
||||||
|
except (ValueError, TypeError) as e:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid value for 'output_tokens' in custom dataset: "
|
||||||
|
f"'{item['output_tokens']}'. Must be an integer."
|
||||||
|
) from e
|
||||||
else:
|
else:
|
||||||
new_output_len = output_len
|
new_output_len = output_len
|
||||||
if output_len is None or output_len == -1:
|
|
||||||
# check that the request has an 'output_tokens' field
|
|
||||||
if "output_tokens" not in item:
|
|
||||||
raise ValueError(
|
|
||||||
"If no output length is provided the "
|
|
||||||
"custom dataset must contain an 'output_tokens' field."
|
|
||||||
)
|
|
||||||
# Use number of output tokens from the request data
|
|
||||||
try:
|
|
||||||
new_output_len = int(item["output_tokens"])
|
|
||||||
except (ValueError, TypeError) as e:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid value for 'output_tokens' in custom dataset: "
|
|
||||||
f"'{item['output_tokens']}'. Must be an integer."
|
|
||||||
) from e
|
|
||||||
|
|
||||||
if tokenizer is None:
|
if tokenizer is None:
|
||||||
prompt_len = 1
|
prompt_len = 1
|
||||||
@@ -2797,11 +2810,15 @@ class CustomImageDataset(CustomDataset):
|
|||||||
self,
|
self,
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
num_requests: int,
|
num_requests: int,
|
||||||
output_len: int | None = None,
|
|
||||||
enable_multimodal_chat: bool = False,
|
|
||||||
ensure_client_side_data: bool = False,
|
|
||||||
request_id_prefix: str = "",
|
request_id_prefix: str = "",
|
||||||
no_oversample: bool = False,
|
no_oversample: bool = False,
|
||||||
|
lora_path: str | None = None,
|
||||||
|
max_loras: int | None = None,
|
||||||
|
output_len: int | None = None,
|
||||||
|
enable_multimodal_chat: bool = False,
|
||||||
|
skip_chat_template: bool = False,
|
||||||
|
chat_template_kwargs: dict | None = None,
|
||||||
|
ensure_client_side_data: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
# load all data if needed
|
# load all data if needed
|
||||||
@@ -2814,7 +2831,7 @@ class CustomImageDataset(CustomDataset):
|
|||||||
num_requests,
|
num_requests,
|
||||||
)
|
)
|
||||||
|
|
||||||
sampled_requests = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
@@ -2837,7 +2854,7 @@ class CustomImageDataset(CustomDataset):
|
|||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
expected_output_len=output_len,
|
expected_output_len=output_len or 0,
|
||||||
multi_modal_data=None,
|
multi_modal_data=None,
|
||||||
request_id=request_id_prefix + str(i),
|
request_id=request_id_prefix + str(i),
|
||||||
)
|
)
|
||||||
@@ -2863,7 +2880,7 @@ class CustomImageDataset(CustomDataset):
|
|||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
expected_output_len=output_len,
|
expected_output_len=output_len or 0,
|
||||||
multi_modal_data=mm_content,
|
multi_modal_data=mm_content,
|
||||||
request_id=request_id_prefix + str(i),
|
request_id=request_id_prefix + str(i),
|
||||||
)
|
)
|
||||||
@@ -2891,17 +2908,20 @@ class CustomAudioDataset(CustomDataset):
|
|||||||
self,
|
self,
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
num_requests: int,
|
num_requests: int,
|
||||||
output_len: int | None = None,
|
|
||||||
request_id_prefix: str = "",
|
request_id_prefix: str = "",
|
||||||
no_oversample: bool = False,
|
no_oversample: bool = False,
|
||||||
skip_chat_template: bool = False,
|
lora_path: str | None = None,
|
||||||
|
max_loras: int | None = None,
|
||||||
|
output_len: int | None = None,
|
||||||
enable_multimodal_chat: bool = False,
|
enable_multimodal_chat: bool = False,
|
||||||
|
skip_chat_template: bool = False,
|
||||||
|
chat_template_kwargs: dict | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
self.num_available_samples = len(self.data)
|
self.num_available_samples = len(self.data)
|
||||||
if num_requests <= 0:
|
if num_requests <= 0:
|
||||||
num_requests = self.num_available_samples
|
num_requests = self.num_available_samples
|
||||||
sampled_requests = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
@@ -2950,7 +2970,6 @@ class CustomAudioDataset(CustomDataset):
|
|||||||
prompt_len = (
|
prompt_len = (
|
||||||
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
|
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
|
||||||
)
|
)
|
||||||
new_output_len = output_len
|
|
||||||
if output_len is None or output_len == -1:
|
if output_len is None or output_len == -1:
|
||||||
if "output_tokens" not in item:
|
if "output_tokens" not in item:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -2958,6 +2977,8 @@ class CustomAudioDataset(CustomDataset):
|
|||||||
"custom dataset must contain an 'output_tokens' field."
|
"custom dataset must contain an 'output_tokens' field."
|
||||||
)
|
)
|
||||||
new_output_len = int(item["output_tokens"])
|
new_output_len = int(item["output_tokens"])
|
||||||
|
else:
|
||||||
|
new_output_len = output_len
|
||||||
sampled_requests.append(
|
sampled_requests.append(
|
||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
@@ -3015,10 +3036,30 @@ class SpecBench(CustomDataset):
|
|||||||
|
|
||||||
def sample(
|
def sample(
|
||||||
self,
|
self,
|
||||||
|
tokenizer: TokenizerLike,
|
||||||
|
num_requests: int,
|
||||||
|
request_id_prefix: str = "",
|
||||||
|
no_oversample: bool = False,
|
||||||
|
lora_path: str | None = None,
|
||||||
|
max_loras: int | None = None,
|
||||||
|
output_len: int | None = None,
|
||||||
|
enable_multimodal_chat: bool = False,
|
||||||
|
skip_chat_template: bool = False,
|
||||||
|
chat_template_kwargs: dict | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
# leverage CustomDataset sample
|
# leverage CustomDataset sample
|
||||||
return super().sample(
|
return super().sample(
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
num_requests=num_requests,
|
||||||
|
request_id_prefix=request_id_prefix,
|
||||||
|
no_oversample=no_oversample,
|
||||||
|
lora_path=lora_path,
|
||||||
|
max_loras=max_loras,
|
||||||
|
output_len=output_len,
|
||||||
|
enable_multimodal_chat=enable_multimodal_chat,
|
||||||
|
skip_chat_template=skip_chat_template,
|
||||||
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -3067,16 +3108,21 @@ class SonnetDataset(BenchmarkDataset):
|
|||||||
return_prompt_formatted: bool = False,
|
return_prompt_formatted: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
|
poem_lines = self.data
|
||||||
|
assert poem_lines is not None
|
||||||
# Calculate average token length for a poem line.
|
# Calculate average token length for a poem line.
|
||||||
tokenized_lines = [tokenizer(line).input_ids for line in self.data]
|
tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
|
||||||
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
|
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
|
||||||
|
|
||||||
# Build the base prompt.
|
# Build the base prompt.
|
||||||
base_prompt = "Pick as many lines as you can from these poem lines:\n"
|
base_prompt = "Pick as many lines as you can from these poem lines:\n"
|
||||||
base_msg = [{"role": "user", "content": base_prompt}]
|
base_msg = [{"role": "user", "content": base_prompt}]
|
||||||
base_fmt = tokenizer.apply_chat_template(
|
base_fmt = tokenizer.apply_chat_template(
|
||||||
base_msg, add_generation_prompt=True, tokenize=False
|
base_msg, # type: ignore[arg-type]
|
||||||
|
add_generation_prompt=True,
|
||||||
|
tokenize=False,
|
||||||
)
|
)
|
||||||
|
assert isinstance(base_fmt, str)
|
||||||
base_offset = len(tokenizer(base_fmt).input_ids)
|
base_offset = len(tokenizer(base_fmt).input_ids)
|
||||||
if input_len <= base_offset:
|
if input_len <= base_offset:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -3085,26 +3131,31 @@ class SonnetDataset(BenchmarkDataset):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Determine how many poem lines to use.
|
# Determine how many poem lines to use.
|
||||||
num_input_lines = round((input_len - base_offset) / avg_len)
|
num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
|
||||||
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
|
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
|
||||||
prefix_lines = self.data[:num_prefix_lines]
|
prefix_lines = poem_lines[:num_prefix_lines]
|
||||||
|
|
||||||
samples: list[SampleRequest] = []
|
samples: list[SampleRequest] = []
|
||||||
ind = 0
|
ind = 0
|
||||||
while len(samples) < num_requests:
|
while len(samples) < num_requests:
|
||||||
extra_lines = random.choices(
|
extra_lines = random.choices(
|
||||||
self.data, k=num_input_lines - num_prefix_lines
|
poem_lines, k=num_input_lines - num_prefix_lines
|
||||||
)
|
)
|
||||||
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
|
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
|
||||||
msg = [{"role": "user", "content": prompt}]
|
msg = [{"role": "user", "content": prompt}]
|
||||||
prompt_formatted = tokenizer.apply_chat_template(
|
prompt_formatted = tokenizer.apply_chat_template(
|
||||||
msg, add_generation_prompt=True, tokenize=False
|
msg, # type: ignore[arg-type]
|
||||||
|
add_generation_prompt=True,
|
||||||
|
tokenize=False,
|
||||||
)
|
)
|
||||||
|
assert isinstance(prompt_formatted, str)
|
||||||
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
||||||
if prompt_len <= input_len:
|
if prompt_len <= input_len:
|
||||||
samples.append(
|
samples.append(
|
||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=prompt_formatted if return_prompt_formatted else prompt,
|
prompt=(
|
||||||
|
prompt_formatted if return_prompt_formatted else prompt
|
||||||
|
),
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
expected_output_len=output_len,
|
expected_output_len=output_len,
|
||||||
request_id=request_id_prefix + str(ind),
|
request_id=request_id_prefix + str(ind),
|
||||||
@@ -3145,6 +3196,7 @@ class BurstGPTDataset(BenchmarkDataset):
|
|||||||
self.data = gpt4_df
|
self.data = gpt4_df
|
||||||
|
|
||||||
def _sample_loaded_data(self, num_requests: int) -> list:
|
def _sample_loaded_data(self, num_requests: int) -> list:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
if num_requests <= len(self.data):
|
if num_requests <= len(self.data):
|
||||||
data = self.data.sample(n=num_requests, random_state=self.random_seed)
|
data = self.data.sample(n=num_requests, random_state=self.random_seed)
|
||||||
else:
|
else:
|
||||||
@@ -3258,6 +3310,7 @@ class ConversationDataset(HuggingFaceDataset):
|
|||||||
enable_multimodal_chat: bool = False,
|
enable_multimodal_chat: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
# Filter examples with at least 2 conversations
|
# Filter examples with at least 2 conversations
|
||||||
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
@@ -3318,6 +3371,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
|
|||||||
enable_multimodal_chat: bool = False,
|
enable_multimodal_chat: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
# Filter examples with at least 2 conversations
|
# Filter examples with at least 2 conversations
|
||||||
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
@@ -3393,7 +3447,8 @@ class VisionArenaDataset(HuggingFaceDataset):
|
|||||||
|
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
|
|
||||||
sampled_requests = []
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
|
sampled_requests: list[SampleRequest] = []
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
@@ -3465,7 +3520,8 @@ class MMVUDataset(HuggingFaceDataset):
|
|||||||
|
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
|
|
||||||
sampled_requests = []
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
|
sampled_requests: list[SampleRequest] = []
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
@@ -3531,18 +3587,21 @@ class InstructCoderDataset(HuggingFaceDataset):
|
|||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
|
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
|
||||||
|
prompt_text = prompt
|
||||||
# apply template
|
# apply template
|
||||||
if not skip_chat_template:
|
if not skip_chat_template:
|
||||||
prompt = tokenizer.apply_chat_template(
|
prompt_text_result = tokenizer.apply_chat_template(
|
||||||
[{"role": "user", "content": prompt}],
|
[{"role": "user", "content": prompt}],
|
||||||
add_generation_prompt=True,
|
add_generation_prompt=True,
|
||||||
tokenize=False,
|
tokenize=False,
|
||||||
)
|
)
|
||||||
|
assert isinstance(prompt_text_result, str)
|
||||||
|
prompt_text = prompt_text_result
|
||||||
|
|
||||||
prompt_len = len(tokenizer(prompt).input_ids)
|
prompt_len = len(tokenizer(prompt_text).input_ids)
|
||||||
sampled_requests.append(
|
sampled_requests.append(
|
||||||
SampleRequest(
|
SampleRequest(
|
||||||
prompt=prompt,
|
prompt=prompt_text,
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
expected_output_len=output_len,
|
expected_output_len=output_len,
|
||||||
request_id=request_id_prefix + str(i),
|
request_id=request_id_prefix + str(i),
|
||||||
@@ -3554,6 +3613,7 @@ class InstructCoderDataset(HuggingFaceDataset):
|
|||||||
return sampled_requests
|
return sampled_requests
|
||||||
|
|
||||||
def sample_prompts(self, n: int) -> Iterator[str]:
|
def sample_prompts(self, n: int) -> Iterator[str]:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
for item in self.data.take(n):
|
for item in self.data.take(n):
|
||||||
prompt = (
|
prompt = (
|
||||||
f"{item['input']}\n\n{item['instruction']} Just output "
|
f"{item['input']}\n\n{item['instruction']} Just output "
|
||||||
@@ -3594,6 +3654,7 @@ class MTBenchDataset(HuggingFaceDataset):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
|
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
@@ -3654,7 +3715,8 @@ class HumanEvalDataset(HuggingFaceDataset):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
sampled_requests = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
|
assert self.data is not None, "Data must be loaded before sampling"
|
||||||
|
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
@@ -3714,7 +3776,8 @@ class GSM8KDataset(HuggingFaceDataset):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
sampled_requests = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
|
assert self.data is not None, "Data must be loaded before sampling"
|
||||||
|
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
@@ -3772,17 +3835,18 @@ class BlazeditDataset(HuggingFaceDataset):
|
|||||||
self,
|
self,
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
num_requests: int,
|
num_requests: int,
|
||||||
output_len: int | None = None,
|
|
||||||
skip_chat_template: bool = False,
|
|
||||||
request_id_prefix: str = "",
|
request_id_prefix: str = "",
|
||||||
no_oversample: bool = False,
|
no_oversample: bool = False,
|
||||||
|
output_len: int | None = None,
|
||||||
|
skip_chat_template: bool = False,
|
||||||
min_distance: float = 0.0,
|
min_distance: float = 0.0,
|
||||||
max_distance: float = 1.0,
|
max_distance: float = 1.0,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
sampled_requests = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
|
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
for i, item in enumerate(self.data):
|
for i, item in enumerate(self.data):
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
@@ -3810,11 +3874,13 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
|
|||||||
|
|
||||||
# apply template
|
# apply template
|
||||||
if not skip_chat_template:
|
if not skip_chat_template:
|
||||||
prompt = tokenizer.apply_chat_template(
|
prompt_result = tokenizer.apply_chat_template(
|
||||||
[{"role": "user", "content": prompt}],
|
[{"role": "user", "content": prompt}],
|
||||||
add_generation_prompt=True,
|
add_generation_prompt=True,
|
||||||
tokenize=False,
|
tokenize=False,
|
||||||
)
|
)
|
||||||
|
assert isinstance(prompt_result, str)
|
||||||
|
prompt = prompt_result
|
||||||
|
|
||||||
prompt_len = len(tokenizer(prompt).input_ids)
|
prompt_len = len(tokenizer(prompt).input_ids)
|
||||||
|
|
||||||
@@ -3858,6 +3924,7 @@ class AIMODataset(HuggingFaceDataset):
|
|||||||
output_len: int | None = None,
|
output_len: int | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
ind = 0
|
ind = 0
|
||||||
dynamic_output = output_len is None
|
dynamic_output = output_len is None
|
||||||
@@ -3970,6 +4037,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
|
|||||||
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
|
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
|
||||||
if formatting_prompt_func is None:
|
if formatting_prompt_func is None:
|
||||||
raise ValueError(f"Unsupported dataset path: {self.hf_name}")
|
raise ValueError(f"Unsupported dataset path: {self.hf_name}")
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
samples = []
|
samples = []
|
||||||
for i, sample in enumerate(self.data):
|
for i, sample in enumerate(self.data):
|
||||||
sample = formatting_prompt_func(sample)
|
sample = formatting_prompt_func(sample)
|
||||||
@@ -4061,12 +4129,14 @@ class ASRDataset(HuggingFaceDataset):
|
|||||||
def _disable_audio_decode(self) -> None:
|
def _disable_audio_decode(self) -> None:
|
||||||
from datasets import Audio
|
from datasets import Audio
|
||||||
|
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
self.data = self.data.cast_column("audio", Audio(decode=False))
|
self.data = self.data.cast_column("audio", Audio(decode=False))
|
||||||
|
|
||||||
def _materialize_local_audio_column(self) -> None:
|
def _materialize_local_audio_column(self) -> None:
|
||||||
local_path_root = Path(
|
local_path_root = Path(
|
||||||
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
|
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
|
||||||
)
|
)
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
self.data = self.data.map(
|
self.data = self.data.map(
|
||||||
lambda item: {
|
lambda item: {
|
||||||
"audio": str(local_path_root / item["url"]),
|
"audio": str(local_path_root / item["url"]),
|
||||||
@@ -4090,15 +4160,19 @@ class ASRDataset(HuggingFaceDataset):
|
|||||||
else:
|
else:
|
||||||
prompt = ""
|
prompt = ""
|
||||||
prompt_len = len(tokenizer(prompt).input_ids)
|
prompt_len = len(tokenizer(prompt).input_ids)
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
ind = 0
|
ind = 0
|
||||||
skipped = 0
|
skipped = 0
|
||||||
asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec")
|
asr_min_audio_len_sec: float = float(kwargs.get("asr_min_audio_len_sec") or 0.0)
|
||||||
asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec")
|
asr_max_audio_len_sec: float = float(
|
||||||
|
kwargs.get("asr_max_audio_len_sec") or float("inf")
|
||||||
|
)
|
||||||
durations = []
|
durations = []
|
||||||
for item in self.data:
|
for item in self.data:
|
||||||
if len(sampled_requests) >= num_requests:
|
if len(sampled_requests) >= num_requests:
|
||||||
break
|
break
|
||||||
|
mm_content: dict[str, Any]
|
||||||
audio = item["audio"]
|
audio = item["audio"]
|
||||||
if (
|
if (
|
||||||
isinstance(audio, dict)
|
isinstance(audio, dict)
|
||||||
@@ -4203,8 +4277,7 @@ class MLPerfDataset(HuggingFaceDataset):
|
|||||||
output_len: int | None = None,
|
output_len: int | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
# Force dynamic output length based on reference completion.
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
dynamic_output = output_len is None
|
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
ind = 0
|
ind = 0
|
||||||
|
|
||||||
@@ -4222,15 +4295,18 @@ class MLPerfDataset(HuggingFaceDataset):
|
|||||||
{"role": "user", "content": question},
|
{"role": "user", "content": question},
|
||||||
]
|
]
|
||||||
prompt_formatted = tokenizer.apply_chat_template(
|
prompt_formatted = tokenizer.apply_chat_template(
|
||||||
messages, add_generation_prompt=True, tokenize=False
|
messages, # type: ignore[arg-type]
|
||||||
|
add_generation_prompt=True,
|
||||||
|
tokenize=False,
|
||||||
)
|
)
|
||||||
|
assert isinstance(prompt_formatted, str)
|
||||||
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
prompt_len = len(tokenizer(prompt_formatted).input_ids)
|
||||||
|
|
||||||
# Determine output length from reference answer tokens.
|
# Determine output length from reference answer tokens.
|
||||||
ref_out_len = len(
|
ref_out_len = len(
|
||||||
tokenizer(reference_answer, add_special_tokens=False).input_ids
|
tokenizer(reference_answer, add_special_tokens=False).input_ids
|
||||||
)
|
)
|
||||||
expected_output_len = ref_out_len if dynamic_output else output_len
|
expected_output_len = ref_out_len if output_len is None else output_len
|
||||||
|
|
||||||
# Validate sequence lengths.
|
# Validate sequence lengths.
|
||||||
if not is_valid_sequence(prompt_len, expected_output_len):
|
if not is_valid_sequence(prompt_len, expected_output_len):
|
||||||
@@ -4371,6 +4447,7 @@ class MMStarDataset(HuggingFaceDataset):
|
|||||||
) -> list[SampleRequest]:
|
) -> list[SampleRequest]:
|
||||||
# If --hf-output-len is not set, use the default output length.
|
# If --hf-output-len is not set, use the default output length.
|
||||||
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
|
||||||
|
assert self.data is not None, "Dataset must be loaded before sampling"
|
||||||
sampled_requests: list[SampleRequest] = []
|
sampled_requests: list[SampleRequest] = []
|
||||||
|
|
||||||
for ind, item in enumerate(self.data):
|
for ind, item in enumerate(self.data):
|
||||||
@@ -4603,7 +4680,7 @@ class BFCLDataset(HuggingFaceDataset):
|
|||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
rendered = None
|
rendered = None
|
||||||
if rendered is not None:
|
if rendered is not None and isinstance(rendered, str):
|
||||||
prompt_len = len(tokenizer(rendered).input_ids)
|
prompt_len = len(tokenizer(rendered).input_ids)
|
||||||
else:
|
else:
|
||||||
text = "\n".join(m.get("content", "") for m in messages)
|
text = "\n".join(m.get("content", "") for m in messages)
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ from tqdm import tqdm
|
|||||||
|
|
||||||
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
||||||
from vllm.engine.arg_utils import EngineArgs
|
from vllm.engine.arg_utils import EngineArgs
|
||||||
from vllm.inputs import PromptType
|
from vllm.inputs import TextPrompt, TokensPrompt
|
||||||
from vllm.sampling_params import BeamSearchParams
|
from vllm.sampling_params import BeamSearchParams
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
def save_to_pytorch_benchmark_format(
|
def save_to_pytorch_benchmark_format(
|
||||||
@@ -30,7 +31,7 @@ def save_to_pytorch_benchmark_format(
|
|||||||
write_to_json(pt_file, pt_records)
|
write_to_json(pt_file, pt_records)
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser):
|
def add_cli_args(parser: FlexibleArgumentParser):
|
||||||
parser.add_argument("--input-len", type=int, default=32)
|
parser.add_argument("--input-len", type=int, default=32)
|
||||||
parser.add_argument("--output-len", type=int, default=128)
|
parser.add_argument("--output-len", type=int, default=128)
|
||||||
parser.add_argument("--batch-size", type=int, default=8)
|
parser.add_argument("--batch-size", type=int, default=8)
|
||||||
@@ -103,8 +104,9 @@ def main(args: argparse.Namespace):
|
|||||||
dummy_prompt_token_ids = np.random.randint(
|
dummy_prompt_token_ids = np.random.randint(
|
||||||
10000, size=(args.batch_size, args.input_len)
|
10000, size=(args.batch_size, args.input_len)
|
||||||
)
|
)
|
||||||
dummy_prompts: list[PromptType] = [
|
dummy_prompts: list[TokensPrompt | TextPrompt] = [
|
||||||
{"prompt_token_ids": batch} for batch in dummy_prompt_token_ids.tolist()
|
TokensPrompt(prompt_token_ids=batch)
|
||||||
|
for batch in dummy_prompt_token_ids.tolist()
|
||||||
]
|
]
|
||||||
|
|
||||||
def llm_generate():
|
def llm_generate():
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ class RequestFuncInput:
|
|||||||
logprobs: int | None = None
|
logprobs: int | None = None
|
||||||
extra_headers: dict | None = None
|
extra_headers: dict | None = None
|
||||||
extra_body: dict | None = None
|
extra_body: dict | None = None
|
||||||
multi_modal_content: dict | list[dict] | None = None
|
multi_modal_content: dict[str, Any] | list[dict[str, Any]] | None = None
|
||||||
ignore_eos: bool = False
|
ignore_eos: bool = False
|
||||||
language: str | None = None
|
language: str | None = None
|
||||||
request_id: str | None = None
|
request_id: str | None = None
|
||||||
@@ -272,13 +272,13 @@ def _get_chat_content(
|
|||||||
request_func_input: RequestFuncInput,
|
request_func_input: RequestFuncInput,
|
||||||
mm_position: Literal["first", "last"] = "last",
|
mm_position: Literal["first", "last"] = "last",
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
mm_contents = []
|
mm_contents: list[dict[str, Any]] = []
|
||||||
if request_func_input.multi_modal_content:
|
if request_func_input.multi_modal_content:
|
||||||
mm_content = request_func_input.multi_modal_content
|
mm_content = request_func_input.multi_modal_content
|
||||||
if isinstance(mm_content, list):
|
if isinstance(mm_content, list):
|
||||||
mm_contents.extend(request_func_input.multi_modal_content)
|
mm_contents.extend(mm_content)
|
||||||
elif isinstance(mm_content, dict):
|
elif isinstance(mm_content, dict):
|
||||||
mm_contents.append(request_func_input.multi_modal_content)
|
mm_contents.append(mm_content)
|
||||||
else:
|
else:
|
||||||
raise TypeError(
|
raise TypeError(
|
||||||
"multi_modal_content must be a dict or list[dict] for openai-chat"
|
"multi_modal_content must be a dict or list[dict] for openai-chat"
|
||||||
@@ -293,10 +293,11 @@ def _get_chat_content(
|
|||||||
for item in prompt
|
for item in prompt
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
|
prompt_dicts: list[dict[str, Any]] = prompt # type: ignore[assignment]
|
||||||
if mm_position == "first":
|
if mm_position == "first":
|
||||||
return mm_contents + prompt
|
return mm_contents + prompt_dicts
|
||||||
|
|
||||||
return prompt + mm_contents
|
return prompt_dicts + mm_contents
|
||||||
|
|
||||||
text_contents = [{"type": "text", "text": prompt}]
|
text_contents = [{"type": "text", "text": prompt}]
|
||||||
|
|
||||||
@@ -307,15 +308,15 @@ def _get_chat_content(
|
|||||||
|
|
||||||
|
|
||||||
def _is_chat_messages(prompt: Any) -> bool:
|
def _is_chat_messages(prompt: Any) -> bool:
|
||||||
return (
|
if not isinstance(prompt, list):
|
||||||
isinstance(prompt, list)
|
return False
|
||||||
and prompt
|
if not prompt:
|
||||||
and all(
|
return False
|
||||||
isinstance(item, dict)
|
return all(
|
||||||
and isinstance(item.get("role"), str)
|
isinstance(item, dict)
|
||||||
and isinstance(item.get("content"), (str, list))
|
and isinstance(item.get("role"), str)
|
||||||
for item in prompt
|
and isinstance(item.get("content"), (str, list))
|
||||||
)
|
for item in prompt
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -325,7 +326,7 @@ def _get_chat_messages(
|
|||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
prompt = request_func_input.prompt
|
prompt = request_func_input.prompt
|
||||||
if _is_chat_messages(prompt):
|
if _is_chat_messages(prompt):
|
||||||
return prompt
|
return prompt # type: ignore[return-value]
|
||||||
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@@ -385,8 +386,8 @@ async def async_request_openai_chat_completions(
|
|||||||
if not chunk_bytes:
|
if not chunk_bytes:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
messages = handler.add_chunk(chunk_bytes)
|
message_strings = handler.add_chunk(chunk_bytes)
|
||||||
for message in messages:
|
for message in message_strings:
|
||||||
# NOTE: SSE comments (often used as pings) start with
|
# NOTE: SSE comments (often used as pings) start with
|
||||||
# a colon. These are not JSON data payload and should
|
# a colon. These are not JSON data payload and should
|
||||||
# be skipped.
|
# be skipped.
|
||||||
@@ -790,7 +791,7 @@ async def async_request_infinity_embeddings(
|
|||||||
api_url = request_func_input.api_url
|
api_url = request_func_input.api_url
|
||||||
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
|
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
|
||||||
|
|
||||||
payload = {
|
payload: dict[str, Any] = {
|
||||||
"model": request_func_input.model_name
|
"model": request_func_input.model_name
|
||||||
if request_func_input.model_name
|
if request_func_input.model_name
|
||||||
else request_func_input.model,
|
else request_func_input.model,
|
||||||
@@ -849,7 +850,10 @@ async def async_request_vllm_pooling(
|
|||||||
"truncate_prompt_tokens": -1,
|
"truncate_prompt_tokens": -1,
|
||||||
}
|
}
|
||||||
|
|
||||||
payload = payload | request_func_input.prompt
|
if isinstance(request_func_input.prompt, dict):
|
||||||
|
payload = payload | request_func_input.prompt
|
||||||
|
else:
|
||||||
|
payload["input"] = request_func_input.prompt
|
||||||
|
|
||||||
_update_payload_common(payload, request_func_input)
|
_update_payload_common(payload, request_func_input)
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ def extract_field(
|
|||||||
if field_name in extra_info:
|
if field_name in extra_info:
|
||||||
return extra_info[field_name]
|
return extra_info[field_name]
|
||||||
|
|
||||||
v = args
|
v: Any = args
|
||||||
# For example, args.compilation_config.mode
|
# For example, args.compilation_config.mode
|
||||||
for nested_field in field_name.split("."):
|
for nested_field in field_name.split("."):
|
||||||
if not hasattr(v, nested_field):
|
if not hasattr(v, nested_field):
|
||||||
@@ -43,7 +43,7 @@ def convert_to_pytorch_benchmark_format(
|
|||||||
on metric per record
|
on metric per record
|
||||||
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
|
https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
|
||||||
"""
|
"""
|
||||||
records = []
|
records: list[Any] = []
|
||||||
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
|
if not os.environ.get("SAVE_TO_PYTORCH_BENCHMARK_FORMAT", False):
|
||||||
return records
|
return records
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from vllm.benchmarks.datasets import (
|
|||||||
)
|
)
|
||||||
from vllm.benchmarks.throughput import get_requests
|
from vllm.benchmarks.throughput import get_requests
|
||||||
from vllm.engine.arg_utils import EngineArgs
|
from vllm.engine.arg_utils import EngineArgs
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.gc_utils import freeze_gc_heap
|
from vllm.utils.gc_utils import freeze_gc_heap
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
@@ -369,7 +370,7 @@ def benchmark_multimodal_processor(
|
|||||||
return benchmark_result
|
return benchmark_result
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||||
"""Add CLI arguments for the multimodal processor benchmark."""
|
"""Add CLI arguments for the multimodal processor benchmark."""
|
||||||
from vllm.engine.arg_utils import EngineArgs
|
from vllm.engine.arg_utils import EngineArgs
|
||||||
|
|
||||||
@@ -532,7 +533,7 @@ def main(args: argparse.Namespace) -> None:
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Benchmark mm processor latency")
|
parser = FlexibleArgumentParser(description="Benchmark mm processor latency")
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
main(args)
|
main(args)
|
||||||
|
|||||||
+37
-14
@@ -30,7 +30,7 @@ import ssl
|
|||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
import warnings
|
import warnings
|
||||||
from collections.abc import AsyncGenerator, Iterable
|
from collections.abc import AsyncGenerator, Iterable, Iterator
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
@@ -52,6 +52,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
|
|||||||
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
||||||
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.gc_utils import freeze_gc_heap
|
from vllm.utils.gc_utils import freeze_gc_heap
|
||||||
from vllm.utils.network_utils import join_host_port
|
from vllm.utils.network_utils import join_host_port
|
||||||
|
|
||||||
@@ -129,6 +130,7 @@ async def _align_prompts_to_server_tokenizer(
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _fix_one(req: SampleRequest) -> SampleRequest:
|
async def _fix_one(req: SampleRequest) -> SampleRequest:
|
||||||
|
assert isinstance(req.prompt, str)
|
||||||
tokens = await _tokenize(req.prompt)
|
tokens = await _tokenize(req.prompt)
|
||||||
if len(tokens) <= req.prompt_len:
|
if len(tokens) <= req.prompt_len:
|
||||||
return req
|
return req
|
||||||
@@ -360,7 +362,7 @@ class EmbedBenchmarkMetrics:
|
|||||||
mean_e2el_ms: float
|
mean_e2el_ms: float
|
||||||
std_e2el_ms: float
|
std_e2el_ms: float
|
||||||
median_e2el_ms: float
|
median_e2el_ms: float
|
||||||
percentiles_e2el_ms: float
|
percentiles_e2el_ms: list[tuple[float, float]]
|
||||||
|
|
||||||
|
|
||||||
def _get_current_request_rate(
|
def _get_current_request_rate(
|
||||||
@@ -433,8 +435,8 @@ async def get_request(
|
|||||||
assert total_requests > 0, "No requests provided."
|
assert total_requests > 0, "No requests provided."
|
||||||
|
|
||||||
# Precompute delays among requests to minimize request send laggings
|
# Precompute delays among requests to minimize request send laggings
|
||||||
request_rates = []
|
request_rates: list[float] = []
|
||||||
delay_ts = []
|
delay_ts: list[float] = []
|
||||||
|
|
||||||
# if the traces have timing info then:
|
# if the traces have timing info then:
|
||||||
if not self_timed:
|
if not self_timed:
|
||||||
@@ -484,7 +486,10 @@ async def get_request(
|
|||||||
else:
|
else:
|
||||||
for request_index, request in enumerate(input_requests):
|
for request_index, request in enumerate(input_requests):
|
||||||
# this is cumulative running ts, from which sleep is calculated later
|
# this is cumulative running ts, from which sleep is calculated later
|
||||||
delay_ts.append(request.timestamp)
|
if request.timestamp is not None:
|
||||||
|
delay_ts.append(request.timestamp)
|
||||||
|
else:
|
||||||
|
delay_ts.append(0.0)
|
||||||
# TODO: there is no notion of RPS here, may be we can calculate
|
# TODO: there is no notion of RPS here, may be we can calculate
|
||||||
# from the trace.
|
# from the trace.
|
||||||
request_rates.append(0.0)
|
request_rates.append(0.0)
|
||||||
@@ -599,7 +604,7 @@ def calculate_metrics(
|
|||||||
)
|
)
|
||||||
actual_output_lens.append(output_len)
|
actual_output_lens.append(output_len)
|
||||||
total_input += outputs[i].prompt_len
|
total_input += outputs[i].prompt_len
|
||||||
tpot = 0
|
tpot = 0.0
|
||||||
if output_len > 1:
|
if output_len > 1:
|
||||||
latency_minus_ttft = outputs[i].latency - outputs[i].ttft
|
latency_minus_ttft = outputs[i].latency - outputs[i].ttft
|
||||||
tpot = latency_minus_ttft / (output_len - 1)
|
tpot = latency_minus_ttft / (output_len - 1)
|
||||||
@@ -894,11 +899,12 @@ async def benchmark(
|
|||||||
|
|
||||||
print("Starting main benchmark run...")
|
print("Starting main benchmark run...")
|
||||||
|
|
||||||
|
lora_modules_iter: Iterator[str] | None = None
|
||||||
if lora_modules:
|
if lora_modules:
|
||||||
lora_modules_list = list(lora_modules)
|
lora_modules_list = list(lora_modules)
|
||||||
if lora_assignment == "round-robin":
|
if lora_assignment == "round-robin":
|
||||||
# Deterministic round-robin assignment across requests.
|
# Deterministic round-robin assignment across requests.
|
||||||
lora_modules = iter(
|
lora_modules_iter = iter(
|
||||||
[
|
[
|
||||||
lora_modules_list[i % len(lora_modules_list)]
|
lora_modules_list[i % len(lora_modules_list)]
|
||||||
for i in range(len(input_requests))
|
for i in range(len(input_requests))
|
||||||
@@ -906,7 +912,7 @@ async def benchmark(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# For each input request, choose a LoRA module at random.
|
# For each input request, choose a LoRA module at random.
|
||||||
lora_modules = iter(
|
lora_modules_iter = iter(
|
||||||
[random.choice(lora_modules_list) for _ in range(len(input_requests))]
|
[random.choice(lora_modules_list) for _ in range(len(input_requests))]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1004,19 +1010,23 @@ async def benchmark(
|
|||||||
)
|
)
|
||||||
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
||||||
req_model_id, req_model_name = model_id, model_name
|
req_model_id, req_model_name = model_id, model_name
|
||||||
if lora_modules:
|
if lora_modules_iter:
|
||||||
req_lora_module = next(lora_modules)
|
req_lora_module = next(lora_modules_iter)
|
||||||
req_model_id, req_model_name = req_lora_module, req_lora_module
|
req_model_id, req_model_name = req_lora_module, req_lora_module
|
||||||
|
|
||||||
|
mm_content_typed: dict[str, Any] | list[dict[str, Any]] | None = None
|
||||||
|
if isinstance(mm_content, (dict, list)):
|
||||||
|
mm_content_typed = mm_content
|
||||||
|
|
||||||
request_func_input = RequestFuncInput(
|
request_func_input = RequestFuncInput(
|
||||||
model=req_model_id,
|
model=req_model_id,
|
||||||
model_name=req_model_name,
|
model_name=req_model_name,
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
api_url=api_url,
|
api_url=api_url,
|
||||||
prompt_len=prompt_len,
|
prompt_len=prompt_len,
|
||||||
output_len=output_len,
|
output_len=output_len or 0,
|
||||||
logprobs=logprobs,
|
logprobs=logprobs,
|
||||||
multi_modal_content=mm_content,
|
multi_modal_content=mm_content_typed,
|
||||||
ignore_eos=ignore_eos,
|
ignore_eos=ignore_eos,
|
||||||
extra_headers=extra_headers,
|
extra_headers=extra_headers,
|
||||||
extra_body=per_request_extra_body,
|
extra_body=per_request_extra_body,
|
||||||
@@ -1107,6 +1117,8 @@ async def benchmark(
|
|||||||
"committed_per_step": delta_committed / denoising_steps,
|
"committed_per_step": delta_committed / denoising_steps,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
metrics: BenchmarkMetrics | EmbedBenchmarkMetrics
|
||||||
|
actual_output_lens: list[int] | int
|
||||||
if task_type == TaskType.GENERATION:
|
if task_type == TaskType.GENERATION:
|
||||||
metrics, actual_output_lens = calculate_metrics(
|
metrics, actual_output_lens = calculate_metrics(
|
||||||
input_requests=input_requests,
|
input_requests=input_requests,
|
||||||
@@ -1140,7 +1152,7 @@ async def benchmark(
|
|||||||
"Request throughput (req/s):", metrics.request_throughput
|
"Request throughput (req/s):", metrics.request_throughput
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if goodput_config_dict:
|
if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
|
||||||
print(
|
print(
|
||||||
"{:<40} {:<10.2f}".format(
|
"{:<40} {:<10.2f}".format(
|
||||||
"Request goodput (req/s):", metrics.request_goodput
|
"Request goodput (req/s):", metrics.request_goodput
|
||||||
@@ -1177,6 +1189,7 @@ async def benchmark(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
result: dict[str, Any]
|
||||||
if isinstance(metrics, BenchmarkMetrics):
|
if isinstance(metrics, BenchmarkMetrics):
|
||||||
result = {
|
result = {
|
||||||
"duration": benchmark_duration,
|
"duration": benchmark_duration,
|
||||||
@@ -1462,7 +1475,7 @@ def compute_result_filename(
|
|||||||
return file_name
|
return file_name
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser):
|
def add_cli_args(parser: FlexibleArgumentParser):
|
||||||
add_dataset_parser(parser)
|
add_dataset_parser(parser)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--label",
|
"--label",
|
||||||
@@ -2023,6 +2036,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
args.self_timed = False
|
args.self_timed = False
|
||||||
|
|
||||||
# Load the dataset.
|
# Load the dataset.
|
||||||
|
assert tokenizer is not None, "Tokenizer must be initialized before loading dataset"
|
||||||
input_requests = get_samples(args, tokenizer)
|
input_requests = get_samples(args, tokenizer)
|
||||||
|
|
||||||
if args.dataset_name in ("random", "prefix_repetition"):
|
if args.dataset_name in ("random", "prefix_repetition"):
|
||||||
@@ -2154,6 +2168,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
|
|
||||||
# Generate timeline plot if requested
|
# Generate timeline plot if requested
|
||||||
if args.plot_timeline:
|
if args.plot_timeline:
|
||||||
|
assert file_name is not None, (
|
||||||
|
"file_name must be set when plot_timeline is enabled"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
from vllm.benchmarks.plot import generate_timeline_plot
|
from vllm.benchmarks.plot import generate_timeline_plot
|
||||||
|
|
||||||
@@ -2200,6 +2217,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
|
|
||||||
# Generate dataset statistics plot if requested
|
# Generate dataset statistics plot if requested
|
||||||
if args.plot_dataset_stats:
|
if args.plot_dataset_stats:
|
||||||
|
assert file_name is not None, (
|
||||||
|
"file_name must be set when plot_dataset_stats is enabled"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
from vllm.benchmarks.plot import generate_dataset_stats_plot
|
from vllm.benchmarks.plot import generate_dataset_stats_plot
|
||||||
|
|
||||||
@@ -2249,6 +2269,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
|
|
||||||
# Save to file
|
# Save to file
|
||||||
if args.save_result or args.append_result:
|
if args.save_result or args.append_result:
|
||||||
|
assert file_name is not None, (
|
||||||
|
"file_name must be set when save_result or append_result is enabled"
|
||||||
|
)
|
||||||
with open(
|
with open(
|
||||||
file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
|
file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
|
||||||
) as outfile:
|
) as outfile:
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from vllm.benchmarks.lib.utils import (
|
|||||||
write_to_json,
|
write_to_json,
|
||||||
)
|
)
|
||||||
from vllm.engine.arg_utils import EngineArgs
|
from vllm.engine.arg_utils import EngineArgs
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
PERCENTAGES = [10, 25, 50, 75, 90, 99]
|
PERCENTAGES = [10, 25, 50, 75, 90, 99]
|
||||||
|
|
||||||
@@ -190,7 +191,7 @@ def save_to_pytorch_benchmark_format(
|
|||||||
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
|
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser):
|
def add_cli_args(parser: FlexibleArgumentParser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--num-iters-cold",
|
"--num-iters-cold",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -234,7 +235,7 @@ def main(args: argparse.Namespace):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Create a queue for inter-process communication
|
# Create a queue for inter-process communication
|
||||||
result_queue = multiprocessing.Queue()
|
result_queue: multiprocessing.Queue[Any] = multiprocessing.Queue()
|
||||||
process = multiprocessing.Process(
|
process = multiprocessing.Process(
|
||||||
target=run_startup_in_subprocess,
|
target=run_startup_in_subprocess,
|
||||||
args=(
|
args=(
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
from .plot import SweepPlotArgs
|
from .plot import SweepPlotArgs
|
||||||
from .plot import main as plot_main
|
from .plot import main as plot_main
|
||||||
@@ -24,7 +25,7 @@ SUBCOMMANDS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser):
|
def add_cli_args(parser: FlexibleArgumentParser):
|
||||||
subparsers = parser.add_subparsers(required=True, dest="sweep_type")
|
subparsers = parser.add_subparsers(required=True, dest="sweep_type")
|
||||||
|
|
||||||
for cmd, entrypoint in SUBCOMMANDS:
|
for cmd, entrypoint in SUBCOMMANDS:
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
class ParameterSweep(list["ParameterSweepItem"]):
|
class ParameterSweep(list["ParameterSweepItem"]):
|
||||||
@@ -62,8 +61,8 @@ class ParameterSweepItem(dict[str, object]):
|
|||||||
|
|
||||||
return cls(record)
|
return cls(record)
|
||||||
|
|
||||||
def __or__(self, other: dict[str, Any]):
|
def __or__(self, other: dict[str, object], /) -> "ParameterSweepItem": # type: ignore[override]
|
||||||
return type(self)(super().__or__(other))
|
return ParameterSweepItem(super().__or__(other))
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def name(self) -> str:
|
def name(self) -> str:
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, ClassVar
|
|||||||
|
|
||||||
from typing_extensions import Self, override
|
from typing_extensions import Self, override
|
||||||
|
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.collection_utils import full_groupby
|
from vllm.utils.collection_utils import full_groupby
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
@@ -51,6 +52,7 @@ class PlotFilterBase(ABC):
|
|||||||
class PlotEqualTo(PlotFilterBase):
|
class PlotEqualTo(PlotFilterBase):
|
||||||
@override
|
@override
|
||||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||||
|
target: float | str
|
||||||
try:
|
try:
|
||||||
target = float(self.target)
|
target = float(self.target)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
@@ -63,6 +65,7 @@ class PlotEqualTo(PlotFilterBase):
|
|||||||
class PlotNotEqualTo(PlotFilterBase):
|
class PlotNotEqualTo(PlotFilterBase):
|
||||||
@override
|
@override
|
||||||
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
|
||||||
|
target: float | str
|
||||||
try:
|
try:
|
||||||
target = float(self.target)
|
target = float(self.target)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
@@ -182,7 +185,7 @@ def _convert_inf_nan_strings(data: list[dict[str, object]]) -> list[dict[str, ob
|
|||||||
"""
|
"""
|
||||||
converted_data = []
|
converted_data = []
|
||||||
for record in data:
|
for record in data:
|
||||||
converted_record = {}
|
converted_record: dict[str, object] = {}
|
||||||
for key, value in record.items():
|
for key, value in record.items():
|
||||||
if isinstance(value, str):
|
if isinstance(value, str):
|
||||||
if value in ["inf", "-inf", "nan"]:
|
if value in ["inf", "-inf", "nan"]:
|
||||||
@@ -531,7 +534,7 @@ class SweepPlotArgs:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"EXPERIMENT_DIR",
|
"EXPERIMENT_DIR",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -682,7 +685,7 @@ def main(args: argparse.Namespace):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description=SweepPlotArgs.parser_help)
|
parser = FlexibleArgumentParser(description=SweepPlotArgs.parser_help)
|
||||||
SweepPlotArgs.add_cli_args(parser)
|
SweepPlotArgs.add_cli_args(parser)
|
||||||
|
|
||||||
main(parser.parse_args())
|
main(parser.parse_args())
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from functools import partial
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, ClassVar
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.collection_utils import full_groupby
|
from vllm.utils.collection_utils import full_groupby
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
@@ -342,7 +343,7 @@ class SweepPlotParetoArgs:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser):
|
def add_cli_args(cls, parser: FlexibleArgumentParser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"EXPERIMENT_DIR",
|
"EXPERIMENT_DIR",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -394,7 +395,7 @@ def main(args: argparse.Namespace):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description=SweepPlotParetoArgs.parser_help)
|
parser = FlexibleArgumentParser(description=SweepPlotParetoArgs.parser_help)
|
||||||
SweepPlotParetoArgs.add_cli_args(parser)
|
SweepPlotParetoArgs.add_cli_args(parser)
|
||||||
|
|
||||||
main(parser.parse_args())
|
main(parser.parse_args())
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from datetime import datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import ClassVar
|
from typing import ClassVar
|
||||||
|
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
from .param_sweep import ParameterSweep, ParameterSweepItem
|
from .param_sweep import ParameterSweep, ParameterSweepItem
|
||||||
@@ -368,7 +369,7 @@ class SweepServeArgs:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--serve-cmd",
|
"--serve-cmd",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -531,7 +532,7 @@ def main(args: argparse.Namespace):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description=SweepServeArgs.parser_help)
|
parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help)
|
||||||
SweepServeArgs.add_cli_args(parser)
|
SweepServeArgs.add_cli_args(parser)
|
||||||
|
|
||||||
main(parser.parse_args())
|
main(parser.parse_args())
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import numpy as np
|
|||||||
from typing_extensions import assert_never
|
from typing_extensions import assert_never
|
||||||
|
|
||||||
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
|
from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
from .param_sweep import ParameterSweep, ParameterSweepItem
|
from .param_sweep import ParameterSweep, ParameterSweepItem
|
||||||
@@ -273,7 +274,7 @@ class SweepServeWorkloadArgs(SweepServeArgs):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||||
parser = super().add_cli_args(parser)
|
parser = super().add_cli_args(parser)
|
||||||
|
|
||||||
workload_group = parser.add_argument_group("workload options")
|
workload_group = parser.add_argument_group("workload options")
|
||||||
@@ -322,7 +323,7 @@ def main(args: argparse.Namespace):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description=SweepServeWorkloadArgs.parser_help)
|
parser = FlexibleArgumentParser(description=SweepServeWorkloadArgs.parser_help)
|
||||||
SweepServeWorkloadArgs.add_cli_args(parser)
|
SweepServeWorkloadArgs.add_cli_args(parser)
|
||||||
|
|
||||||
main(parser.parse_args())
|
main(parser.parse_args())
|
||||||
|
|||||||
@@ -317,7 +317,7 @@ class SweepStartupArgs:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--startup-cmd",
|
"--startup-cmd",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -438,6 +438,6 @@ def main(args: argparse.Namespace):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description=SweepStartupArgs.parser_help)
|
parser = FlexibleArgumentParser(description=SweepStartupArgs.parser_help)
|
||||||
SweepStartupArgs.add_cli_args(parser)
|
SweepStartupArgs.add_cli_args(parser)
|
||||||
main(parser.parse_args())
|
main(parser.parse_args())
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
|
|||||||
from vllm.benchmarks.datasets import (
|
from vllm.benchmarks.datasets import (
|
||||||
AIMODataset,
|
AIMODataset,
|
||||||
ASRDataset,
|
ASRDataset,
|
||||||
|
BenchmarkDataset,
|
||||||
BurstGPTDataset,
|
BurstGPTDataset,
|
||||||
ConversationDataset,
|
ConversationDataset,
|
||||||
InstructCoderDataset,
|
InstructCoderDataset,
|
||||||
@@ -41,6 +42,7 @@ from vllm.outputs import RequestOutput
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.sampling_params import BeamSearchParams
|
from vllm.sampling_params import BeamSearchParams
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.utils.async_utils import merge_async_iterators
|
from vllm.utils.async_utils import merge_async_iterators
|
||||||
|
|
||||||
|
|
||||||
@@ -102,13 +104,15 @@ def _run_vllm_requests(
|
|||||||
|
|
||||||
prompts: list[TextPrompt | TokensPrompt] = []
|
prompts: list[TextPrompt | TokensPrompt] = []
|
||||||
sampling_params: list[SamplingParams] = []
|
sampling_params: list[SamplingParams] = []
|
||||||
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
|
lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None
|
||||||
for request in requests:
|
for request in requests:
|
||||||
prompt = (
|
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
|
||||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
prompt_token_ids = request.prompt["prompt_token_ids"]
|
||||||
if "prompt_token_ids" in request.prompt
|
assert isinstance(prompt_token_ids, list)
|
||||||
else TextPrompt(prompt=request.prompt)
|
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||||
)
|
else:
|
||||||
|
assert isinstance(request.prompt, str)
|
||||||
|
prompt = TextPrompt(prompt=request.prompt)
|
||||||
if request.multi_modal_data:
|
if request.multi_modal_data:
|
||||||
assert isinstance(request.multi_modal_data, dict)
|
assert isinstance(request.multi_modal_data, dict)
|
||||||
prompt["multi_modal_data"] = request.multi_modal_data
|
prompt["multi_modal_data"] = request.multi_modal_data
|
||||||
@@ -159,7 +163,20 @@ def _run_vllm_requests(
|
|||||||
end = time.perf_counter()
|
end = time.perf_counter()
|
||||||
else:
|
else:
|
||||||
assert lora_requests is None, "BeamSearch API does not support LoRA"
|
assert lora_requests is None, "BeamSearch API does not support LoRA"
|
||||||
beam_prompts = [request.prompt for request in requests]
|
beam_prompts: list[TextPrompt | TokensPrompt] = []
|
||||||
|
for request in requests:
|
||||||
|
if isinstance(request.prompt, str):
|
||||||
|
beam_prompts.append(TextPrompt(prompt=request.prompt))
|
||||||
|
elif (
|
||||||
|
isinstance(request.prompt, dict)
|
||||||
|
and "prompt_token_ids" in request.prompt
|
||||||
|
):
|
||||||
|
token_ids = request.prompt["prompt_token_ids"]
|
||||||
|
assert isinstance(token_ids, list)
|
||||||
|
beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids))
|
||||||
|
else:
|
||||||
|
# Fallback: convert to string
|
||||||
|
beam_prompts.append(TextPrompt(prompt=str(request.prompt)))
|
||||||
# output_len should be the same for all requests.
|
# output_len should be the same for all requests.
|
||||||
output_len = requests[0].expected_output_len
|
output_len = requests[0].expected_output_len
|
||||||
for request in requests:
|
for request in requests:
|
||||||
@@ -268,7 +285,7 @@ def _run_vllm_chat_requests(
|
|||||||
llm.wake_up(tags=["scheduling"])
|
llm.wake_up(tags=["scheduling"])
|
||||||
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
|
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
|
||||||
else:
|
else:
|
||||||
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
|
outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type]
|
||||||
|
|
||||||
if do_profile:
|
if do_profile:
|
||||||
llm.stop_profile()
|
llm.stop_profile()
|
||||||
@@ -339,11 +356,13 @@ async def _run_vllm_async_requests(
|
|||||||
sampling_params: list[SamplingParams] = []
|
sampling_params: list[SamplingParams] = []
|
||||||
lora_requests: list[LoRARequest | None] = []
|
lora_requests: list[LoRARequest | None] = []
|
||||||
for request in requests:
|
for request in requests:
|
||||||
prompt = (
|
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
|
||||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
prompt_token_ids = request.prompt["prompt_token_ids"]
|
||||||
if "prompt_token_ids" in request.prompt
|
assert isinstance(prompt_token_ids, list)
|
||||||
else TextPrompt(prompt=request.prompt)
|
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||||
)
|
else:
|
||||||
|
assert isinstance(request.prompt, str)
|
||||||
|
prompt = TextPrompt(prompt=request.prompt)
|
||||||
|
|
||||||
if request.multi_modal_data:
|
if request.multi_modal_data:
|
||||||
assert isinstance(request.multi_modal_data, dict)
|
assert isinstance(request.multi_modal_data, dict)
|
||||||
@@ -366,9 +385,11 @@ async def _run_vllm_async_requests(
|
|||||||
start = time.perf_counter()
|
start = time.perf_counter()
|
||||||
if do_profile:
|
if do_profile:
|
||||||
await llm.start_profile()
|
await llm.start_profile()
|
||||||
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
|
for i, (prompt_item, sp, lr) in enumerate(
|
||||||
|
zip(prompts, sampling_params, lora_requests)
|
||||||
|
):
|
||||||
generator = llm.generate(
|
generator = llm.generate(
|
||||||
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
|
prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
|
||||||
)
|
)
|
||||||
generators.append(generator)
|
generators.append(generator)
|
||||||
all_gens = merge_async_iterators(*generators)
|
all_gens = merge_async_iterators(*generators)
|
||||||
@@ -440,6 +461,7 @@ def _run_hf_requests(
|
|||||||
prompt_len = requests[i].prompt_len
|
prompt_len = requests[i].prompt_len
|
||||||
output_len = requests[i].expected_output_len
|
output_len = requests[i].expected_output_len
|
||||||
# Add the prompt to the batch.
|
# Add the prompt to the batch.
|
||||||
|
assert isinstance(prompt, str), "Prompt must be a string for HF backend"
|
||||||
batch.append(prompt)
|
batch.append(prompt)
|
||||||
max_prompt_len = max(max_prompt_len, prompt_len)
|
max_prompt_len = max(max_prompt_len, prompt_len)
|
||||||
max_output_len = max(max_output_len, output_len)
|
max_output_len = max(max_output_len, output_len)
|
||||||
@@ -500,6 +522,7 @@ def save_to_pytorch_benchmark_format(
|
|||||||
|
|
||||||
def get_requests(args, tokenizer):
|
def get_requests(args, tokenizer):
|
||||||
# Common parameters for all dataset types.
|
# Common parameters for all dataset types.
|
||||||
|
dataset_cls: type[BenchmarkDataset]
|
||||||
common_kwargs = {
|
common_kwargs = {
|
||||||
"dataset_path": args.dataset_path,
|
"dataset_path": args.dataset_path,
|
||||||
"random_seed": args.seed,
|
"random_seed": args.seed,
|
||||||
@@ -875,7 +898,7 @@ def validate_args(args):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_cli_args(parser: argparse.ArgumentParser):
|
def add_cli_args(parser: FlexibleArgumentParser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--backend",
|
"--backend",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -1178,7 +1201,11 @@ def main(args: argparse.Namespace):
|
|||||||
total_prompt_tokens += (
|
total_prompt_tokens += (
|
||||||
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
|
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
|
||||||
)
|
)
|
||||||
total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o)
|
total_output_tokens += sum(
|
||||||
|
len(o.token_ids)
|
||||||
|
for o in ro.outputs
|
||||||
|
if o is not None and o.token_ids is not None
|
||||||
|
)
|
||||||
total_num_tokens = total_prompt_tokens + total_output_tokens
|
total_num_tokens = total_prompt_tokens + total_output_tokens
|
||||||
else:
|
else:
|
||||||
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)
|
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
from vllm.entrypoints.cli.types import CLISubcommand
|
from vllm.entrypoints.cli.types import CLISubcommand
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkSubcommandBase(CLISubcommand):
|
class BenchmarkSubcommandBase(CLISubcommand):
|
||||||
@@ -11,7 +12,7 @@ class BenchmarkSubcommandBase(CLISubcommand):
|
|||||||
help: str
|
help: str
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
"""Add the CLI arguments to the parser."""
|
"""Add the CLI arguments to the parser."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.latency import add_cli_args, main
|
from vllm.benchmarks.latency import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
|
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark the latency of a single batch of requests."
|
help = "Benchmark the latency of a single batch of requests."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.mm_processor import add_cli_args, main
|
from vllm.benchmarks.mm_processor import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark multimodal processor latency across different configurations."
|
help = "Benchmark multimodal processor latency across different configurations."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.serve import add_cli_args, main
|
from vllm.benchmarks.serve import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark the online serving throughput."
|
help = "Benchmark the online serving throughput."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.startup import add_cli_args, main
|
from vllm.benchmarks.startup import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
|
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark the startup time of vLLM models."
|
help = "Benchmark the startup time of vLLM models."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.sweep.cli import add_cli_args, main
|
from vllm.benchmarks.sweep.cli import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
|
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark for a parameter sweep."
|
help = "Benchmark for a parameter sweep."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import argparse
|
|||||||
|
|
||||||
from vllm.benchmarks.throughput import add_cli_args, main
|
from vllm.benchmarks.throughput import add_cli_args, main
|
||||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
|
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
|
||||||
@@ -13,7 +14,7 @@ class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
|
|||||||
help = "Benchmark offline inference throughput."
|
help = "Benchmark offline inference throughput."
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||||
add_cli_args(parser)
|
add_cli_args(parser)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user