[MyPy] Fix mypy for vllm/benchmarks (#39896)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Martin Hickey
2026-06-23 15:22:29 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 547d2c40d7
commit 275b43183c
24 changed files with 297 additions and 151 deletions
+141 -64
View File
@@ -81,7 +81,7 @@ class SampleRequest:
prompt: str | list[str] | list[dict]
prompt_len: int
expected_output_len: int | None
expected_output_len: int = 0
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
lora_request: LoRARequest | None = None
request_id: str | None = None
@@ -634,7 +634,7 @@ class RandomDataset(BenchmarkDataset):
# Generate prefix once
prefix_token_ids = self.get_prefix(tokenizer, allowed_tokens, prefix_len)
requests = []
requests: list[SampleRequest] = []
token_mismatch_total = 0
for i in range(num_requests):
prompt, total_input_len, token_mismatch = self.generate_token_sequence( # noqa: E501
@@ -665,15 +665,14 @@ class RandomDataset(BenchmarkDataset):
)
# only used for embeddings benchmark.
if batchsize > 1:
batch_requests = []
batch_requests: list[SampleRequest] = []
# Create batched requests
for i in range(0, num_requests, batchsize):
batch = requests[i : i + batchsize]
batch_requests.append(
SampleRequest(
prompt=[req.prompt for req in batch],
prompt=[req.prompt for req in batch], # type: ignore[arg-type]
prompt_len=sum(req.prompt_len for req in batch),
expected_output_len=0,
request_id=request_id_prefix + str(i // batchsize),
)
)
@@ -797,6 +796,9 @@ class RandomDatasetForReranking(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
is_reranker: bool = True,
**kwargs,
) -> list[SampleRequest]:
@@ -1207,6 +1209,9 @@ class RandomMultiModalDataset(RandomDataset):
input_len: int = RandomDataset.DEFAULT_INPUT_LEN,
output_len: int = RandomDataset.DEFAULT_OUTPUT_LEN,
batchsize: int = 1,
max_loras: int | None = None,
lora_path: str | None = None,
lora_assignment: str = "random",
limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,
base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,
num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
@@ -1251,6 +1256,7 @@ class RandomMultiModalDataset(RandomDataset):
# We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it
# may break prompt replacement logic.
assert hasattr(tokenizer, "added_tokens_decoder")
prohibited_tokens = list(
tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items()
@@ -1376,6 +1382,7 @@ class ShareGPTDataset(BenchmarkDataset):
lora_assignment: str = "random",
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
samples: list[SampleRequest] = []
ind = 0
for entry in self.data:
@@ -1454,7 +1461,7 @@ class TimedTrace(BenchmarkDataset):
f'label_output_length: "{self.label_output_length}", '
f'label_hash_ids: "{self.label_hash_ids}"'
)
self._expanded_generated_prompts = {}
self._expanded_generated_prompts: dict[str, Any] = {}
self.load_data()
def load_data(self) -> None:
@@ -1529,10 +1536,12 @@ class TimedTrace(BenchmarkDataset):
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list:
samples: list = []
) -> list[SampleRequest]:
samples: list[SampleRequest] = []
assert tokenizer is not None, "Tokenizer must be provided, now is Null"
assert self.data is not None, "Data must be loaded before sampling"
for ind, entry in enumerate(self.data):
if len(samples) >= num_requests:
@@ -2139,12 +2148,12 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
)
elif args.dataset_name == "sonnet":
dataset = SonnetDataset(
sonnet_dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
if args.backend == "openai-chat":
input_requests = dataset.sample(
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
@@ -2155,10 +2164,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample,
)
else:
assert tokenizer.chat_template or tokenizer.default_chat_template, (
"Tokenizer/model must have chat template for sonnet dataset."
)
input_requests = dataset.sample(
assert (
hasattr(tokenizer, "chat_template") and tokenizer.chat_template
) or (
hasattr(tokenizer, "default_chat_template")
and tokenizer.default_chat_template
), "Tokenizer/model must have chat template for sonnet dataset."
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
output_len=args.sonnet_output_len,
@@ -2173,6 +2185,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
# all following datasets are implemented from the
# HuggingFaceDataset base class
hf_kwargs = {}
dataset_class: type[BenchmarkDataset]
if (
args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
@@ -2548,23 +2561,23 @@ class CustomDataset(BenchmarkDataset):
if tokenizer is None:
new_output_len = 1
elif output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
else:
new_output_len = output_len
if output_len is None or output_len == -1:
# check that the request has an 'output_tokens' field
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
# Use number of output tokens from the request data
try:
new_output_len = int(item["output_tokens"])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid value for 'output_tokens' in custom dataset: "
f"'{item['output_tokens']}'. Must be an integer."
) from e
if tokenizer is None:
prompt_len = 1
@@ -2797,11 +2810,15 @@ class CustomImageDataset(CustomDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
ensure_client_side_data: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
ensure_client_side_data: bool = False,
**kwargs,
) -> list[SampleRequest]:
# load all data if needed
@@ -2814,7 +2831,7 @@ class CustomImageDataset(CustomDataset):
num_requests,
)
sampled_requests = []
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -2837,7 +2854,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
expected_output_len=output_len or 0,
multi_modal_data=None,
request_id=request_id_prefix + str(i),
)
@@ -2863,7 +2880,7 @@ class CustomImageDataset(CustomDataset):
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
expected_output_len=output_len or 0,
multi_modal_data=mm_content,
request_id=request_id_prefix + str(i),
)
@@ -2891,17 +2908,20 @@ class CustomAudioDataset(CustomDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
skip_chat_template: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs,
) -> list[SampleRequest]:
self.num_available_samples = len(self.data)
if num_requests <= 0:
num_requests = self.num_available_samples
sampled_requests = []
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -2950,7 +2970,6 @@ class CustomAudioDataset(CustomDataset):
prompt_len = (
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
)
new_output_len = output_len
if output_len is None or output_len == -1:
if "output_tokens" not in item:
raise ValueError(
@@ -2958,6 +2977,8 @@ class CustomAudioDataset(CustomDataset):
"custom dataset must contain an 'output_tokens' field."
)
new_output_len = int(item["output_tokens"])
else:
new_output_len = output_len
sampled_requests.append(
SampleRequest(
prompt=prompt,
@@ -3015,10 +3036,30 @@ class SpecBench(CustomDataset):
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
request_id_prefix: str = "",
no_oversample: bool = False,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
chat_template_kwargs: dict | None = None,
**kwargs,
) -> list[SampleRequest]:
# leverage CustomDataset sample
return super().sample(
tokenizer=tokenizer,
num_requests=num_requests,
request_id_prefix=request_id_prefix,
no_oversample=no_oversample,
lora_path=lora_path,
max_loras=max_loras,
output_len=output_len,
enable_multimodal_chat=enable_multimodal_chat,
skip_chat_template=skip_chat_template,
chat_template_kwargs=chat_template_kwargs,
**kwargs,
)
@@ -3067,16 +3108,21 @@ class SonnetDataset(BenchmarkDataset):
return_prompt_formatted: bool = False,
**kwargs,
) -> list[SampleRequest]:
poem_lines = self.data
assert poem_lines is not None
# Calculate average token length for a poem line.
tokenized_lines = [tokenizer(line).input_ids for line in self.data]
tokenized_lines = [tokenizer(line).input_ids for line in poem_lines]
avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
# Build the base prompt.
base_prompt = "Pick as many lines as you can from these poem lines:\n"
base_msg = [{"role": "user", "content": base_prompt}]
base_fmt = tokenizer.apply_chat_template(
base_msg, add_generation_prompt=True, tokenize=False
base_msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(base_fmt, str)
base_offset = len(tokenizer(base_fmt).input_ids)
if input_len <= base_offset:
raise ValueError(
@@ -3085,26 +3131,31 @@ class SonnetDataset(BenchmarkDataset):
)
# Determine how many poem lines to use.
num_input_lines = round((input_len - base_offset) / avg_len)
num_input_lines = max(round((input_len - base_offset) / avg_len), 1)
num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
prefix_lines = self.data[:num_prefix_lines]
prefix_lines = poem_lines[:num_prefix_lines]
samples: list[SampleRequest] = []
ind = 0
while len(samples) < num_requests:
extra_lines = random.choices(
self.data, k=num_input_lines - num_prefix_lines
poem_lines, k=num_input_lines - num_prefix_lines
)
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
msg = [{"role": "user", "content": prompt}]
prompt_formatted = tokenizer.apply_chat_template(
msg, add_generation_prompt=True, tokenize=False
msg, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids)
if prompt_len <= input_len:
samples.append(
SampleRequest(
prompt=prompt_formatted if return_prompt_formatted else prompt,
prompt=(
prompt_formatted if return_prompt_formatted else prompt
),
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(ind),
@@ -3145,6 +3196,7 @@ class BurstGPTDataset(BenchmarkDataset):
self.data = gpt4_df
def _sample_loaded_data(self, num_requests: int) -> list:
assert self.data is not None, "Dataset must be loaded before sampling"
if num_requests <= len(self.data):
data = self.data.sample(n=num_requests, random_state=self.random_seed)
else:
@@ -3258,6 +3310,7 @@ class ConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = []
@@ -3318,6 +3371,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):
enable_multimodal_chat: bool = False,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
# Filter examples with at least 2 conversations
filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
sampled_requests: list[SampleRequest] = []
@@ -3393,7 +3447,8 @@ class VisionArenaDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3465,7 +3520,8 @@ class MMVUDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3531,18 +3587,21 @@ class InstructCoderDataset(HuggingFaceDataset):
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests: list[SampleRequest] = []
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
prompt_text = prompt
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
prompt_text_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_text_result, str)
prompt_text = prompt_text_result
prompt_len = len(tokenizer(prompt).input_ids)
prompt_len = len(tokenizer(prompt_text).input_ids)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt=prompt_text,
prompt_len=prompt_len,
expected_output_len=output_len,
request_id=request_id_prefix + str(i),
@@ -3554,6 +3613,7 @@ class InstructCoderDataset(HuggingFaceDataset):
return sampled_requests
def sample_prompts(self, n: int) -> Iterator[str]:
assert self.data is not None, "Dataset must be loaded before sampling"
for item in self.data.take(n):
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
@@ -3594,6 +3654,7 @@ class MTBenchDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for i, item in enumerate(self.data):
@@ -3654,7 +3715,8 @@ class HumanEvalDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
@@ -3714,7 +3776,8 @@ class GSM8KDataset(HuggingFaceDataset):
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Data must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
@@ -3772,17 +3835,18 @@ class BlazeditDataset(HuggingFaceDataset):
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
skip_chat_template: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
output_len: int | None = None,
skip_chat_template: bool = False,
min_distance: float = 0.0,
max_distance: float = 1.0,
**kwargs,
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
sampled_requests: list[SampleRequest] = []
assert self.data is not None, "Dataset must be loaded before sampling"
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
@@ -3810,11 +3874,13 @@ Please generate the new code file in the "New file" section below.""" # noqa: E
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
prompt_result = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_result, str)
prompt = prompt_result
prompt_len = len(tokenizer(prompt).input_ids)
@@ -3858,6 +3924,7 @@ class AIMODataset(HuggingFaceDataset):
output_len: int | None = None,
**kwargs,
) -> list[SampleRequest]:
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
dynamic_output = output_len is None
@@ -3970,6 +4037,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):
formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)
if formatting_prompt_func is None:
raise ValueError(f"Unsupported dataset path: {self.hf_name}")
assert self.data is not None, "Dataset must be loaded before sampling"
samples = []
for i, sample in enumerate(self.data):
sample = formatting_prompt_func(sample)
@@ -4061,12 +4129,14 @@ class ASRDataset(HuggingFaceDataset):
def _disable_audio_decode(self) -> None:
from datasets import Audio
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None:
local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
)
assert self.data is not None, "Dataset must be loaded before sampling"
self.data = self.data.map(
lambda item: {
"audio": str(local_path_root / item["url"]),
@@ -4090,15 +4160,19 @@ class ASRDataset(HuggingFaceDataset):
else:
prompt = ""
prompt_len = len(tokenizer(prompt).input_ids)
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
skipped = 0
asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec")
asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec")
asr_min_audio_len_sec: float = float(kwargs.get("asr_min_audio_len_sec") or 0.0)
asr_max_audio_len_sec: float = float(
kwargs.get("asr_max_audio_len_sec") or float("inf")
)
durations = []
for item in self.data:
if len(sampled_requests) >= num_requests:
break
mm_content: dict[str, Any]
audio = item["audio"]
if (
isinstance(audio, dict)
@@ -4203,8 +4277,7 @@ class MLPerfDataset(HuggingFaceDataset):
output_len: int | None = None,
**kwargs,
) -> list[SampleRequest]:
# Force dynamic output length based on reference completion.
dynamic_output = output_len is None
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
ind = 0
@@ -4222,15 +4295,18 @@ class MLPerfDataset(HuggingFaceDataset):
{"role": "user", "content": question},
]
prompt_formatted = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, tokenize=False
messages, # type: ignore[arg-type]
add_generation_prompt=True,
tokenize=False,
)
assert isinstance(prompt_formatted, str)
prompt_len = len(tokenizer(prompt_formatted).input_ids)
# Determine output length from reference answer tokens.
ref_out_len = len(
tokenizer(reference_answer, add_special_tokens=False).input_ids
)
expected_output_len = ref_out_len if dynamic_output else output_len
expected_output_len = ref_out_len if output_len is None else output_len
# Validate sequence lengths.
if not is_valid_sequence(prompt_len, expected_output_len):
@@ -4371,6 +4447,7 @@ class MMStarDataset(HuggingFaceDataset):
) -> list[SampleRequest]:
# If --hf-output-len is not set, use the default output length.
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
assert self.data is not None, "Dataset must be loaded before sampling"
sampled_requests: list[SampleRequest] = []
for ind, item in enumerate(self.data):
@@ -4603,7 +4680,7 @@ class BFCLDataset(HuggingFaceDataset):
exc_info=True,
)
rendered = None
if rendered is not None:
if rendered is not None and isinstance(rendered, str):
prompt_len = len(tokenizer(rendered).input_ids)
else:
text = "\n".join(m.get("content", "") for m in messages)