[MyPy] Fix mypy for vllm/benchmarks (#39896)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Martin Hickey
2026-06-23 15:22:29 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 547d2c40d7
commit 275b43183c
24 changed files with 297 additions and 151 deletions
+44 -17
View File
@@ -18,6 +18,7 @@ from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import (
AIMODataset,
ASRDataset,
BenchmarkDataset,
BurstGPTDataset,
ConversationDataset,
InstructCoderDataset,
@@ -41,6 +42,7 @@ from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.async_utils import merge_async_iterators
@@ -102,13 +104,15 @@ def _run_vllm_requests(
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
)
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
prompt_token_ids = request.prompt["prompt_token_ids"]
assert isinstance(prompt_token_ids, list)
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
prompt["multi_modal_data"] = request.multi_modal_data
@@ -159,7 +163,20 @@ def _run_vllm_requests(
end = time.perf_counter()
else:
assert lora_requests is None, "BeamSearch API does not support LoRA"
beam_prompts = [request.prompt for request in requests]
beam_prompts: list[TextPrompt | TokensPrompt] = []
for request in requests:
if isinstance(request.prompt, str):
beam_prompts.append(TextPrompt(prompt=request.prompt))
elif (
isinstance(request.prompt, dict)
and "prompt_token_ids" in request.prompt
):
token_ids = request.prompt["prompt_token_ids"]
assert isinstance(token_ids, list)
beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids))
else:
# Fallback: convert to string
beam_prompts.append(TextPrompt(prompt=str(request.prompt)))
# output_len should be the same for all requests.
output_len = requests[0].expected_output_len
for request in requests:
@@ -268,7 +285,7 @@ def _run_vllm_chat_requests(
llm.wake_up(tags=["scheduling"])
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
else:
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type]
if do_profile:
llm.stop_profile()
@@ -339,11 +356,13 @@ async def _run_vllm_async_requests(
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
)
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
prompt_token_ids = request.prompt["prompt_token_ids"]
assert isinstance(prompt_token_ids, list)
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
else:
assert isinstance(request.prompt, str)
prompt = TextPrompt(prompt=request.prompt)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
@@ -366,9 +385,11 @@ async def _run_vllm_async_requests(
start = time.perf_counter()
if do_profile:
await llm.start_profile()
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
for i, (prompt_item, sp, lr) in enumerate(
zip(prompts, sampling_params, lora_requests)
):
generator = llm.generate(
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
)
generators.append(generator)
all_gens = merge_async_iterators(*generators)
@@ -440,6 +461,7 @@ def _run_hf_requests(
prompt_len = requests[i].prompt_len
output_len = requests[i].expected_output_len
# Add the prompt to the batch.
assert isinstance(prompt, str), "Prompt must be a string for HF backend"
batch.append(prompt)
max_prompt_len = max(max_prompt_len, prompt_len)
max_output_len = max(max_output_len, output_len)
@@ -500,6 +522,7 @@ def save_to_pytorch_benchmark_format(
def get_requests(args, tokenizer):
# Common parameters for all dataset types.
dataset_cls: type[BenchmarkDataset]
common_kwargs = {
"dataset_path": args.dataset_path,
"random_seed": args.seed,
@@ -875,7 +898,7 @@ def validate_args(args):
)
def add_cli_args(parser: argparse.ArgumentParser):
def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument(
"--backend",
type=str,
@@ -1178,7 +1201,11 @@ def main(args: argparse.Namespace):
total_prompt_tokens += (
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
)
total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o)
total_output_tokens += sum(
len(o.token_ids)
for o in ro.outputs
if o is not None and o.token_ids is not None
)
total_num_tokens = total_prompt_tokens + total_output_tokens
else:
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)