[MyPy] Fix mypy for vllm/benchmarks (#39896)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
547d2c40d7
commit
275b43183c
@@ -18,6 +18,7 @@ from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
|
||||
from vllm.benchmarks.datasets import (
|
||||
AIMODataset,
|
||||
ASRDataset,
|
||||
BenchmarkDataset,
|
||||
BurstGPTDataset,
|
||||
ConversationDataset,
|
||||
InstructCoderDataset,
|
||||
@@ -41,6 +42,7 @@ from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
|
||||
@@ -102,13 +104,15 @@ def _run_vllm_requests(
|
||||
|
||||
prompts: list[TextPrompt | TokensPrompt] = []
|
||||
sampling_params: list[SamplingParams] = []
|
||||
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
|
||||
lora_requests: list[LoRARequest | None] | None = [] if enable_lora else None
|
||||
for request in requests:
|
||||
prompt = (
|
||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
||||
if "prompt_token_ids" in request.prompt
|
||||
else TextPrompt(prompt=request.prompt)
|
||||
)
|
||||
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
|
||||
prompt_token_ids = request.prompt["prompt_token_ids"]
|
||||
assert isinstance(prompt_token_ids, list)
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||
else:
|
||||
assert isinstance(request.prompt, str)
|
||||
prompt = TextPrompt(prompt=request.prompt)
|
||||
if request.multi_modal_data:
|
||||
assert isinstance(request.multi_modal_data, dict)
|
||||
prompt["multi_modal_data"] = request.multi_modal_data
|
||||
@@ -159,7 +163,20 @@ def _run_vllm_requests(
|
||||
end = time.perf_counter()
|
||||
else:
|
||||
assert lora_requests is None, "BeamSearch API does not support LoRA"
|
||||
beam_prompts = [request.prompt for request in requests]
|
||||
beam_prompts: list[TextPrompt | TokensPrompt] = []
|
||||
for request in requests:
|
||||
if isinstance(request.prompt, str):
|
||||
beam_prompts.append(TextPrompt(prompt=request.prompt))
|
||||
elif (
|
||||
isinstance(request.prompt, dict)
|
||||
and "prompt_token_ids" in request.prompt
|
||||
):
|
||||
token_ids = request.prompt["prompt_token_ids"]
|
||||
assert isinstance(token_ids, list)
|
||||
beam_prompts.append(TokensPrompt(prompt_token_ids=token_ids))
|
||||
else:
|
||||
# Fallback: convert to string
|
||||
beam_prompts.append(TextPrompt(prompt=str(request.prompt)))
|
||||
# output_len should be the same for all requests.
|
||||
output_len = requests[0].expected_output_len
|
||||
for request in requests:
|
||||
@@ -268,7 +285,7 @@ def _run_vllm_chat_requests(
|
||||
llm.wake_up(tags=["scheduling"])
|
||||
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
|
||||
else:
|
||||
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
|
||||
outputs = llm.chat(prompts, sampling_params, use_tqdm=True) # type: ignore[arg-type]
|
||||
|
||||
if do_profile:
|
||||
llm.stop_profile()
|
||||
@@ -339,11 +356,13 @@ async def _run_vllm_async_requests(
|
||||
sampling_params: list[SamplingParams] = []
|
||||
lora_requests: list[LoRARequest | None] = []
|
||||
for request in requests:
|
||||
prompt = (
|
||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
||||
if "prompt_token_ids" in request.prompt
|
||||
else TextPrompt(prompt=request.prompt)
|
||||
)
|
||||
if isinstance(request.prompt, dict) and "prompt_token_ids" in request.prompt:
|
||||
prompt_token_ids = request.prompt["prompt_token_ids"]
|
||||
assert isinstance(prompt_token_ids, list)
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||
else:
|
||||
assert isinstance(request.prompt, str)
|
||||
prompt = TextPrompt(prompt=request.prompt)
|
||||
|
||||
if request.multi_modal_data:
|
||||
assert isinstance(request.multi_modal_data, dict)
|
||||
@@ -366,9 +385,11 @@ async def _run_vllm_async_requests(
|
||||
start = time.perf_counter()
|
||||
if do_profile:
|
||||
await llm.start_profile()
|
||||
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
|
||||
for i, (prompt_item, sp, lr) in enumerate(
|
||||
zip(prompts, sampling_params, lora_requests)
|
||||
):
|
||||
generator = llm.generate(
|
||||
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
|
||||
prompt_item, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
|
||||
)
|
||||
generators.append(generator)
|
||||
all_gens = merge_async_iterators(*generators)
|
||||
@@ -440,6 +461,7 @@ def _run_hf_requests(
|
||||
prompt_len = requests[i].prompt_len
|
||||
output_len = requests[i].expected_output_len
|
||||
# Add the prompt to the batch.
|
||||
assert isinstance(prompt, str), "Prompt must be a string for HF backend"
|
||||
batch.append(prompt)
|
||||
max_prompt_len = max(max_prompt_len, prompt_len)
|
||||
max_output_len = max(max_output_len, output_len)
|
||||
@@ -500,6 +522,7 @@ def save_to_pytorch_benchmark_format(
|
||||
|
||||
def get_requests(args, tokenizer):
|
||||
# Common parameters for all dataset types.
|
||||
dataset_cls: type[BenchmarkDataset]
|
||||
common_kwargs = {
|
||||
"dataset_path": args.dataset_path,
|
||||
"random_seed": args.seed,
|
||||
@@ -875,7 +898,7 @@ def validate_args(args):
|
||||
)
|
||||
|
||||
|
||||
def add_cli_args(parser: argparse.ArgumentParser):
|
||||
def add_cli_args(parser: FlexibleArgumentParser):
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
type=str,
|
||||
@@ -1178,7 +1201,11 @@ def main(args: argparse.Namespace):
|
||||
total_prompt_tokens += (
|
||||
len(ro.prompt_token_ids) if ro.prompt_token_ids else 0
|
||||
)
|
||||
total_output_tokens += sum(len(o.token_ids) for o in ro.outputs if o)
|
||||
total_output_tokens += sum(
|
||||
len(o.token_ids)
|
||||
for o in ro.outputs
|
||||
if o is not None and o.token_ids is not None
|
||||
)
|
||||
total_num_tokens = total_prompt_tokens + total_output_tokens
|
||||
else:
|
||||
total_num_tokens = sum(r.prompt_len + r.expected_output_len for r in requests)
|
||||
|
||||
Reference in New Issue
Block a user