[MyPy] Fix mypy for vllm/benchmarks (#39896)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
547d2c40d7
commit
275b43183c
+37
-14
@@ -30,7 +30,7 @@ import ssl
|
||||
import time
|
||||
import uuid
|
||||
import warnings
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from collections.abc import AsyncGenerator, Iterable, Iterator
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
@@ -52,6 +52,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
|
||||
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.utils.network_utils import join_host_port
|
||||
|
||||
@@ -129,6 +130,7 @@ async def _align_prompts_to_server_tokenizer(
|
||||
)
|
||||
|
||||
async def _fix_one(req: SampleRequest) -> SampleRequest:
|
||||
assert isinstance(req.prompt, str)
|
||||
tokens = await _tokenize(req.prompt)
|
||||
if len(tokens) <= req.prompt_len:
|
||||
return req
|
||||
@@ -360,7 +362,7 @@ class EmbedBenchmarkMetrics:
|
||||
mean_e2el_ms: float
|
||||
std_e2el_ms: float
|
||||
median_e2el_ms: float
|
||||
percentiles_e2el_ms: float
|
||||
percentiles_e2el_ms: list[tuple[float, float]]
|
||||
|
||||
|
||||
def _get_current_request_rate(
|
||||
@@ -433,8 +435,8 @@ async def get_request(
|
||||
assert total_requests > 0, "No requests provided."
|
||||
|
||||
# Precompute delays among requests to minimize request send laggings
|
||||
request_rates = []
|
||||
delay_ts = []
|
||||
request_rates: list[float] = []
|
||||
delay_ts: list[float] = []
|
||||
|
||||
# if the traces have timing info then:
|
||||
if not self_timed:
|
||||
@@ -484,7 +486,10 @@ async def get_request(
|
||||
else:
|
||||
for request_index, request in enumerate(input_requests):
|
||||
# this is cumulative running ts, from which sleep is calculated later
|
||||
delay_ts.append(request.timestamp)
|
||||
if request.timestamp is not None:
|
||||
delay_ts.append(request.timestamp)
|
||||
else:
|
||||
delay_ts.append(0.0)
|
||||
# TODO: there is no notion of RPS here, may be we can calculate
|
||||
# from the trace.
|
||||
request_rates.append(0.0)
|
||||
@@ -599,7 +604,7 @@ def calculate_metrics(
|
||||
)
|
||||
actual_output_lens.append(output_len)
|
||||
total_input += outputs[i].prompt_len
|
||||
tpot = 0
|
||||
tpot = 0.0
|
||||
if output_len > 1:
|
||||
latency_minus_ttft = outputs[i].latency - outputs[i].ttft
|
||||
tpot = latency_minus_ttft / (output_len - 1)
|
||||
@@ -894,11 +899,12 @@ async def benchmark(
|
||||
|
||||
print("Starting main benchmark run...")
|
||||
|
||||
lora_modules_iter: Iterator[str] | None = None
|
||||
if lora_modules:
|
||||
lora_modules_list = list(lora_modules)
|
||||
if lora_assignment == "round-robin":
|
||||
# Deterministic round-robin assignment across requests.
|
||||
lora_modules = iter(
|
||||
lora_modules_iter = iter(
|
||||
[
|
||||
lora_modules_list[i % len(lora_modules_list)]
|
||||
for i in range(len(input_requests))
|
||||
@@ -906,7 +912,7 @@ async def benchmark(
|
||||
)
|
||||
else:
|
||||
# For each input request, choose a LoRA module at random.
|
||||
lora_modules = iter(
|
||||
lora_modules_iter = iter(
|
||||
[random.choice(lora_modules_list) for _ in range(len(input_requests))]
|
||||
)
|
||||
|
||||
@@ -1004,19 +1010,23 @@ async def benchmark(
|
||||
)
|
||||
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
||||
req_model_id, req_model_name = model_id, model_name
|
||||
if lora_modules:
|
||||
req_lora_module = next(lora_modules)
|
||||
if lora_modules_iter:
|
||||
req_lora_module = next(lora_modules_iter)
|
||||
req_model_id, req_model_name = req_lora_module, req_lora_module
|
||||
|
||||
mm_content_typed: dict[str, Any] | list[dict[str, Any]] | None = None
|
||||
if isinstance(mm_content, (dict, list)):
|
||||
mm_content_typed = mm_content
|
||||
|
||||
request_func_input = RequestFuncInput(
|
||||
model=req_model_id,
|
||||
model_name=req_model_name,
|
||||
prompt=prompt,
|
||||
api_url=api_url,
|
||||
prompt_len=prompt_len,
|
||||
output_len=output_len,
|
||||
output_len=output_len or 0,
|
||||
logprobs=logprobs,
|
||||
multi_modal_content=mm_content,
|
||||
multi_modal_content=mm_content_typed,
|
||||
ignore_eos=ignore_eos,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=per_request_extra_body,
|
||||
@@ -1107,6 +1117,8 @@ async def benchmark(
|
||||
"committed_per_step": delta_committed / denoising_steps,
|
||||
}
|
||||
|
||||
metrics: BenchmarkMetrics | EmbedBenchmarkMetrics
|
||||
actual_output_lens: list[int] | int
|
||||
if task_type == TaskType.GENERATION:
|
||||
metrics, actual_output_lens = calculate_metrics(
|
||||
input_requests=input_requests,
|
||||
@@ -1140,7 +1152,7 @@ async def benchmark(
|
||||
"Request throughput (req/s):", metrics.request_throughput
|
||||
)
|
||||
)
|
||||
if goodput_config_dict:
|
||||
if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
"Request goodput (req/s):", metrics.request_goodput
|
||||
@@ -1177,6 +1189,7 @@ async def benchmark(
|
||||
)
|
||||
)
|
||||
|
||||
result: dict[str, Any]
|
||||
if isinstance(metrics, BenchmarkMetrics):
|
||||
result = {
|
||||
"duration": benchmark_duration,
|
||||
@@ -1462,7 +1475,7 @@ def compute_result_filename(
|
||||
return file_name
|
||||
|
||||
|
||||
def add_cli_args(parser: argparse.ArgumentParser):
|
||||
def add_cli_args(parser: FlexibleArgumentParser):
|
||||
add_dataset_parser(parser)
|
||||
parser.add_argument(
|
||||
"--label",
|
||||
@@ -2023,6 +2036,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
args.self_timed = False
|
||||
|
||||
# Load the dataset.
|
||||
assert tokenizer is not None, "Tokenizer must be initialized before loading dataset"
|
||||
input_requests = get_samples(args, tokenizer)
|
||||
|
||||
if args.dataset_name in ("random", "prefix_repetition"):
|
||||
@@ -2154,6 +2168,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
|
||||
# Generate timeline plot if requested
|
||||
if args.plot_timeline:
|
||||
assert file_name is not None, (
|
||||
"file_name must be set when plot_timeline is enabled"
|
||||
)
|
||||
try:
|
||||
from vllm.benchmarks.plot import generate_timeline_plot
|
||||
|
||||
@@ -2200,6 +2217,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
|
||||
# Generate dataset statistics plot if requested
|
||||
if args.plot_dataset_stats:
|
||||
assert file_name is not None, (
|
||||
"file_name must be set when plot_dataset_stats is enabled"
|
||||
)
|
||||
try:
|
||||
from vllm.benchmarks.plot import generate_dataset_stats_plot
|
||||
|
||||
@@ -2249,6 +2269,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
|
||||
# Save to file
|
||||
if args.save_result or args.append_result:
|
||||
assert file_name is not None, (
|
||||
"file_name must be set when save_result or append_result is enabled"
|
||||
)
|
||||
with open(
|
||||
file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
|
||||
) as outfile:
|
||||
|
||||
Reference in New Issue
Block a user