[Benchmark] Add num-warmup to vllm bench throughput (#43245)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-05-21 05:13:15 +00:00
committed by GitHub
parent a6682d1d25
commit 905b97adfa
+200 -54
View File
@@ -50,22 +50,59 @@ def run_vllm(
engine_args: EngineArgs,
do_profile: bool,
disable_detokenize: bool = False,
warmup_requests: list[SampleRequest] | None = None,
prequeue_requests: bool = False,
) -> tuple[float, list[RequestOutput] | None]:
from vllm import LLM, SamplingParams
from vllm import LLM
llm = LLM.from_engine_args(engine_args)
all_requests = list(warmup_requests or []) + requests
assert all(
llm.llm_engine.model_config.max_model_len
>= (request.prompt_len + request.expected_output_len)
for request in requests
for request in all_requests
), (
"Please ensure that max_model_len is greater than the sum of"
" prompt_len and expected_output_len for all requests."
)
# Add the requests to the engine.
if warmup_requests:
print(f"Warming up with {len(warmup_requests)} requests...")
_run_vllm_requests(
llm,
warmup_requests,
n,
disable_detokenize,
do_profile=False,
prequeue_requests=prequeue_requests,
enable_lora=engine_args.enable_lora,
)
return _run_vllm_requests(
llm,
requests,
n,
disable_detokenize,
do_profile=do_profile,
prequeue_requests=prequeue_requests,
enable_lora=engine_args.enable_lora,
)
def _run_vllm_requests(
llm: Any,
requests: list[SampleRequest],
n: int,
disable_detokenize: bool,
do_profile: bool,
prequeue_requests: bool,
enable_lora: bool,
) -> tuple[float, list[RequestOutput] | None]:
from vllm import SamplingParams
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
@@ -87,9 +124,8 @@ def run_vllm(
detokenize=not disable_detokenize,
)
)
lora_requests: list[LoRARequest] | None = None
if engine_args.enable_lora:
lora_requests = [request.lora_request for request in requests]
if lora_requests is not None:
lora_requests.append(request.lora_request)
use_beam_search = False
@@ -123,7 +159,7 @@ def run_vllm(
end = time.perf_counter()
else:
assert lora_requests is None, "BeamSearch API does not support LoRA"
prompts = [request.prompt for request in requests]
beam_prompts = [request.prompt for request in requests]
# output_len should be the same for all requests.
output_len = requests[0].expected_output_len
for request in requests:
@@ -132,7 +168,7 @@ def run_vllm(
if do_profile:
llm.start_profile()
llm.beam_search(
prompts,
beam_prompts,
BeamSearchParams(
beam_width=n,
max_tokens=output_len,
@@ -151,6 +187,7 @@ def run_vllm_chat(
engine_args: EngineArgs,
do_profile: bool,
disable_detokenize: bool = False,
warmup_requests: list[SampleRequest] | None = None,
prequeue_requests: bool = False,
) -> tuple[float, list[RequestOutput]]:
"""
@@ -158,23 +195,54 @@ def run_vllm_chat(
multimodal models as it properly handles multimodal inputs and chat
formatting. For non-multimodal models, use run_vllm() instead.
"""
from vllm import LLM, SamplingParams
from vllm import LLM
llm = LLM.from_engine_args(engine_args)
all_requests = list(warmup_requests or []) + requests
assert all(
llm.llm_engine.model_config.max_model_len
>= (request.prompt_len + request.expected_output_len)
for request in requests
for request in all_requests
), (
"Please ensure that max_model_len is greater than the sum of "
"prompt_len and expected_output_len for all requests."
)
prompts = []
if warmup_requests:
print(f"Warming up with {len(warmup_requests)} requests...")
_run_vllm_chat_requests(
llm,
warmup_requests,
n,
disable_detokenize,
do_profile=False,
prequeue_requests=prequeue_requests,
)
return _run_vllm_chat_requests(
llm,
requests,
n,
disable_detokenize,
do_profile=do_profile,
prequeue_requests=prequeue_requests,
)
def _run_vllm_chat_requests(
llm: Any,
requests: list[SampleRequest],
n: int,
disable_detokenize: bool,
do_profile: bool,
prequeue_requests: bool,
) -> tuple[float, list[RequestOutput]]:
from vllm import SamplingParams
prompts = [request.prompt for request in requests]
sampling_params: list[SamplingParams] = []
for request in requests:
prompts.append(request.prompt)
sampling_params.append(
SamplingParams(
n=n,
@@ -185,6 +253,7 @@ def run_vllm_chat(
detokenize=not disable_detokenize,
)
)
if prequeue_requests:
llm.sleep(level=0, mode="abort")
@@ -214,8 +283,8 @@ async def run_vllm_async(
engine_args: AsyncEngineArgs,
do_profile: bool,
disable_detokenize: bool = False,
warmup_requests: list[SampleRequest] | None = None,
) -> float:
from vllm import SamplingParams
from vllm.entrypoints.openai.api_server import (
build_async_engine_client_from_engine_args,
)
@@ -224,59 +293,91 @@ async def run_vllm_async(
engine_args,
) as llm:
model_config = llm.model_config
all_requests = list(warmup_requests or []) + requests
assert all(
model_config.max_model_len
>= (request.prompt_len + request.expected_output_len)
for request in requests
for request in all_requests
), (
"Please ensure that max_model_len is greater than the sum of"
" prompt_len and expected_output_len for all requests."
)
# Add the requests to the engine.
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
if warmup_requests:
print(f"Warming up with {len(warmup_requests)} requests...")
await _run_vllm_async_requests(
llm,
warmup_requests,
n,
disable_detokenize,
do_profile=False,
request_id_prefix="warmup",
)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
prompt["multi_modal_data"] = request.multi_modal_data
elapsed_time, _ = await _run_vllm_async_requests(
llm,
requests,
n,
disable_detokenize,
do_profile=do_profile,
request_id_prefix="test",
)
return elapsed_time
sampling_params.append(
SamplingParams(
n=n,
temperature=1.0,
top_p=1.0,
ignore_eos=True,
max_tokens=request.expected_output_len,
detokenize=not disable_detokenize,
)
async def _run_vllm_async_requests(
llm: Any,
requests: list[SampleRequest],
n: int,
disable_detokenize: bool,
do_profile: bool,
request_id_prefix: str,
) -> tuple[float, None]:
from vllm import SamplingParams
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
if "prompt_token_ids" in request.prompt
else TextPrompt(prompt=request.prompt)
)
if request.multi_modal_data:
assert isinstance(request.multi_modal_data, dict)
prompt["multi_modal_data"] = request.multi_modal_data
sampling_params.append(
SamplingParams(
n=n,
temperature=1.0,
top_p=1.0,
ignore_eos=True,
max_tokens=request.expected_output_len,
detokenize=not disable_detokenize,
)
prompts.append(prompt)
lora_requests.append(request.lora_request)
)
prompts.append(prompt)
lora_requests.append(request.lora_request)
generators = []
start = time.perf_counter()
if do_profile:
await llm.start_profile()
for i, (prompt, sp, lr) in enumerate(
zip(prompts, sampling_params, lora_requests)
):
generator = llm.generate(prompt, sp, lora_request=lr, request_id=f"test{i}")
generators.append(generator)
all_gens = merge_async_iterators(*generators)
async for i, res in all_gens:
pass
if do_profile:
await llm.stop_profile()
end = time.perf_counter()
return end - start
generators = []
start = time.perf_counter()
if do_profile:
await llm.start_profile()
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
generator = llm.generate(
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
)
generators.append(generator)
all_gens = merge_async_iterators(*generators)
async for _i, _res in all_gens:
pass
if do_profile:
await llm.stop_profile()
end = time.perf_counter()
return end - start, None
def run_hf(
@@ -289,6 +390,7 @@ def run_hf(
disable_detokenize: bool = False,
dtype: torch.dtype | None = torch.float16,
enable_torch_compile: bool = False,
warmup_requests: list[SampleRequest] | None = None,
) -> float:
assert isinstance(tokenizer, PreTrainedTokenizerBase), (
"the hf backend only supports HF tokenizers"
@@ -303,6 +405,31 @@ def run_hf(
if enable_torch_compile:
llm = torch.compile(llm)
if warmup_requests:
print(f"Warming up with {len(warmup_requests)} requests...")
_run_hf_requests(
llm,
tokenizer,
warmup_requests,
n,
max_batch_size,
disable_detokenize,
)
elapsed_time, _ = _run_hf_requests(
llm, tokenizer, requests, n, max_batch_size, disable_detokenize
)
return elapsed_time
def _run_hf_requests(
llm: Any,
tokenizer: PreTrainedTokenizerBase,
requests: list[SampleRequest],
n: int,
max_batch_size: int,
disable_detokenize: bool,
) -> tuple[float, None]:
pbar = tqdm(total=len(requests))
start = time.perf_counter()
batch: list[str] = []
@@ -347,8 +474,9 @@ def run_hf(
batch = []
max_prompt_len = 0
max_output_len = 0
pbar.close()
end = time.perf_counter()
return end - start
return end - start, None
def save_to_pytorch_benchmark_format(
@@ -801,6 +929,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
parser.add_argument(
"--num-prompts", type=int, default=1000, help="Number of prompts to process."
)
parser.add_argument(
"--num-warmups",
type=int,
default=0,
help="Number of warmup prompts to process before the timed benchmark.",
)
parser.add_argument(
"--hf-max-batch-size",
type=int,
@@ -971,6 +1105,14 @@ def main(args: argparse.Namespace):
tokenizer_mode=args.tokenizer_mode,
trust_remote_code=args.trust_remote_code,
)
num_warmups = args.num_warmups
warmup_requests: list[SampleRequest] | None = None
if num_warmups > 0:
warmup_args = argparse.Namespace(**vars(args))
warmup_args.num_prompts = num_warmups
warmup_args.seed += 1
warmup_requests = get_requests(warmup_args, tokenizer)
requests = get_requests(args, tokenizer)
is_multi_modal = any(request.multi_modal_data is not None for request in requests)
request_outputs: list[RequestOutput] | None = None
@@ -983,6 +1125,7 @@ def main(args: argparse.Namespace):
AsyncEngineArgs.from_cli_args(args),
disable_detokenize=args.disable_detokenize,
do_profile=args.profile,
warmup_requests=warmup_requests,
)
)
else:
@@ -992,6 +1135,7 @@ def main(args: argparse.Namespace):
EngineArgs.from_cli_args(args),
disable_detokenize=args.disable_detokenize,
do_profile=args.profile,
warmup_requests=warmup_requests,
prequeue_requests=args.prequeue_requests,
)
elif args.backend == "hf":
@@ -1008,6 +1152,7 @@ def main(args: argparse.Namespace):
args.disable_detokenize,
dtype=args.dtype,
enable_torch_compile=args.hf_enable_torch_compile,
warmup_requests=warmup_requests,
)
elif args.backend == "vllm-chat":
elapsed_time, request_outputs = run_vllm_chat(
@@ -1016,6 +1161,7 @@ def main(args: argparse.Namespace):
EngineArgs.from_cli_args(args),
disable_detokenize=args.disable_detokenize,
do_profile=args.profile,
warmup_requests=warmup_requests,
prequeue_requests=args.prequeue_requests,
)
else: