From 905b97adfaf7b08f3cc95b328579e5336ed6d3b6 Mon Sep 17 00:00:00 2001 From: yzong-rh Date: Thu, 21 May 2026 01:13:15 -0400 Subject: [PATCH] [Benchmark] Add num-warmup to vllm bench throughput (#43245) Signed-off-by: Yifan Zong --- vllm/benchmarks/throughput.py | 254 ++++++++++++++++++++++++++-------- 1 file changed, 200 insertions(+), 54 deletions(-) diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py index 95a87e6be1..9f1bf4487e 100644 --- a/vllm/benchmarks/throughput.py +++ b/vllm/benchmarks/throughput.py @@ -50,22 +50,59 @@ def run_vllm( engine_args: EngineArgs, do_profile: bool, disable_detokenize: bool = False, + warmup_requests: list[SampleRequest] | None = None, prequeue_requests: bool = False, ) -> tuple[float, list[RequestOutput] | None]: - from vllm import LLM, SamplingParams + from vllm import LLM llm = LLM.from_engine_args(engine_args) + all_requests = list(warmup_requests or []) + requests assert all( llm.llm_engine.model_config.max_model_len >= (request.prompt_len + request.expected_output_len) - for request in requests + for request in all_requests ), ( "Please ensure that max_model_len is greater than the sum of" " prompt_len and expected_output_len for all requests." ) - # Add the requests to the engine. + + if warmup_requests: + print(f"Warming up with {len(warmup_requests)} requests...") + _run_vllm_requests( + llm, + warmup_requests, + n, + disable_detokenize, + do_profile=False, + prequeue_requests=prequeue_requests, + enable_lora=engine_args.enable_lora, + ) + + return _run_vllm_requests( + llm, + requests, + n, + disable_detokenize, + do_profile=do_profile, + prequeue_requests=prequeue_requests, + enable_lora=engine_args.enable_lora, + ) + + +def _run_vllm_requests( + llm: Any, + requests: list[SampleRequest], + n: int, + disable_detokenize: bool, + do_profile: bool, + prequeue_requests: bool, + enable_lora: bool, +) -> tuple[float, list[RequestOutput] | None]: + from vllm import SamplingParams + prompts: list[TextPrompt | TokensPrompt] = [] sampling_params: list[SamplingParams] = [] + lora_requests: list[LoRARequest] | None = [] if enable_lora else None for request in requests: prompt = ( TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"]) @@ -87,9 +124,8 @@ def run_vllm( detokenize=not disable_detokenize, ) ) - lora_requests: list[LoRARequest] | None = None - if engine_args.enable_lora: - lora_requests = [request.lora_request for request in requests] + if lora_requests is not None: + lora_requests.append(request.lora_request) use_beam_search = False @@ -123,7 +159,7 @@ def run_vllm( end = time.perf_counter() else: assert lora_requests is None, "BeamSearch API does not support LoRA" - prompts = [request.prompt for request in requests] + beam_prompts = [request.prompt for request in requests] # output_len should be the same for all requests. output_len = requests[0].expected_output_len for request in requests: @@ -132,7 +168,7 @@ def run_vllm( if do_profile: llm.start_profile() llm.beam_search( - prompts, + beam_prompts, BeamSearchParams( beam_width=n, max_tokens=output_len, @@ -151,6 +187,7 @@ def run_vllm_chat( engine_args: EngineArgs, do_profile: bool, disable_detokenize: bool = False, + warmup_requests: list[SampleRequest] | None = None, prequeue_requests: bool = False, ) -> tuple[float, list[RequestOutput]]: """ @@ -158,23 +195,54 @@ def run_vllm_chat( multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead. """ - from vllm import LLM, SamplingParams + from vllm import LLM llm = LLM.from_engine_args(engine_args) + all_requests = list(warmup_requests or []) + requests assert all( llm.llm_engine.model_config.max_model_len >= (request.prompt_len + request.expected_output_len) - for request in requests + for request in all_requests ), ( "Please ensure that max_model_len is greater than the sum of " "prompt_len and expected_output_len for all requests." ) - prompts = [] + if warmup_requests: + print(f"Warming up with {len(warmup_requests)} requests...") + _run_vllm_chat_requests( + llm, + warmup_requests, + n, + disable_detokenize, + do_profile=False, + prequeue_requests=prequeue_requests, + ) + + return _run_vllm_chat_requests( + llm, + requests, + n, + disable_detokenize, + do_profile=do_profile, + prequeue_requests=prequeue_requests, + ) + + +def _run_vllm_chat_requests( + llm: Any, + requests: list[SampleRequest], + n: int, + disable_detokenize: bool, + do_profile: bool, + prequeue_requests: bool, +) -> tuple[float, list[RequestOutput]]: + from vllm import SamplingParams + + prompts = [request.prompt for request in requests] sampling_params: list[SamplingParams] = [] for request in requests: - prompts.append(request.prompt) sampling_params.append( SamplingParams( n=n, @@ -185,6 +253,7 @@ def run_vllm_chat( detokenize=not disable_detokenize, ) ) + if prequeue_requests: llm.sleep(level=0, mode="abort") @@ -214,8 +283,8 @@ async def run_vllm_async( engine_args: AsyncEngineArgs, do_profile: bool, disable_detokenize: bool = False, + warmup_requests: list[SampleRequest] | None = None, ) -> float: - from vllm import SamplingParams from vllm.entrypoints.openai.api_server import ( build_async_engine_client_from_engine_args, ) @@ -224,59 +293,91 @@ async def run_vllm_async( engine_args, ) as llm: model_config = llm.model_config + all_requests = list(warmup_requests or []) + requests assert all( model_config.max_model_len >= (request.prompt_len + request.expected_output_len) - for request in requests + for request in all_requests ), ( "Please ensure that max_model_len is greater than the sum of" " prompt_len and expected_output_len for all requests." ) - # Add the requests to the engine. - prompts: list[TextPrompt | TokensPrompt] = [] - sampling_params: list[SamplingParams] = [] - lora_requests: list[LoRARequest | None] = [] - for request in requests: - prompt = ( - TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"]) - if "prompt_token_ids" in request.prompt - else TextPrompt(prompt=request.prompt) + if warmup_requests: + print(f"Warming up with {len(warmup_requests)} requests...") + await _run_vllm_async_requests( + llm, + warmup_requests, + n, + disable_detokenize, + do_profile=False, + request_id_prefix="warmup", ) - if request.multi_modal_data: - assert isinstance(request.multi_modal_data, dict) - prompt["multi_modal_data"] = request.multi_modal_data + elapsed_time, _ = await _run_vllm_async_requests( + llm, + requests, + n, + disable_detokenize, + do_profile=do_profile, + request_id_prefix="test", + ) + return elapsed_time - sampling_params.append( - SamplingParams( - n=n, - temperature=1.0, - top_p=1.0, - ignore_eos=True, - max_tokens=request.expected_output_len, - detokenize=not disable_detokenize, - ) + +async def _run_vllm_async_requests( + llm: Any, + requests: list[SampleRequest], + n: int, + disable_detokenize: bool, + do_profile: bool, + request_id_prefix: str, +) -> tuple[float, None]: + from vllm import SamplingParams + + prompts: list[TextPrompt | TokensPrompt] = [] + sampling_params: list[SamplingParams] = [] + lora_requests: list[LoRARequest | None] = [] + for request in requests: + prompt = ( + TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"]) + if "prompt_token_ids" in request.prompt + else TextPrompt(prompt=request.prompt) + ) + + if request.multi_modal_data: + assert isinstance(request.multi_modal_data, dict) + prompt["multi_modal_data"] = request.multi_modal_data + + sampling_params.append( + SamplingParams( + n=n, + temperature=1.0, + top_p=1.0, + ignore_eos=True, + max_tokens=request.expected_output_len, + detokenize=not disable_detokenize, ) - prompts.append(prompt) - lora_requests.append(request.lora_request) + ) + prompts.append(prompt) + lora_requests.append(request.lora_request) - generators = [] - start = time.perf_counter() - if do_profile: - await llm.start_profile() - for i, (prompt, sp, lr) in enumerate( - zip(prompts, sampling_params, lora_requests) - ): - generator = llm.generate(prompt, sp, lora_request=lr, request_id=f"test{i}") - generators.append(generator) - all_gens = merge_async_iterators(*generators) - async for i, res in all_gens: - pass - if do_profile: - await llm.stop_profile() - end = time.perf_counter() - return end - start + generators = [] + start = time.perf_counter() + if do_profile: + await llm.start_profile() + for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)): + generator = llm.generate( + prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}" + ) + generators.append(generator) + all_gens = merge_async_iterators(*generators) + async for _i, _res in all_gens: + pass + if do_profile: + await llm.stop_profile() + end = time.perf_counter() + return end - start, None def run_hf( @@ -289,6 +390,7 @@ def run_hf( disable_detokenize: bool = False, dtype: torch.dtype | None = torch.float16, enable_torch_compile: bool = False, + warmup_requests: list[SampleRequest] | None = None, ) -> float: assert isinstance(tokenizer, PreTrainedTokenizerBase), ( "the hf backend only supports HF tokenizers" @@ -303,6 +405,31 @@ def run_hf( if enable_torch_compile: llm = torch.compile(llm) + if warmup_requests: + print(f"Warming up with {len(warmup_requests)} requests...") + _run_hf_requests( + llm, + tokenizer, + warmup_requests, + n, + max_batch_size, + disable_detokenize, + ) + + elapsed_time, _ = _run_hf_requests( + llm, tokenizer, requests, n, max_batch_size, disable_detokenize + ) + return elapsed_time + + +def _run_hf_requests( + llm: Any, + tokenizer: PreTrainedTokenizerBase, + requests: list[SampleRequest], + n: int, + max_batch_size: int, + disable_detokenize: bool, +) -> tuple[float, None]: pbar = tqdm(total=len(requests)) start = time.perf_counter() batch: list[str] = [] @@ -347,8 +474,9 @@ def run_hf( batch = [] max_prompt_len = 0 max_output_len = 0 + pbar.close() end = time.perf_counter() - return end - start + return end - start, None def save_to_pytorch_benchmark_format( @@ -801,6 +929,12 @@ def add_cli_args(parser: argparse.ArgumentParser): parser.add_argument( "--num-prompts", type=int, default=1000, help="Number of prompts to process." ) + parser.add_argument( + "--num-warmups", + type=int, + default=0, + help="Number of warmup prompts to process before the timed benchmark.", + ) parser.add_argument( "--hf-max-batch-size", type=int, @@ -971,6 +1105,14 @@ def main(args: argparse.Namespace): tokenizer_mode=args.tokenizer_mode, trust_remote_code=args.trust_remote_code, ) + num_warmups = args.num_warmups + warmup_requests: list[SampleRequest] | None = None + if num_warmups > 0: + warmup_args = argparse.Namespace(**vars(args)) + warmup_args.num_prompts = num_warmups + warmup_args.seed += 1 + warmup_requests = get_requests(warmup_args, tokenizer) + requests = get_requests(args, tokenizer) is_multi_modal = any(request.multi_modal_data is not None for request in requests) request_outputs: list[RequestOutput] | None = None @@ -983,6 +1125,7 @@ def main(args: argparse.Namespace): AsyncEngineArgs.from_cli_args(args), disable_detokenize=args.disable_detokenize, do_profile=args.profile, + warmup_requests=warmup_requests, ) ) else: @@ -992,6 +1135,7 @@ def main(args: argparse.Namespace): EngineArgs.from_cli_args(args), disable_detokenize=args.disable_detokenize, do_profile=args.profile, + warmup_requests=warmup_requests, prequeue_requests=args.prequeue_requests, ) elif args.backend == "hf": @@ -1008,6 +1152,7 @@ def main(args: argparse.Namespace): args.disable_detokenize, dtype=args.dtype, enable_torch_compile=args.hf_enable_torch_compile, + warmup_requests=warmup_requests, ) elif args.backend == "vllm-chat": elapsed_time, request_outputs = run_vllm_chat( @@ -1016,6 +1161,7 @@ def main(args: argparse.Namespace): EngineArgs.from_cli_args(args), disable_detokenize=args.disable_detokenize, do_profile=args.profile, + warmup_requests=warmup_requests, prequeue_requests=args.prequeue_requests, ) else: