[Benchmark] Add num-warmup to vllm bench throughput (#43245)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
+200
-54
@@ -50,22 +50,59 @@ def run_vllm(
|
||||
engine_args: EngineArgs,
|
||||
do_profile: bool,
|
||||
disable_detokenize: bool = False,
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput] | None]:
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import LLM
|
||||
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
all_requests = list(warmup_requests or []) + requests
|
||||
assert all(
|
||||
llm.llm_engine.model_config.max_model_len
|
||||
>= (request.prompt_len + request.expected_output_len)
|
||||
for request in requests
|
||||
for request in all_requests
|
||||
), (
|
||||
"Please ensure that max_model_len is greater than the sum of"
|
||||
" prompt_len and expected_output_len for all requests."
|
||||
)
|
||||
# Add the requests to the engine.
|
||||
|
||||
if warmup_requests:
|
||||
print(f"Warming up with {len(warmup_requests)} requests...")
|
||||
_run_vllm_requests(
|
||||
llm,
|
||||
warmup_requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=False,
|
||||
prequeue_requests=prequeue_requests,
|
||||
enable_lora=engine_args.enable_lora,
|
||||
)
|
||||
|
||||
return _run_vllm_requests(
|
||||
llm,
|
||||
requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=do_profile,
|
||||
prequeue_requests=prequeue_requests,
|
||||
enable_lora=engine_args.enable_lora,
|
||||
)
|
||||
|
||||
|
||||
def _run_vllm_requests(
|
||||
llm: Any,
|
||||
requests: list[SampleRequest],
|
||||
n: int,
|
||||
disable_detokenize: bool,
|
||||
do_profile: bool,
|
||||
prequeue_requests: bool,
|
||||
enable_lora: bool,
|
||||
) -> tuple[float, list[RequestOutput] | None]:
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts: list[TextPrompt | TokensPrompt] = []
|
||||
sampling_params: list[SamplingParams] = []
|
||||
lora_requests: list[LoRARequest] | None = [] if enable_lora else None
|
||||
for request in requests:
|
||||
prompt = (
|
||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
||||
@@ -87,9 +124,8 @@ def run_vllm(
|
||||
detokenize=not disable_detokenize,
|
||||
)
|
||||
)
|
||||
lora_requests: list[LoRARequest] | None = None
|
||||
if engine_args.enable_lora:
|
||||
lora_requests = [request.lora_request for request in requests]
|
||||
if lora_requests is not None:
|
||||
lora_requests.append(request.lora_request)
|
||||
|
||||
use_beam_search = False
|
||||
|
||||
@@ -123,7 +159,7 @@ def run_vllm(
|
||||
end = time.perf_counter()
|
||||
else:
|
||||
assert lora_requests is None, "BeamSearch API does not support LoRA"
|
||||
prompts = [request.prompt for request in requests]
|
||||
beam_prompts = [request.prompt for request in requests]
|
||||
# output_len should be the same for all requests.
|
||||
output_len = requests[0].expected_output_len
|
||||
for request in requests:
|
||||
@@ -132,7 +168,7 @@ def run_vllm(
|
||||
if do_profile:
|
||||
llm.start_profile()
|
||||
llm.beam_search(
|
||||
prompts,
|
||||
beam_prompts,
|
||||
BeamSearchParams(
|
||||
beam_width=n,
|
||||
max_tokens=output_len,
|
||||
@@ -151,6 +187,7 @@ def run_vllm_chat(
|
||||
engine_args: EngineArgs,
|
||||
do_profile: bool,
|
||||
disable_detokenize: bool = False,
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput]]:
|
||||
"""
|
||||
@@ -158,23 +195,54 @@ def run_vllm_chat(
|
||||
multimodal models as it properly handles multimodal inputs and chat
|
||||
formatting. For non-multimodal models, use run_vllm() instead.
|
||||
"""
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import LLM
|
||||
|
||||
llm = LLM.from_engine_args(engine_args)
|
||||
|
||||
all_requests = list(warmup_requests or []) + requests
|
||||
assert all(
|
||||
llm.llm_engine.model_config.max_model_len
|
||||
>= (request.prompt_len + request.expected_output_len)
|
||||
for request in requests
|
||||
for request in all_requests
|
||||
), (
|
||||
"Please ensure that max_model_len is greater than the sum of "
|
||||
"prompt_len and expected_output_len for all requests."
|
||||
)
|
||||
|
||||
prompts = []
|
||||
if warmup_requests:
|
||||
print(f"Warming up with {len(warmup_requests)} requests...")
|
||||
_run_vllm_chat_requests(
|
||||
llm,
|
||||
warmup_requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=False,
|
||||
prequeue_requests=prequeue_requests,
|
||||
)
|
||||
|
||||
return _run_vllm_chat_requests(
|
||||
llm,
|
||||
requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=do_profile,
|
||||
prequeue_requests=prequeue_requests,
|
||||
)
|
||||
|
||||
|
||||
def _run_vllm_chat_requests(
|
||||
llm: Any,
|
||||
requests: list[SampleRequest],
|
||||
n: int,
|
||||
disable_detokenize: bool,
|
||||
do_profile: bool,
|
||||
prequeue_requests: bool,
|
||||
) -> tuple[float, list[RequestOutput]]:
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts = [request.prompt for request in requests]
|
||||
sampling_params: list[SamplingParams] = []
|
||||
for request in requests:
|
||||
prompts.append(request.prompt)
|
||||
sampling_params.append(
|
||||
SamplingParams(
|
||||
n=n,
|
||||
@@ -185,6 +253,7 @@ def run_vllm_chat(
|
||||
detokenize=not disable_detokenize,
|
||||
)
|
||||
)
|
||||
|
||||
if prequeue_requests:
|
||||
llm.sleep(level=0, mode="abort")
|
||||
|
||||
@@ -214,8 +283,8 @@ async def run_vllm_async(
|
||||
engine_args: AsyncEngineArgs,
|
||||
do_profile: bool,
|
||||
disable_detokenize: bool = False,
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
) -> float:
|
||||
from vllm import SamplingParams
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
@@ -224,59 +293,91 @@ async def run_vllm_async(
|
||||
engine_args,
|
||||
) as llm:
|
||||
model_config = llm.model_config
|
||||
all_requests = list(warmup_requests or []) + requests
|
||||
assert all(
|
||||
model_config.max_model_len
|
||||
>= (request.prompt_len + request.expected_output_len)
|
||||
for request in requests
|
||||
for request in all_requests
|
||||
), (
|
||||
"Please ensure that max_model_len is greater than the sum of"
|
||||
" prompt_len and expected_output_len for all requests."
|
||||
)
|
||||
|
||||
# Add the requests to the engine.
|
||||
prompts: list[TextPrompt | TokensPrompt] = []
|
||||
sampling_params: list[SamplingParams] = []
|
||||
lora_requests: list[LoRARequest | None] = []
|
||||
for request in requests:
|
||||
prompt = (
|
||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
||||
if "prompt_token_ids" in request.prompt
|
||||
else TextPrompt(prompt=request.prompt)
|
||||
if warmup_requests:
|
||||
print(f"Warming up with {len(warmup_requests)} requests...")
|
||||
await _run_vllm_async_requests(
|
||||
llm,
|
||||
warmup_requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=False,
|
||||
request_id_prefix="warmup",
|
||||
)
|
||||
|
||||
if request.multi_modal_data:
|
||||
assert isinstance(request.multi_modal_data, dict)
|
||||
prompt["multi_modal_data"] = request.multi_modal_data
|
||||
elapsed_time, _ = await _run_vllm_async_requests(
|
||||
llm,
|
||||
requests,
|
||||
n,
|
||||
disable_detokenize,
|
||||
do_profile=do_profile,
|
||||
request_id_prefix="test",
|
||||
)
|
||||
return elapsed_time
|
||||
|
||||
sampling_params.append(
|
||||
SamplingParams(
|
||||
n=n,
|
||||
temperature=1.0,
|
||||
top_p=1.0,
|
||||
ignore_eos=True,
|
||||
max_tokens=request.expected_output_len,
|
||||
detokenize=not disable_detokenize,
|
||||
)
|
||||
|
||||
async def _run_vllm_async_requests(
|
||||
llm: Any,
|
||||
requests: list[SampleRequest],
|
||||
n: int,
|
||||
disable_detokenize: bool,
|
||||
do_profile: bool,
|
||||
request_id_prefix: str,
|
||||
) -> tuple[float, None]:
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts: list[TextPrompt | TokensPrompt] = []
|
||||
sampling_params: list[SamplingParams] = []
|
||||
lora_requests: list[LoRARequest | None] = []
|
||||
for request in requests:
|
||||
prompt = (
|
||||
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
|
||||
if "prompt_token_ids" in request.prompt
|
||||
else TextPrompt(prompt=request.prompt)
|
||||
)
|
||||
|
||||
if request.multi_modal_data:
|
||||
assert isinstance(request.multi_modal_data, dict)
|
||||
prompt["multi_modal_data"] = request.multi_modal_data
|
||||
|
||||
sampling_params.append(
|
||||
SamplingParams(
|
||||
n=n,
|
||||
temperature=1.0,
|
||||
top_p=1.0,
|
||||
ignore_eos=True,
|
||||
max_tokens=request.expected_output_len,
|
||||
detokenize=not disable_detokenize,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
lora_requests.append(request.lora_request)
|
||||
)
|
||||
prompts.append(prompt)
|
||||
lora_requests.append(request.lora_request)
|
||||
|
||||
generators = []
|
||||
start = time.perf_counter()
|
||||
if do_profile:
|
||||
await llm.start_profile()
|
||||
for i, (prompt, sp, lr) in enumerate(
|
||||
zip(prompts, sampling_params, lora_requests)
|
||||
):
|
||||
generator = llm.generate(prompt, sp, lora_request=lr, request_id=f"test{i}")
|
||||
generators.append(generator)
|
||||
all_gens = merge_async_iterators(*generators)
|
||||
async for i, res in all_gens:
|
||||
pass
|
||||
if do_profile:
|
||||
await llm.stop_profile()
|
||||
end = time.perf_counter()
|
||||
return end - start
|
||||
generators = []
|
||||
start = time.perf_counter()
|
||||
if do_profile:
|
||||
await llm.start_profile()
|
||||
for i, (prompt, sp, lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
|
||||
generator = llm.generate(
|
||||
prompt, sp, lora_request=lr, request_id=f"{request_id_prefix}{i}"
|
||||
)
|
||||
generators.append(generator)
|
||||
all_gens = merge_async_iterators(*generators)
|
||||
async for _i, _res in all_gens:
|
||||
pass
|
||||
if do_profile:
|
||||
await llm.stop_profile()
|
||||
end = time.perf_counter()
|
||||
return end - start, None
|
||||
|
||||
|
||||
def run_hf(
|
||||
@@ -289,6 +390,7 @@ def run_hf(
|
||||
disable_detokenize: bool = False,
|
||||
dtype: torch.dtype | None = torch.float16,
|
||||
enable_torch_compile: bool = False,
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
) -> float:
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerBase), (
|
||||
"the hf backend only supports HF tokenizers"
|
||||
@@ -303,6 +405,31 @@ def run_hf(
|
||||
if enable_torch_compile:
|
||||
llm = torch.compile(llm)
|
||||
|
||||
if warmup_requests:
|
||||
print(f"Warming up with {len(warmup_requests)} requests...")
|
||||
_run_hf_requests(
|
||||
llm,
|
||||
tokenizer,
|
||||
warmup_requests,
|
||||
n,
|
||||
max_batch_size,
|
||||
disable_detokenize,
|
||||
)
|
||||
|
||||
elapsed_time, _ = _run_hf_requests(
|
||||
llm, tokenizer, requests, n, max_batch_size, disable_detokenize
|
||||
)
|
||||
return elapsed_time
|
||||
|
||||
|
||||
def _run_hf_requests(
|
||||
llm: Any,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
requests: list[SampleRequest],
|
||||
n: int,
|
||||
max_batch_size: int,
|
||||
disable_detokenize: bool,
|
||||
) -> tuple[float, None]:
|
||||
pbar = tqdm(total=len(requests))
|
||||
start = time.perf_counter()
|
||||
batch: list[str] = []
|
||||
@@ -347,8 +474,9 @@ def run_hf(
|
||||
batch = []
|
||||
max_prompt_len = 0
|
||||
max_output_len = 0
|
||||
pbar.close()
|
||||
end = time.perf_counter()
|
||||
return end - start
|
||||
return end - start, None
|
||||
|
||||
|
||||
def save_to_pytorch_benchmark_format(
|
||||
@@ -801,6 +929,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
|
||||
parser.add_argument(
|
||||
"--num-prompts", type=int, default=1000, help="Number of prompts to process."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-warmups",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Number of warmup prompts to process before the timed benchmark.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-max-batch-size",
|
||||
type=int,
|
||||
@@ -971,6 +1105,14 @@ def main(args: argparse.Namespace):
|
||||
tokenizer_mode=args.tokenizer_mode,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
)
|
||||
num_warmups = args.num_warmups
|
||||
warmup_requests: list[SampleRequest] | None = None
|
||||
if num_warmups > 0:
|
||||
warmup_args = argparse.Namespace(**vars(args))
|
||||
warmup_args.num_prompts = num_warmups
|
||||
warmup_args.seed += 1
|
||||
warmup_requests = get_requests(warmup_args, tokenizer)
|
||||
|
||||
requests = get_requests(args, tokenizer)
|
||||
is_multi_modal = any(request.multi_modal_data is not None for request in requests)
|
||||
request_outputs: list[RequestOutput] | None = None
|
||||
@@ -983,6 +1125,7 @@ def main(args: argparse.Namespace):
|
||||
AsyncEngineArgs.from_cli_args(args),
|
||||
disable_detokenize=args.disable_detokenize,
|
||||
do_profile=args.profile,
|
||||
warmup_requests=warmup_requests,
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -992,6 +1135,7 @@ def main(args: argparse.Namespace):
|
||||
EngineArgs.from_cli_args(args),
|
||||
disable_detokenize=args.disable_detokenize,
|
||||
do_profile=args.profile,
|
||||
warmup_requests=warmup_requests,
|
||||
prequeue_requests=args.prequeue_requests,
|
||||
)
|
||||
elif args.backend == "hf":
|
||||
@@ -1008,6 +1152,7 @@ def main(args: argparse.Namespace):
|
||||
args.disable_detokenize,
|
||||
dtype=args.dtype,
|
||||
enable_torch_compile=args.hf_enable_torch_compile,
|
||||
warmup_requests=warmup_requests,
|
||||
)
|
||||
elif args.backend == "vllm-chat":
|
||||
elapsed_time, request_outputs = run_vllm_chat(
|
||||
@@ -1016,6 +1161,7 @@ def main(args: argparse.Namespace):
|
||||
EngineArgs.from_cli_args(args),
|
||||
disable_detokenize=args.disable_detokenize,
|
||||
do_profile=args.profile,
|
||||
warmup_requests=warmup_requests,
|
||||
prequeue_requests=args.prequeue_requests,
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user