add enqueue all option to throughput benchmark (#42975)
Signed-off-by: Philip Maybank <pmaybank@amd.com> Signed-off-by: pmaybank <113125070+pmaybank@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c628a93a64
commit
fadf5d332c
@@ -50,6 +50,7 @@ def run_vllm(
|
||||
engine_args: EngineArgs,
|
||||
do_profile: bool,
|
||||
disable_detokenize: bool = False,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput] | None]:
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
@@ -94,12 +95,29 @@ def run_vllm(
|
||||
|
||||
outputs = None
|
||||
if not use_beam_search:
|
||||
if prequeue_requests:
|
||||
llm.sleep(level=0, mode="abort")
|
||||
|
||||
start = time.perf_counter()
|
||||
if do_profile:
|
||||
llm.start_profile()
|
||||
outputs = llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_requests, use_tqdm=True
|
||||
)
|
||||
|
||||
if prequeue_requests:
|
||||
try:
|
||||
llm.enqueue(
|
||||
prompts,
|
||||
sampling_params,
|
||||
lora_request=lora_requests,
|
||||
use_tqdm=True,
|
||||
)
|
||||
finally:
|
||||
llm.wake_up(tags=["scheduling"])
|
||||
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
|
||||
else:
|
||||
outputs = llm.generate(
|
||||
prompts, sampling_params, lora_request=lora_requests, use_tqdm=True
|
||||
)
|
||||
|
||||
if do_profile:
|
||||
llm.stop_profile()
|
||||
end = time.perf_counter()
|
||||
@@ -133,6 +151,7 @@ def run_vllm_chat(
|
||||
engine_args: EngineArgs,
|
||||
do_profile: bool,
|
||||
disable_detokenize: bool = False,
|
||||
prequeue_requests: bool = False,
|
||||
) -> tuple[float, list[RequestOutput]]:
|
||||
"""
|
||||
Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
|
||||
@@ -166,12 +185,25 @@ def run_vllm_chat(
|
||||
detokenize=not disable_detokenize,
|
||||
)
|
||||
)
|
||||
if prequeue_requests:
|
||||
llm.sleep(level=0, mode="abort")
|
||||
|
||||
start = time.perf_counter()
|
||||
if do_profile:
|
||||
llm.start_profile()
|
||||
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
|
||||
|
||||
if prequeue_requests:
|
||||
try:
|
||||
llm.enqueue_chat(prompts, sampling_params, use_tqdm=True)
|
||||
finally:
|
||||
llm.wake_up(tags=["scheduling"])
|
||||
outputs = llm.wait_for_completion(output_type=RequestOutput, use_tqdm=True)
|
||||
else:
|
||||
outputs = llm.chat(prompts, sampling_params, use_tqdm=True)
|
||||
|
||||
if do_profile:
|
||||
llm.stop_profile()
|
||||
|
||||
end = time.perf_counter()
|
||||
return end - start, outputs
|
||||
|
||||
@@ -543,6 +575,10 @@ def validate_args(args):
|
||||
valid_backends = {"vllm", "hf", "mii", "vllm-chat"}
|
||||
if args.backend not in valid_backends:
|
||||
raise ValueError(f"Unsupported backend: {args.backend}")
|
||||
if args.prequeue_requests and args.backend not in {"vllm", "vllm-chat"}:
|
||||
raise ValueError("--prequeue-requests requires --backend vllm or vllm-chat")
|
||||
if args.prequeue_requests and args.async_engine:
|
||||
raise ValueError("--prequeue-requests is not supported with --async-engine")
|
||||
|
||||
# === Dataset Configuration ===
|
||||
if (
|
||||
@@ -789,6 +825,20 @@ def add_cli_args(parser: argparse.ArgumentParser):
|
||||
default=False,
|
||||
help="Use vLLM async engine rather than LLM class.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prequeue-requests",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"For the vLLM backends, enqueue all requests before allowing the "
|
||||
"scheduler to process them. This can improve benchmark "
|
||||
"reproducibility by removing overlap between request rendering "
|
||||
"and engine scheduling, but may reduce measured throughput. "
|
||||
"Request rendering is typically fast relative to scheduling and "
|
||||
"processing; the intended use case of this flag is multimodal "
|
||||
"benchmarks with time-consuming image rendering."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-detokenize",
|
||||
action="store_true",
|
||||
@@ -942,6 +992,7 @@ def main(args: argparse.Namespace):
|
||||
EngineArgs.from_cli_args(args),
|
||||
disable_detokenize=args.disable_detokenize,
|
||||
do_profile=args.profile,
|
||||
prequeue_requests=args.prequeue_requests,
|
||||
)
|
||||
elif args.backend == "hf":
|
||||
assert args.tensor_parallel_size == 1
|
||||
@@ -965,6 +1016,7 @@ def main(args: argparse.Namespace):
|
||||
EngineArgs.from_cli_args(args),
|
||||
disable_detokenize=args.disable_detokenize,
|
||||
do_profile=args.profile,
|
||||
prequeue_requests=args.prequeue_requests,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown backend: {args.backend}")
|
||||
|
||||
Reference in New Issue
Block a user