[Rust][Benchmark] Make vllm bench serve Rust delegation opt-in (#50081)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 16:21:12 +00:00
committed by GitHub
parent ba702e978e
commit 01661cc57f
8 changed files with 61 additions and 76 deletions
+20
View File
@@ -2,18 +2,38 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import os
import sys
import typing
from vllm import envs
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
def maybe_exec_rust_bench() -> None:
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
return
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
if rust_cli is None:
raise RuntimeError(
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
"to resolve to the vllm-rs binary."
)
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
def _import_bench_subcommand_modules() -> None:
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
+2 -60
View File
@@ -1,68 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import os
import sys
from pathlib import Path
from vllm.benchmarks.serve import add_cli_args
from vllm.benchmarks.serve import main as python_main
from vllm.benchmarks.serve import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
_RUST_CLI_PATH = Path(__file__).resolve().parents[3] / "vllm-rs"
_RUST_SUPPORTED_DATASETS = frozenset(
{
"custom",
"hf",
"prefix_repetition",
"random",
"random-mm",
"random-rerank",
"sharegpt",
"sonnet",
"speed_bench",
}
)
_RUST_SUPPORTED_BACKENDS = frozenset(
{
"openai",
"openai-chat",
"openai-embeddings",
"openai-embeddings-chat",
"vllm",
"vllm-pooling",
"vllm-rerank",
}
)
def _rust_unsupported_reason(args: argparse.Namespace) -> str | None:
if args.dataset_name not in _RUST_SUPPORTED_DATASETS:
return f"dataset {args.dataset_name!r} is not supported by the Rust benchmark"
if args.backend not in _RUST_SUPPORTED_BACKENDS:
return f"backend {args.backend!r} is not supported by the Rust benchmark"
return None
def _maybe_exec_rust_bench(args: argparse.Namespace) -> None:
if reason := _rust_unsupported_reason(args):
logger.info("Using Python benchmark: %s.", reason)
return
if not _RUST_CLI_PATH.is_file():
logger.warning(
"Rust benchmark binary not found at %s; falling back to Python.",
_RUST_CLI_PATH,
)
return
rust_cli = str(_RUST_CLI_PATH)
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
"""The `serve` subcommand for `vllm bench`."""
@@ -76,5 +19,4 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
@staticmethod
def cmd(args: argparse.Namespace) -> None:
_maybe_exec_rust_bench(args)
python_main(args)
main(args)
+2
View File
@@ -54,6 +54,8 @@ def main():
logger.info("Delegating entrypoint handling to vllm-omni")
omni_main()
else:
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
if len(sys.argv) > 1 and sys.argv[1] == "bench":
logger.debug(
+10 -4
View File
@@ -58,6 +58,10 @@ class ServeSubcommand(CLISubcommand):
uvloop.run(serve_grpc(args))
return
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
if args.headless:
if args.api_server_count is not None and args.api_server_count > 0:
raise ValueError(
@@ -103,7 +107,7 @@ class ServeSubcommand(CLISubcommand):
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
if is_multi_port or is_external_lb or rust_frontend_path:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -120,7 +124,7 @@ class ServeSubcommand(CLISubcommand):
"Defaulting api_server_count to data_parallel_size (%d).",
args.api_server_count,
)
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
elif rust_frontend_path and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process",
args.api_server_count,
@@ -140,7 +144,7 @@ class ServeSubcommand(CLISubcommand):
run_dp_supervisor(args)
elif args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
elif args.api_server_count > 1 or rust_frontend_path:
run_multi_api_server(args)
else:
# Single API server (this process).
@@ -256,7 +260,9 @@ def run_headless(args: argparse.Namespace):
def run_multi_api_server(args: argparse.Namespace):
assert not args.headless
rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
num_api_servers: int = args.api_server_count
assert num_api_servers > 0