[Rust][Benchmark] Make vllm bench serve Rust delegation opt-in (#50081)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -201,6 +201,7 @@ def test_run_vllm_dp_server_uses_rust_frontend_when_enabled(monkeypatch):
|
||||
monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
|
||||
monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup.envs, "VLLM_USE_RUST_FRONTEND", True, raising=False)
|
||||
monkeypatch.setattr(
|
||||
dp_sup.envs,
|
||||
"VLLM_RUST_FRONTEND_PATH",
|
||||
|
||||
@@ -145,6 +145,15 @@ def test_precompiled_install_flags_are_orthogonal() -> None:
|
||||
assert environment_variables["VLLM_USE_PRECOMPILED_RUST"]() is True
|
||||
|
||||
|
||||
def test_rust_bench_auto_path_missing_fails_fast() -> None:
|
||||
with (
|
||||
patch.dict(os.environ, {"VLLM_USE_RUST_BENCH": "1"}, clear=True),
|
||||
patch("vllm.envs.os.path.isfile", return_value=False),
|
||||
pytest.raises(FileNotFoundError, match="vllm-rs binary was not found"),
|
||||
):
|
||||
environment_variables["VLLM_RUST_FRONTEND_PATH"]()
|
||||
|
||||
|
||||
class TestEnvWithChoices:
|
||||
"""Test cases for env_with_choices function."""
|
||||
|
||||
|
||||
@@ -2,18 +2,38 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def maybe_exec_rust_bench() -> None:
|
||||
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
|
||||
return
|
||||
|
||||
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
|
||||
if rust_cli is None:
|
||||
raise RuntimeError(
|
||||
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
|
||||
"to resolve to the vllm-rs binary."
|
||||
)
|
||||
|
||||
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
|
||||
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
|
||||
|
||||
|
||||
def _import_bench_subcommand_modules() -> None:
|
||||
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
|
||||
|
||||
@@ -1,68 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from vllm.benchmarks.serve import add_cli_args
|
||||
from vllm.benchmarks.serve import main as python_main
|
||||
from vllm.benchmarks.serve import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_RUST_CLI_PATH = Path(__file__).resolve().parents[3] / "vllm-rs"
|
||||
_RUST_SUPPORTED_DATASETS = frozenset(
|
||||
{
|
||||
"custom",
|
||||
"hf",
|
||||
"prefix_repetition",
|
||||
"random",
|
||||
"random-mm",
|
||||
"random-rerank",
|
||||
"sharegpt",
|
||||
"sonnet",
|
||||
"speed_bench",
|
||||
}
|
||||
)
|
||||
_RUST_SUPPORTED_BACKENDS = frozenset(
|
||||
{
|
||||
"openai",
|
||||
"openai-chat",
|
||||
"openai-embeddings",
|
||||
"openai-embeddings-chat",
|
||||
"vllm",
|
||||
"vllm-pooling",
|
||||
"vllm-rerank",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _rust_unsupported_reason(args: argparse.Namespace) -> str | None:
|
||||
if args.dataset_name not in _RUST_SUPPORTED_DATASETS:
|
||||
return f"dataset {args.dataset_name!r} is not supported by the Rust benchmark"
|
||||
if args.backend not in _RUST_SUPPORTED_BACKENDS:
|
||||
return f"backend {args.backend!r} is not supported by the Rust benchmark"
|
||||
return None
|
||||
|
||||
|
||||
def _maybe_exec_rust_bench(args: argparse.Namespace) -> None:
|
||||
if reason := _rust_unsupported_reason(args):
|
||||
logger.info("Using Python benchmark: %s.", reason)
|
||||
return
|
||||
|
||||
if not _RUST_CLI_PATH.is_file():
|
||||
logger.warning(
|
||||
"Rust benchmark binary not found at %s; falling back to Python.",
|
||||
_RUST_CLI_PATH,
|
||||
)
|
||||
return
|
||||
|
||||
rust_cli = str(_RUST_CLI_PATH)
|
||||
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
|
||||
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
|
||||
|
||||
|
||||
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `serve` subcommand for `vllm bench`."""
|
||||
@@ -76,5 +19,4 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
_maybe_exec_rust_bench(args)
|
||||
python_main(args)
|
||||
main(args)
|
||||
|
||||
@@ -54,6 +54,8 @@ def main():
|
||||
logger.info("Delegating entrypoint handling to vllm-omni")
|
||||
omni_main()
|
||||
else:
|
||||
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
|
||||
|
||||
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "bench":
|
||||
logger.debug(
|
||||
|
||||
@@ -58,6 +58,10 @@ class ServeSubcommand(CLISubcommand):
|
||||
uvloop.run(serve_grpc(args))
|
||||
return
|
||||
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
|
||||
if args.headless:
|
||||
if args.api_server_count is not None and args.api_server_count > 0:
|
||||
raise ValueError(
|
||||
@@ -103,7 +107,7 @@ class ServeSubcommand(CLISubcommand):
|
||||
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
|
||||
# - Internal LB: Use full DP size
|
||||
if args.api_server_count is None:
|
||||
if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
|
||||
if is_multi_port or is_external_lb or rust_frontend_path:
|
||||
args.api_server_count = 1
|
||||
elif is_hybrid_lb:
|
||||
args.api_server_count = args.data_parallel_size_local or 1
|
||||
@@ -120,7 +124,7 @@ class ServeSubcommand(CLISubcommand):
|
||||
"Defaulting api_server_count to data_parallel_size (%d).",
|
||||
args.api_server_count,
|
||||
)
|
||||
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
|
||||
elif rust_frontend_path and args.api_server_count > 1:
|
||||
logger.warning(
|
||||
"Ignoring --api-server-count=%d when using rust front-end process",
|
||||
args.api_server_count,
|
||||
@@ -140,7 +144,7 @@ class ServeSubcommand(CLISubcommand):
|
||||
run_dp_supervisor(args)
|
||||
elif args.api_server_count < 1:
|
||||
run_headless(args)
|
||||
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
|
||||
elif args.api_server_count > 1 or rust_frontend_path:
|
||||
run_multi_api_server(args)
|
||||
else:
|
||||
# Single API server (this process).
|
||||
@@ -256,7 +260,9 @@ def run_headless(args: argparse.Namespace):
|
||||
|
||||
def run_multi_api_server(args: argparse.Namespace):
|
||||
assert not args.headless
|
||||
rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
num_api_servers: int = args.api_server_count
|
||||
assert num_api_servers > 0
|
||||
|
||||
|
||||
@@ -257,7 +257,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
name = f"APIServer_DP{child_args.data_parallel_rank}"
|
||||
set_process_title(name)
|
||||
decorate_logs(name)
|
||||
if envs.VLLM_RUST_FRONTEND_PATH:
|
||||
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
|
||||
_run_rust_vllm_dp_server(child_args)
|
||||
else:
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
|
||||
+16
-11
@@ -153,6 +153,7 @@ if TYPE_CHECKING:
|
||||
K_SCALE_CONSTANT: int = 200
|
||||
V_SCALE_CONSTANT: int = 100
|
||||
VLLM_USE_RUST_FRONTEND: bool = False
|
||||
VLLM_USE_RUST_BENCH: bool = False
|
||||
VLLM_RUST_FRONTEND_PATH: str | None = "auto"
|
||||
VLLM_SERVER_DEV_MODE: bool = False
|
||||
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
|
||||
@@ -548,22 +549,24 @@ def _deprecated_triton_attn_use_td() -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_rust_frontend_path() -> str | None:
|
||||
"""Resolve the Rust frontend binary path.
|
||||
def _resolve_rust_cli_path() -> str | None:
|
||||
"""Resolve the vllm-rs binary path.
|
||||
|
||||
Returns None if VLLM_USE_RUST_FRONTEND is not enabled.
|
||||
Returns None unless VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH is enabled.
|
||||
When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
|
||||
to the actual binary path.
|
||||
"""
|
||||
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0")))
|
||||
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) or bool(
|
||||
int(os.environ.get("VLLM_USE_RUST_BENCH", "0"))
|
||||
)
|
||||
raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
|
||||
|
||||
if not use_rust:
|
||||
if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
|
||||
logger.warning(
|
||||
"VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND "
|
||||
"is not enabled. The Rust frontend will not be used. "
|
||||
"Set VLLM_USE_RUST_FRONTEND=1 to enable it."
|
||||
"VLLM_RUST_FRONTEND_PATH is set without enabling "
|
||||
"VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH. "
|
||||
"Set one of them to 1 to use the vllm-rs binary."
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -1340,10 +1343,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_RUST_FRONTEND": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
|
||||
),
|
||||
# Path to the Rust frontend binary. Defaults to "auto" which discovers
|
||||
# the binary installed with the vllm package. Only used when
|
||||
# VLLM_USE_RUST_FRONTEND=1.
|
||||
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(),
|
||||
# If set, use the packaged Rust client for `vllm bench serve`.
|
||||
"VLLM_USE_RUST_BENCH": lambda: bool(int(os.getenv("VLLM_USE_RUST_BENCH", "0"))),
|
||||
# Path to the vllm-rs binary. Defaults to "auto" which discovers the
|
||||
# binary installed with the vllm package. Used when VLLM_USE_RUST_FRONTEND=1
|
||||
# or VLLM_USE_RUST_BENCH=1.
|
||||
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_cli_path(),
|
||||
# If set, vllm will run in development mode, which will enable
|
||||
# some additional endpoints for developing and debugging,
|
||||
# e.g. `/reset_prefix_cache`
|
||||
|
||||
Reference in New Issue
Block a user