diff --git a/tests/entrypoints/openai/test_dp_supervisor.py b/tests/entrypoints/openai/test_dp_supervisor.py index 576e7ef16d..df80053deb 100644 --- a/tests/entrypoints/openai/test_dp_supervisor.py +++ b/tests/entrypoints/openai/test_dp_supervisor.py @@ -201,6 +201,7 @@ def test_run_vllm_dp_server_uses_rust_frontend_when_enabled(monkeypatch): monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None) monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None) monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None) + monkeypatch.setattr(dp_sup.envs, "VLLM_USE_RUST_FRONTEND", True, raising=False) monkeypatch.setattr( dp_sup.envs, "VLLM_RUST_FRONTEND_PATH", diff --git a/tests/test_envs.py b/tests/test_envs.py index 56c04dd6f2..5917c28fab 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -145,6 +145,15 @@ def test_precompiled_install_flags_are_orthogonal() -> None: assert environment_variables["VLLM_USE_PRECOMPILED_RUST"]() is True +def test_rust_bench_auto_path_missing_fails_fast() -> None: + with ( + patch.dict(os.environ, {"VLLM_USE_RUST_BENCH": "1"}, clear=True), + patch("vllm.envs.os.path.isfile", return_value=False), + pytest.raises(FileNotFoundError, match="vllm-rs binary was not found"), + ): + environment_variables["VLLM_RUST_FRONTEND_PATH"]() + + class TestEnvWithChoices: """Test cases for env_with_choices function.""" diff --git a/vllm/entrypoints/cli/benchmark/main.py b/vllm/entrypoints/cli/benchmark/main.py index 1afac64b14..9ea4998709 100644 --- a/vllm/entrypoints/cli/benchmark/main.py +++ b/vllm/entrypoints/cli/benchmark/main.py @@ -2,18 +2,38 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import os import sys import typing +from vllm import envs from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG +from vllm.logger import init_logger if typing.TYPE_CHECKING: from vllm.utils.argparse_utils import FlexibleArgumentParser else: FlexibleArgumentParser = argparse.ArgumentParser +logger = init_logger(__name__) + + +def maybe_exec_rust_bench() -> None: + if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH: + return + + rust_cli = envs.VLLM_RUST_FRONTEND_PATH + if rust_cli is None: + raise RuntimeError( + "VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH " + "to resolve to the vllm-rs binary." + ) + + logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli) + os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]]) + def _import_bench_subcommand_modules() -> None: # Imported lazily so `BenchmarkSubcommandBase` subclasses register only diff --git a/vllm/entrypoints/cli/benchmark/serve.py b/vllm/entrypoints/cli/benchmark/serve.py index 41a65273ba..188afd6c70 100644 --- a/vllm/entrypoints/cli/benchmark/serve.py +++ b/vllm/entrypoints/cli/benchmark/serve.py @@ -1,68 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse -import os -import sys -from pathlib import Path -from vllm.benchmarks.serve import add_cli_args -from vllm.benchmarks.serve import main as python_main +from vllm.benchmarks.serve import add_cli_args, main from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase -from vllm.logger import init_logger from vllm.utils.argparse_utils import FlexibleArgumentParser -logger = init_logger(__name__) -_RUST_CLI_PATH = Path(__file__).resolve().parents[3] / "vllm-rs" -_RUST_SUPPORTED_DATASETS = frozenset( - { - "custom", - "hf", - "prefix_repetition", - "random", - "random-mm", - "random-rerank", - "sharegpt", - "sonnet", - "speed_bench", - } -) -_RUST_SUPPORTED_BACKENDS = frozenset( - { - "openai", - "openai-chat", - "openai-embeddings", - "openai-embeddings-chat", - "vllm", - "vllm-pooling", - "vllm-rerank", - } -) - - -def _rust_unsupported_reason(args: argparse.Namespace) -> str | None: - if args.dataset_name not in _RUST_SUPPORTED_DATASETS: - return f"dataset {args.dataset_name!r} is not supported by the Rust benchmark" - if args.backend not in _RUST_SUPPORTED_BACKENDS: - return f"backend {args.backend!r} is not supported by the Rust benchmark" - return None - - -def _maybe_exec_rust_bench(args: argparse.Namespace) -> None: - if reason := _rust_unsupported_reason(args): - logger.info("Using Python benchmark: %s.", reason) - return - - if not _RUST_CLI_PATH.is_file(): - logger.warning( - "Rust benchmark binary not found at %s; falling back to Python.", - _RUST_CLI_PATH, - ) - return - - rust_cli = str(_RUST_CLI_PATH) - logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli) - os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]]) - class BenchmarkServingSubcommand(BenchmarkSubcommandBase): """The `serve` subcommand for `vllm bench`.""" @@ -76,5 +19,4 @@ class BenchmarkServingSubcommand(BenchmarkSubcommandBase): @staticmethod def cmd(args: argparse.Namespace) -> None: - _maybe_exec_rust_bench(args) - python_main(args) + main(args) diff --git a/vllm/entrypoints/cli/main.py b/vllm/entrypoints/cli/main.py index fe0b339b3e..3dc69dd3ad 100644 --- a/vllm/entrypoints/cli/main.py +++ b/vllm/entrypoints/cli/main.py @@ -54,6 +54,8 @@ def main(): logger.info("Delegating entrypoint handling to vllm-omni") omni_main() else: + vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench() + # For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default if len(sys.argv) > 1 and sys.argv[1] == "bench": logger.debug( diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index d5e9b2bc87..08cb79f208 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -58,6 +58,10 @@ class ServeSubcommand(CLISubcommand): uvloop.run(serve_grpc(args)) return + rust_frontend_path = ( + envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None + ) + if args.headless: if args.api_server_count is not None and args.api_server_count > 0: raise ValueError( @@ -103,7 +107,7 @@ class ServeSubcommand(CLISubcommand): # - Hybrid LB: Use local DP size (internal LB for local ranks only) # - Internal LB: Use full DP size if args.api_server_count is None: - if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH: + if is_multi_port or is_external_lb or rust_frontend_path: args.api_server_count = 1 elif is_hybrid_lb: args.api_server_count = args.data_parallel_size_local or 1 @@ -120,7 +124,7 @@ class ServeSubcommand(CLISubcommand): "Defaulting api_server_count to data_parallel_size (%d).", args.api_server_count, ) - elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1: + elif rust_frontend_path and args.api_server_count > 1: logger.warning( "Ignoring --api-server-count=%d when using rust front-end process", args.api_server_count, @@ -140,7 +144,7 @@ class ServeSubcommand(CLISubcommand): run_dp_supervisor(args) elif args.api_server_count < 1: run_headless(args) - elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH: + elif args.api_server_count > 1 or rust_frontend_path: run_multi_api_server(args) else: # Single API server (this process). @@ -256,7 +260,9 @@ def run_headless(args: argparse.Namespace): def run_multi_api_server(args: argparse.Namespace): assert not args.headless - rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH + rust_frontend_path = ( + envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None + ) num_api_servers: int = args.api_server_count assert num_api_servers > 0 diff --git a/vllm/entrypoints/openai/dp_supervisor.py b/vllm/entrypoints/openai/dp_supervisor.py index d669ec4d1d..8ce6233c1b 100644 --- a/vllm/entrypoints/openai/dp_supervisor.py +++ b/vllm/entrypoints/openai/dp_supervisor.py @@ -257,7 +257,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None: name = f"APIServer_DP{child_args.data_parallel_rank}" set_process_title(name) decorate_logs(name) - if envs.VLLM_RUST_FRONTEND_PATH: + if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH: _run_rust_vllm_dp_server(child_args) else: _run_python_vllm_dp_server(child_args) diff --git a/vllm/envs.py b/vllm/envs.py index f7e01c3327..84ec5a8af8 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -153,6 +153,7 @@ if TYPE_CHECKING: K_SCALE_CONSTANT: int = 200 V_SCALE_CONSTANT: int = 100 VLLM_USE_RUST_FRONTEND: bool = False + VLLM_USE_RUST_BENCH: bool = False VLLM_RUST_FRONTEND_PATH: str | None = "auto" VLLM_SERVER_DEV_MODE: bool = False VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128 @@ -548,22 +549,24 @@ def _deprecated_triton_attn_use_td() -> None: return None -def _resolve_rust_frontend_path() -> str | None: - """Resolve the Rust frontend binary path. +def _resolve_rust_cli_path() -> str | None: + """Resolve the vllm-rs binary path. - Returns None if VLLM_USE_RUST_FRONTEND is not enabled. + Returns None unless VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH is enabled. When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default) to the actual binary path. """ - use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) + use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0"))) or bool( + int(os.environ.get("VLLM_USE_RUST_BENCH", "0")) + ) raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto") if not use_rust: if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None: logger.warning( - "VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND " - "is not enabled. The Rust frontend will not be used. " - "Set VLLM_USE_RUST_FRONTEND=1 to enable it." + "VLLM_RUST_FRONTEND_PATH is set without enabling " + "VLLM_USE_RUST_FRONTEND or VLLM_USE_RUST_BENCH. " + "Set one of them to 1 to use the vllm-rs binary." ) return None @@ -1340,10 +1343,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_USE_RUST_FRONTEND": lambda: bool( int(os.getenv("VLLM_USE_RUST_FRONTEND", "0")) ), - # Path to the Rust frontend binary. Defaults to "auto" which discovers - # the binary installed with the vllm package. Only used when - # VLLM_USE_RUST_FRONTEND=1. - "VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(), + # If set, use the packaged Rust client for `vllm bench serve`. + "VLLM_USE_RUST_BENCH": lambda: bool(int(os.getenv("VLLM_USE_RUST_BENCH", "0"))), + # Path to the vllm-rs binary. Defaults to "auto" which discovers the + # binary installed with the vllm package. Used when VLLM_USE_RUST_FRONTEND=1 + # or VLLM_USE_RUST_BENCH=1. + "VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_cli_path(), # If set, vllm will run in development mode, which will enable # some additional endpoints for developing and debugging, # e.g. `/reset_prefix_cache`