[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Alec
2026-10-02 08:14:37 +00:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent b558f160a2
commit 9d2a6f52b2
11 changed files with 320 additions and 64 deletions
+9
View File
@@ -17,6 +17,7 @@ from vllm.entrypoints.launchers.cli_args import (
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
from vllm.entrypoints.launchers.launcher import create_server_socket
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
@@ -295,6 +296,13 @@ def run_multi_api_server(args: argparse.Namespace):
signal.signal(signal.SIGINT, signal_handler)
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
# loopback when HTTP uses a Unix socket.
grpc_sock = None
if args.grpc_port is not None:
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
propagate_flash_late_interaction(args, engine_args)
@@ -355,6 +363,7 @@ def run_multi_api_server(args: argparse.Namespace):
api_server_manager = RustFrontendProcessManager(
binary_path=rust_frontend_path,
sock=sock,
grpc_sock=grpc_sock,
args=args,
input_address=addresses.inputs[0],
output_address=addresses.outputs[0],
+43 -2
View File
@@ -284,6 +284,11 @@ class FrontendArgs(BaseFrontendArgs):
"""Host name."""
port: int = 8000
"""Port number."""
grpc_port: int | None = None
"""Enable the Rust frontend's additional gRPC Inference and Control services
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
with the Python gRPC server's `--grpc` flag."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
@@ -451,8 +456,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"--grpc",
action="store_true",
default=False,
help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
"server. Requires: pip install vllm[grpc].",
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
"Cannot be combined with --grpc-port, which enables the Rust "
"frontend's gRPC services.",
)
parser = FrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
@@ -460,6 +467,38 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
return parser
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
"""Validate the Rust frontend's optional gRPC listener."""
if getattr(args, "grpc_port", None) is None:
return
if args.grpc:
raise ValueError(
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
"with --grpc-port for the Rust frontend."
)
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
if args.headless or (
args.api_server_count is not None and args.api_server_count <= 0
):
raise ValueError(
"--grpc-port requires a Rust frontend; remove --headless "
"and non-positive --api-server-count"
)
if args.data_parallel_multi_port_external_lb:
raise ValueError(
"--grpc-port is incompatible with "
"--data-parallel-multi-port-external-lb: "
"its frontends would share the same gRPC port"
)
if not 0 <= args.grpc_port <= 65535:
raise ValueError("--grpc-port must be between 0 and 65535")
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
@@ -478,6 +517,8 @@ def validate_parsed_serve_args(args: argparse.Namespace):
"the checkpoint's output format."
)
validate_grpc_port_arg(args)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")