[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Alec Flowers <aflowers@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
b558f160a2
commit
9d2a6f52b2
@@ -17,6 +17,7 @@ from vllm.entrypoints.launchers.cli_args import (
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
|
||||
from vllm.entrypoints.launchers.launcher import create_server_socket
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
@@ -295,6 +296,13 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
|
||||
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
|
||||
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
|
||||
# loopback when HTTP uses a Unix socket.
|
||||
grpc_sock = None
|
||||
if args.grpc_port is not None:
|
||||
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
|
||||
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
@@ -355,6 +363,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
api_server_manager = RustFrontendProcessManager(
|
||||
binary_path=rust_frontend_path,
|
||||
sock=sock,
|
||||
grpc_sock=grpc_sock,
|
||||
args=args,
|
||||
input_address=addresses.inputs[0],
|
||||
output_address=addresses.outputs[0],
|
||||
|
||||
@@ -284,6 +284,11 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
grpc_port: int | None = None
|
||||
"""Enable the Rust frontend's additional gRPC Inference and Control services
|
||||
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
|
||||
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
|
||||
with the Python gRPC server's `--grpc` flag."""
|
||||
data_parallel_supervisor_port: int = 9256
|
||||
"""HTTP port for aggregated health endpoints in multi-port external LB
|
||||
mode."""
|
||||
@@ -451,8 +456,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"--grpc",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
|
||||
"server. Requires: pip install vllm[grpc].",
|
||||
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
|
||||
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
|
||||
"Cannot be combined with --grpc-port, which enables the Rust "
|
||||
"frontend's gRPC services.",
|
||||
)
|
||||
parser = FrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
@@ -460,6 +467,38 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
|
||||
"""Validate the Rust frontend's optional gRPC listener."""
|
||||
if getattr(args, "grpc_port", None) is None:
|
||||
return
|
||||
|
||||
if args.grpc:
|
||||
raise ValueError(
|
||||
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
|
||||
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
|
||||
"with --grpc-port for the Rust frontend."
|
||||
)
|
||||
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
|
||||
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
|
||||
if args.headless or (
|
||||
args.api_server_count is not None and args.api_server_count <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
"--grpc-port requires a Rust frontend; remove --headless "
|
||||
"and non-positive --api-server-count"
|
||||
)
|
||||
if args.data_parallel_multi_port_external_lb:
|
||||
raise ValueError(
|
||||
"--grpc-port is incompatible with "
|
||||
"--data-parallel-multi-port-external-lb: "
|
||||
"its frontends would share the same gRPC port"
|
||||
)
|
||||
if not 0 <= args.grpc_port <= 65535:
|
||||
raise ValueError("--grpc-port must be between 0 and 65535")
|
||||
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
|
||||
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
|
||||
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
@@ -478,6 +517,8 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"the checkpoint's output format."
|
||||
)
|
||||
|
||||
validate_grpc_port_arg(args)
|
||||
|
||||
# Enable auto tool needs a tool call parser to be valid
|
||||
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
||||
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
|
||||
|
||||
Reference in New Issue
Block a user