[Frontend][RFC] Rust front-end integration (#40848)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Nick Hill
2026-05-21 12:24:48 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent d97ba29fdc
commit f2ace1d57d
35 changed files with 893 additions and 91 deletions
+9
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import signal
import uvloop
@@ -110,6 +111,14 @@ def cmd_init() -> list[CLISubcommand]:
async def run_launch_fastapi(args: argparse.Namespace) -> None:
"""Run the online serving layer with FastAPI (no GPU inference)."""
# Interrupt initialization if SIGTERM arrives before uvicorn installs
# its own signal handlers. Once uvicorn is running it replaces this.
def _interrupt_init(*_) -> None:
raise KeyboardInterrupt("terminated")
signal.signal(signal.SIGTERM, _interrupt_init)
# 1. Socket binding
listen_address, sock = setup_server(args)
+48 -19
View File
@@ -21,7 +21,11 @@ from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
from vllm.v1.executor import Executor
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
from vllm.v1.utils import APIServerProcessManager, wait_for_completion_or_failure
from vllm.v1.utils import (
APIServerProcessManager,
RustFrontendProcessManager,
wait_for_completion_or_failure,
)
logger = init_logger(__name__)
@@ -81,11 +85,12 @@ class ServeSubcommand(CLISubcommand):
)
# Default api_server_count if not explicitly set.
# - Rust frontend: Use 1 (not applicable as it's multithreaded)
# - External LB: Leave as 1 (external LB handles distribution)
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_external_lb:
if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -102,6 +107,11 @@ class ServeSubcommand(CLISubcommand):
"Defaulting api_server_count to data_parallel_size (%d).",
args.api_server_count,
)
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process"
)
args.api_server_count = 1
# Elastic EP currently only supports running with at most one API server.
if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
@@ -114,7 +124,7 @@ class ServeSubcommand(CLISubcommand):
if args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1:
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
run_multi_api_server(args)
else:
# Single API server (this process).
@@ -230,9 +240,15 @@ def run_headless(args: argparse.Namespace):
def run_multi_api_server(args: argparse.Namespace):
assert not args.headless
rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH
num_api_servers: int = args.api_server_count
assert num_api_servers > 0
if rust_frontend_path and num_api_servers > 1:
raise ValueError(
"VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
)
if num_api_servers > 1:
setup_multiprocess_prometheus()
@@ -270,7 +286,9 @@ def run_multi_api_server(args: argparse.Namespace):
dp_rank = parallel_config.data_parallel_rank
assert parallel_config.local_engines_only or dp_rank == 0
api_server_manager: APIServerProcessManager | None = None
api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
None
)
from vllm.v1.engine.utils import get_engine_zmq_addresses
@@ -279,23 +297,34 @@ def run_multi_api_server(args: argparse.Namespace):
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses, num_api_servers
) as (local_engine_manager, coordinator, addresses, tensor_queue):
# Construct common args for the APIServerProcessManager up-front.
stats_update_address = None
if coordinator:
stats_update_address = coordinator.get_stats_publish_address()
# Start API servers.
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_addresses=addresses.inputs,
output_addresses=addresses.outputs,
stats_update_address=stats_update_address,
tensor_queue=tensor_queue,
stats_update_address = (
coordinator.get_stats_publish_address() if coordinator else None
)
if rust_frontend_path:
# Start rust front-end process.
api_server_manager = RustFrontendProcessManager(
binary_path=rust_frontend_path,
sock=sock,
args=args,
input_address=addresses.inputs[0],
output_address=addresses.outputs[0],
engine_count=parallel_config.data_parallel_size,
stats_update_address=stats_update_address,
)
else:
# Start API server(s).
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_addresses=addresses.inputs,
output_addresses=addresses.outputs,
stats_update_address=stats_update_address,
tensor_queue=tensor_queue,
)
# Wait for API servers.
try:
wait_for_completion_or_failure(
+8 -8
View File
@@ -533,8 +533,7 @@ def validate_api_server_args(args):
@instrument(span_name="API server setup")
def setup_server(args):
"""Validate API server args, set up signal handler, create socket
ready to serve."""
"""Validate API server args and create the server socket."""
log_version_and_model(logger, VLLM_VERSION, args.model)
log_non_default_args(args)
@@ -560,12 +559,6 @@ def setup_server(args):
# many concurrent requests active
set_ulimit()
def signal_handler(*_) -> None:
# Interrupt server on sigterm while initializing
raise KeyboardInterrupt("terminated")
signal.signal(signal.SIGTERM, signal_handler)
if args.uds:
listen_address = f"unix:{args.uds}"
else:
@@ -675,6 +668,13 @@ async def run_server(args, **uvicorn_kwargs) -> None:
# Add process-specific prefix to stdout and stderr.
decorate_logs("APIServer")
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
# own signal handlers. Once uvicorn is running it replaces this.
def _interrupt_init(*_) -> None:
raise KeyboardInterrupt("terminated")
signal.signal(signal.SIGTERM, _interrupt_init)
listen_address, sock = setup_server(args)
await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
+39 -1
View File
@@ -9,6 +9,7 @@ from argparse import Namespace
from http import HTTPStatus
from logging import Logger
from string import Template
from typing import Any
import regex as re
from fastapi import Request
@@ -210,7 +211,7 @@ def get_max_tokens(
)
def log_non_default_args(args: Namespace | EngineArgs):
def get_non_default_args(args: Namespace | EngineArgs) -> dict[str, Any]:
from vllm.entrypoints.openai.cli_args import make_arg_parser
non_default_args = {}
@@ -237,6 +238,43 @@ def log_non_default_args(args: Namespace | EngineArgs):
"Unsupported argument type. Must be Namespace or EngineArgs instance."
)
return non_default_args
def _jsonify_arg_value(value: Any) -> Any:
if value is None or isinstance(value, bool | int | float | str):
return value
if dataclasses.is_dataclass(value) and not isinstance(value, type):
return {
key: _jsonify_arg_value(val)
for key, val in dataclasses.asdict(value).items()
}
if isinstance(value, dict):
return {str(key): _jsonify_arg_value(val) for key, val in value.items()}
if isinstance(value, tuple | list):
return [_jsonify_arg_value(item) for item in value]
if (model_dump := getattr(value, "model_dump", None)) is not None:
return _jsonify_arg_value(model_dump(mode="json"))
if (to_dict := getattr(value, "dict", None)) is not None:
return _jsonify_arg_value(to_dict())
return repr(value)
def jsonify_non_default_args(
args: Namespace | EngineArgs,
*,
exclude: set[str] | None = None,
) -> dict[str, Any]:
non_default_args = get_non_default_args(args)
if exclude is not None:
for key in exclude:
non_default_args.pop(key, None)
return {key: _jsonify_arg_value(value) for key, value in non_default_args.items()}
def log_non_default_args(args: Namespace | EngineArgs):
non_default_args = get_non_default_args(args)
logger.info("non-default args: %s", non_default_args)