[Frontend][RFC] Rust front-end integration (#40848)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Nick Hill
2026-05-21 12:24:48 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent d97ba29fdc
commit f2ace1d57d
35 changed files with 893 additions and 91 deletions
+143 -2
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import contextlib
import json
import multiprocessing
import threading
import time
@@ -233,6 +234,146 @@ class APIServerProcessManager:
shutdown(self.processes, timeout=timeout)
class RustFrontendProcessManager:
"""Manages a single Rust frontend subprocess.
Launches the Rust vllm-rs binary in 'frontend' mode, passing the
listening socket fd and ZMQ transport addresses. Provides the same
interface as APIServerProcessManager for process monitoring.
"""
def __init__(
self,
binary_path: str,
sock: Any,
args: argparse.Namespace,
input_address: str,
output_address: str,
engine_count: int,
stats_update_address: str | None = None,
):
import os
import subprocess
fd = sock.fileno()
os.set_inheritable(fd, True)
cmd = [
binary_path,
"frontend",
"--listen-fd",
str(fd),
"--input-address",
input_address,
"--output-address",
output_address,
"--engine-count",
str(engine_count),
]
if stats_update_address is not None:
cmd.extend(["--coordinator-address", stats_update_address])
from vllm.entrypoints.utils import jsonify_non_default_args
args_json = json.dumps(
jsonify_non_default_args(args, exclude={"api_server_count"}),
sort_keys=True,
)
cmd.extend(["--args-json", args_json])
logger.info("Launching Rust frontend: %s", " ".join(cmd))
self._proc = subprocess.Popen(cmd, pass_fds=(fd,))
# Create a process wrapper with a sentinel fd for monitoring
self.processes: list[_SubprocessWrapper] = [
_SubprocessWrapper(self._proc, "RustFrontend")
]
self._finalizer = weakref.finalize(self, _shutdown_subprocesses, self.processes)
def shutdown(self, timeout: float | None = None) -> None:
if self._finalizer.detach() is not None:
_shutdown_subprocesses(self.processes, timeout=timeout)
class _SubprocessWrapper:
"""Wraps subprocess.Popen to provide the BaseProcess-like interface
needed by wait_for_completion_or_failure."""
def __init__(self, proc, name: str):
self._proc = proc
self.name = name
self.pid = proc.pid
self._sentinel_conn: connection.Connection | None = None
self._sentinel_send: connection.Connection | None = None
# Use a Pipe-based sentinel so subprocess monitoring works uniformly
# across platforms with multiprocessing.connection.wait().
recv, send = connection.Pipe(duplex=False)
self._sentinel_conn = recv
self._sentinel_send = send
def monitor_subprocess() -> None:
try:
proc.wait()
finally:
with contextlib.suppress(Exception):
send.close()
threading.Thread(
target=monitor_subprocess, daemon=True, name=f"{name}Monitor"
).start()
@property
def sentinel(self):
return self._sentinel_conn
@property
def exitcode(self) -> int | None:
return self._proc.returncode if self._proc.poll() is not None else None
def is_alive(self) -> bool:
return self._proc.poll() is None
def terminate(self):
self._proc.terminate()
def join(self, timeout=None):
with contextlib.suppress(Exception):
self._proc.wait(timeout=timeout)
def __del__(self):
with contextlib.suppress(Exception):
if self._sentinel_conn is not None:
self._sentinel_conn.close()
if self._sentinel_send is not None:
self._sentinel_send.close()
def _shutdown_subprocesses(
procs: list[_SubprocessWrapper], timeout: float | None = None
) -> None:
"""Shutdown subprocess wrappers (mirrors the shutdown() function)."""
if timeout is None:
timeout = 0.0
timeout = max(timeout, 5.0)
for proc in procs:
if proc.is_alive():
proc.terminate()
deadline = time.monotonic() + timeout
for proc in procs:
remaining = deadline - time.monotonic()
if remaining <= 0:
break
if proc.is_alive():
proc.join(remaining)
for proc in procs:
if proc.is_alive() and (pid := proc.pid) is not None:
kill_process_tree(pid)
def run_api_server_worker_proc(
listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
@@ -253,7 +394,7 @@ def run_api_server_worker_proc(
def wait_for_completion_or_failure(
api_server_manager: APIServerProcessManager,
api_server_manager: "APIServerProcessManager | RustFrontendProcessManager",
engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]
| None = None,
coordinator: "DPCoordinator | None" = None,
@@ -274,7 +415,7 @@ def wait_for_completion_or_failure(
logger.info("Waiting for API servers to complete ...")
# Create a mapping of sentinels to their corresponding processes
# for efficient lookup
sentinel_to_proc: dict[Any, BaseProcess] = {
sentinel_to_proc: dict[Any, BaseProcess | _SubprocessWrapper | None] = {
proc.sentinel: proc for proc in api_server_manager.processes
}