[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "vllm-managed-engine"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
anyhow.workspace = true
clap.workspace = true
libc.workspace = true
tokio = { workspace = true, features = ["process"] }
tracing.workspace = true
[dev-dependencies]
expect-test.workspace = true
[lints]
workspace = true
+349
View File
@@ -0,0 +1,349 @@
use std::collections::HashSet;
use std::ffi::OsString;
use clap::error::ErrorKind;
use clap::{Args, CommandFactory};
use crate::{ManagedEngineConfig, allocate_handshake_port};
/// Managed Python headless-engine CLI arguments.
#[derive(Debug, Clone, Args, PartialEq, Eq)]
pub struct ManagedEngineArgs {
/// Python executable used to launch the managed headless vLLM engine.
#[arg(long, env = "VLLM_RS_PYTHON", default_value = "python3")]
pub python: String,
/// Host/IP used both for the managed-engine handshake endpoint and the
/// frontend-advertised input/output ZMQ socket addresses.
#[arg(
long = "data-parallel-address",
visible_alias = "handshake-host",
default_value = "127.0.0.1"
)]
pub handshake_host: String,
/// Optional TCP port for the managed-engine handshake / data-parallel RPC
/// endpoint.
///
/// When omitted, the CLI allocates an ephemeral port automatically.
#[arg(
long = "data-parallel-rpc-port",
visible_alias = "handshake-port",
value_parser = clap::value_parser!(u16).range(1..)
)]
pub handshake_port: Option<u16>,
/// Number of data parallel replicas across the whole deployment.
#[arg(long, default_value_t = 1)]
pub data_parallel_size: usize,
/// Number of data parallel replicas to run on this node.
#[arg(long)]
pub data_parallel_size_local: Option<usize>,
/// Additional arguments forwarded to `python -m vllm.entrypoints.cli.main
/// serve ...`.
///
/// Arguments after an explicit `--` are forwarded verbatim. Before `--`,
/// `vllm-rs serve` automatically keeps recognized frontend options on
/// the Rust side and forwards everything else to Python.
#[arg(
last = true,
allow_hyphen_values = true,
help_heading = "Passthrough arguments"
)]
pub python_args: Vec<String>,
}
impl ManagedEngineArgs {
/// Build the handshake address shared by the Rust frontend and managed
/// Python engine.
pub fn handshake_address(&self, handshake_port: u16) -> String {
format!("tcp://{}:{}", self.handshake_host, handshake_port)
}
/// Resolve the handshake port, either from the CLI argument (if specified)
/// or by allocating a fresh port.
pub fn resolve_handshake_port(&self) -> anyhow::Result<u16> {
self.handshake_port
.map(Ok)
.unwrap_or_else(|| allocate_handshake_port(&self.handshake_host))
}
/// Build the managed Python-engine spawn configuration.
pub fn into_config(
self,
model: String,
max_model_len: Option<u32>,
handshake_port: u16,
) -> ManagedEngineConfig {
let mut python_args = self.python_args;
// Manually forward some args to the Python engine.
if let Some(max_model_len) = max_model_len {
python_args.push("--max-model-len".to_string());
python_args.push(max_model_len.to_string());
}
if let Some(data_parallel_size_local) = self.data_parallel_size_local {
python_args.push("--data-parallel-size-local".to_string());
python_args.push(data_parallel_size_local.to_string());
}
ManagedEngineConfig {
python: self.python,
model,
handshake_host: self.handshake_host,
handshake_port,
data_parallel_size: self.data_parallel_size,
python_args,
}
}
/// Return the number of engines that the Rust frontend should expect to
/// coordinate with.
fn local_engine_count(&self) -> usize {
self.data_parallel_size_local.unwrap_or(self.data_parallel_size)
}
/// Return whether the managed Rust frontend only needs to communicate with
/// colocated engines.
pub fn frontend_local_only(&self) -> bool {
self.data_parallel_size_local != Some(0)
&& self.local_engine_count() == self.data_parallel_size
}
}
/// Python `argparse` accepts these multi-character single-dash aliases, but
/// `clap` cannot model them directly.
const PYTHON_MULTI_CHAR_ALIASES: &[(&str, &str)] = &[
("-asc", "--api-server-count"),
("-pp", "--pipeline-parallel-size"),
("-tp", "--tensor-parallel-size"),
("-dcp", "--decode-context-parallel-size"),
("-pcp", "--prefill-context-parallel-size"),
("-dp", "--data-parallel-size"),
("-dpn", "--data-parallel-rank"),
("-dpr", "--data-parallel-start-rank"),
("-dpl", "--data-parallel-size-local"),
("-dpa", "--data-parallel-address"),
("-dpp", "--data-parallel-rpc-port"),
("-dpb", "--data-parallel-backend"),
("-dph", "--data-parallel-hybrid-lb"),
("-dpe", "--data-parallel-external-lb"),
("-ep", "--enable-expert-parallel"),
("-cc", "--compilation-config"),
("-ac", "--attention-config"),
];
/// Repartition managed-engine argv so Rust-owned flags stay before `--`, while
/// everything else is forwarded to Python.
pub fn repartition_managed_engine_args<C>(
args: &[OsString],
subcommand: Option<&str>,
) -> Result<Vec<OsString>, clap::Error>
where
C: CommandFactory,
{
let command = C::command();
let (prefix, real_args, command) = match subcommand {
Some(subcommand) => {
if !matches_subcommand(args, subcommand) {
return Ok(args.to_vec());
};
let subcommand = command
.find_subcommand(subcommand)
.expect("managed-engine subcommand should exist");
(args[..2].to_vec(), &args[2..], subcommand)
}
None => {
let Some(program) = args.first() else {
return Ok(args.to_vec());
};
(vec![program.clone()], &args[1..], &command)
}
};
let mut repartitioned = prefix;
repartitioned.extend(repartition_real_managed_engine_args(real_args, command)?);
Ok(repartitioned)
}
fn repartition_real_managed_engine_args(
args: &[OsString],
command: &clap::Command,
) -> Result<Vec<OsString>, clap::Error> {
let Some(model) = args.first() else {
return Ok(args.to_vec());
};
let model = model.to_string_lossy();
if is_help_flag(&model) {
return Ok(args.to_vec());
}
if model == "--" || is_option_like(&model) {
return Err(build_missing_model_error(command));
}
let (long_flags, short_flags) = collect_option_names(command);
let (front_args, explicit_passthrough, had_separator) = split_managed_engine_args(&args[1..]);
let normalized_front_args = normalize_python_arg_aliases(front_args);
let mut frontend_chunks = Vec::new();
let mut python_chunks = Vec::new();
let mut current_chunk = Vec::new();
for arg in normalized_front_args {
let text = arg.to_string_lossy();
if is_option_like(&text) && !current_chunk.is_empty() {
push_chunk(
&mut frontend_chunks,
&mut python_chunks,
std::mem::take(&mut current_chunk),
&long_flags,
&short_flags,
);
}
current_chunk.push(arg);
}
if !current_chunk.is_empty() {
push_chunk(
&mut frontend_chunks,
&mut python_chunks,
current_chunk,
&long_flags,
&short_flags,
);
}
let mut repartitioned = vec![args[0].clone()];
repartitioned.extend(frontend_chunks);
if had_separator || !python_chunks.is_empty() || !explicit_passthrough.is_empty() {
repartitioned.push("--".into());
repartitioned.extend(python_chunks);
repartitioned.extend(explicit_passthrough.iter().cloned());
}
Ok(repartitioned)
}
fn matches_subcommand(args: &[OsString], subcommand: &str) -> bool {
args.get(1)
.and_then(|arg| arg.to_str())
.is_some_and(|candidate| candidate == subcommand)
}
fn split_managed_engine_args(args: &[OsString]) -> (&[OsString], &[OsString], bool) {
if let Some(index) = args.iter().position(|arg| arg == "--") {
(&args[..index], &args[index + 1..], true)
} else {
(args, &[], false)
}
}
fn normalize_python_arg_aliases(args: &[OsString]) -> Vec<OsString> {
args.iter()
.map(|arg| {
let text = arg.to_string_lossy();
normalize_python_multi_char_alias(&text)
.map(Into::into)
.unwrap_or_else(|| arg.clone())
})
.collect()
}
fn normalize_python_multi_char_alias(arg: &str) -> Option<String> {
find_python_multi_char_alias(arg).map(|canonical| match arg.split_once('=') {
Some((_, value)) => format!("{canonical}={value}"),
None => canonical.to_string(),
})
}
fn find_python_multi_char_alias(arg: &str) -> Option<&'static str> {
PYTHON_MULTI_CHAR_ALIASES.iter().find_map(|&(alias, canonical)| {
(arg == alias || arg.starts_with(&format!("{alias}="))).then_some(canonical)
})
}
fn push_chunk(
frontend_chunks: &mut Vec<OsString>,
python_chunks: &mut Vec<OsString>,
chunk: Vec<OsString>,
long_flags: &HashSet<String>,
short_flags: &HashSet<char>,
) {
if chunk_head_is_frontend_owned(&chunk, long_flags, short_flags) {
frontend_chunks.extend(chunk);
} else {
python_chunks.extend(chunk);
}
}
fn chunk_head_is_frontend_owned(
chunk: &[OsString],
long_flags: &HashSet<String>,
short_flags: &HashSet<char>,
) -> bool {
let Some(head) = chunk.first() else {
return false;
};
let head = head.to_string_lossy();
if let Some(rest) = head.strip_prefix("--") {
let name = rest.split_once('=').map_or(rest, |(name, _)| name);
return long_flags.contains(name);
}
let Some(rest) = head.strip_prefix('-') else {
return false;
};
let Some(short) = rest.chars().next() else {
return false;
};
short_flags.contains(&short)
}
fn collect_option_names(command: &clap::Command) -> (HashSet<String>, HashSet<char>) {
let mut long_flags = HashSet::new();
let mut short_flags = HashSet::new();
for arg in command.get_arguments() {
if let Some(names) = arg.get_long_and_visible_aliases() {
long_flags.extend(names.into_iter().map(str::to_owned));
}
if let Some(short) = arg.get_short() {
short_flags.insert(short);
}
if let Some(short_aliases) = arg.get_visible_short_aliases() {
short_flags.extend(short_aliases);
}
}
long_flags.insert("help".to_string());
short_flags.insert('h');
(long_flags, short_flags)
}
fn is_option_like(arg: &str) -> bool {
if arg == "--" {
return false;
}
if let Some(rest) = arg.strip_prefix("--") {
return rest.chars().next().is_some_and(char::is_alphabetic);
}
if let Some(rest) = arg.strip_prefix('-') {
return rest.chars().next().is_some_and(char::is_alphabetic);
}
false
}
fn is_help_flag(arg: &str) -> bool {
arg == "-h" || arg == "--help"
}
fn build_missing_model_error(command: &clap::Command) -> clap::Error {
command.clone().error(
ErrorKind::MissingRequiredArgument,
"the model must appear immediately after the command",
)
}
+4
View File
@@ -0,0 +1,4 @@
pub mod cli;
mod process;
pub use process::{ManagedEngineConfig, ManagedEngineHandle, allocate_handshake_port};
+263
View File
@@ -0,0 +1,263 @@
use std::io;
use std::net::TcpListener;
use std::process::{Command as StdCommand, ExitStatus, Stdio};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use anyhow::{Context, Result};
use tokio::process::{Child, Command};
use tokio::sync::Mutex;
use tokio::time::interval;
use tracing::info;
const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(200);
const MIN_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
/// Allocate one ephemeral TCP port for the managed headless-engine handshake on
/// the given host.
pub fn allocate_handshake_port(host: &str) -> Result<u16> {
let listener = TcpListener::bind((host, 0)).context("failed to allocate handshake port")?;
let port = listener
.local_addr()
.context("failed to inspect allocated handshake listener address")?
.port();
Ok(port)
}
/// Spawn configuration for one managed headless Python vLLM engine.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ManagedEngineConfig {
/// Python executable used to launch `vllm.entrypoints.cli.main`.
pub python: String,
/// Model identifier passed to `vllm ... serve <model>`.
pub model: String,
/// Host portion of the headless-engine handshake endpoint.
pub handshake_host: String,
/// Port portion of the headless-engine handshake endpoint.
pub handshake_port: u16,
/// Number of data parallel replicas across the whole deployment.
///
/// The per-node replica count is forwarded separately in `python_args` as
/// `--data-parallel-size-local`.
pub data_parallel_size: usize,
/// Extra CLI arguments forwarded verbatim to Python vLLM.
pub python_args: Vec<String>,
}
impl ManagedEngineConfig {
/// Render the handshake address that the Rust frontend should dial.
pub fn handshake_address(&self) -> String {
format!("tcp://{}:{}", self.handshake_host, self.handshake_port)
}
/// Build the concrete Python command line for the managed headless engine.
pub fn to_command(&self) -> StdCommand {
let mut command = StdCommand::new(&self.python);
command
.arg("-m")
.arg("vllm.entrypoints.cli.main")
.arg("serve")
.arg(&self.model)
.arg("--headless")
.arg("--data-parallel-address")
.arg(&self.handshake_host)
.arg("--data-parallel-rpc-port")
.arg(self.handshake_port.to_string())
.arg("--data-parallel-size")
.arg(self.data_parallel_size.to_string())
.args(&self.python_args);
command
}
}
/// RAII-style handle for one managed Python headless engine subprocess.
#[derive(Clone)]
pub struct ManagedEngineHandle {
child: Arc<Mutex<Child>>,
shutdown_started: Arc<AtomicBool>,
}
impl ManagedEngineHandle {
/// Spawn one managed Python headless engine and return a handle for
/// monitoring it.
pub async fn spawn(config: ManagedEngineConfig) -> Result<Self> {
let command = config.to_command();
info!(
handshake_address = %config.handshake_address(),
?command,
"starting managed Python headless engine"
);
let mut command = Command::from(command);
command.stdin(Stdio::null()).stdout(Stdio::inherit()).stderr(Stdio::inherit());
process_group::configure(&mut command);
let child = command.spawn().context("failed to spawn managed engine")?;
Ok(Self {
child: Arc::new(Mutex::new(child)),
shutdown_started: Arc::new(AtomicBool::new(false)),
})
}
/// Poll whether the managed engine has exited yet.
pub async fn try_wait(&self) -> Option<ExitStatus> {
let mut child = self.child.lock().await;
child.try_wait().expect("failed to poll the status of managed engine")
}
/// Wait until the managed engine exits.
pub async fn wait_for_exit(&self) -> ExitStatus {
let mut interval = interval(CHILD_POLL_INTERVAL);
loop {
interval.tick().await;
if let Some(status) = self.try_wait().await {
return status;
}
}
}
/// Terminate the managed engine process group and wait for it to stop.
pub async fn shutdown(&self, timeout: Duration) -> Result<()> {
if self.shutdown_started.swap(true, Ordering::SeqCst) {
return Ok(());
}
let Some(pid) = self.child.lock().await.id() else {
return Ok(());
};
// Enforce a minimum shutdown timeout to give the engine process enough time to
// clean up.
let shutdown_timeout = std::cmp::max(timeout, MIN_SHUTDOWN_TIMEOUT);
// First, try to gracefully terminate.
info!(
pid,
?shutdown_timeout,
"shutting down managed engine with SIGTERM"
);
process_group::terminate(pid)?;
// Wait for the process to exit on its own.
if tokio::time::timeout(shutdown_timeout, self.wait_for_exit()).await.is_ok() {
return Ok(());
}
// If it doesn't exit within the timeout, force kill it.
info!(
pid,
"managed engine did not exit within timeout, sending SIGKILL"
);
process_group::kill(pid)?;
let _ = self.wait_for_exit().await;
Ok(())
}
}
/// Process group helper functions for managing the Python subprocess and its
/// potential children in a platform-aware way.
mod process_group {
use super::*;
/// Place the Python child into its own process group so `serve` can tear
/// down the whole subtree rather than just the immediate shell process.
pub fn configure(command: &mut Command) {
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) != 0 {
return Err(io::Error::last_os_error());
}
Ok(())
});
}
}
/// Send SIGTERM to the managed Python process group.
pub fn terminate(pid: u32) -> Result<()> {
signal(pid, libc::SIGTERM)
}
/// Send SIGKILL to the managed Python process group.
pub fn kill(pid: u32) -> Result<()> {
signal(pid, libc::SIGKILL)
}
/// Deliver one signal to the managed Python process group.
fn signal(pid: u32, signal: i32) -> Result<()> {
let rc = unsafe { libc::kill(-(pid as i32), signal) };
if rc == 0 {
return Ok(());
}
let error = io::Error::last_os_error();
if matches!(error.raw_os_error(), Some(code) if code == libc::ESRCH) {
return Ok(());
}
Err(error).context("failed to signal managed engine process group")
}
}
#[cfg(test)]
mod tests {
use expect_test::expect;
use super::{ManagedEngineConfig, allocate_handshake_port};
#[test]
fn command_snapshot() {
let config = ManagedEngineConfig {
python: "python3".to_string(),
model: "Qwen/Qwen3-0.6B".to_string(),
handshake_host: "127.0.0.1".to_string(),
handshake_port: 62100,
data_parallel_size: 4,
python_args: vec![
"--data-parallel-size-local".to_string(),
"2".to_string(),
"--data-parallel-start-rank".to_string(),
"2".to_string(),
"--dtype".to_string(),
"float16".to_string(),
"--max-model-len".to_string(),
"512".to_string(),
],
};
let command = config.to_command();
let args = command.get_args().collect::<Vec<_>>();
expect![[r#"
[
"-m",
"vllm.entrypoints.cli.main",
"serve",
"Qwen/Qwen3-0.6B",
"--headless",
"--data-parallel-address",
"127.0.0.1",
"--data-parallel-rpc-port",
"62100",
"--data-parallel-size",
"4",
"--data-parallel-size-local",
"2",
"--data-parallel-start-rank",
"2",
"--dtype",
"float16",
"--max-model-len",
"512",
]
"#]]
.assert_debug_eq(&args);
}
#[test]
fn allocate_handshake_port_returns_non_zero_port() {
let port = allocate_handshake_port("127.0.0.1").unwrap();
assert_ne!(port, 0);
}
}