[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "vllm-managed-engine"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
libc.workspace = true
|
||||
tokio = { workspace = true, features = ["process"] }
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
expect-test.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,349 @@
|
||||
use std::collections::HashSet;
|
||||
use std::ffi::OsString;
|
||||
|
||||
use clap::error::ErrorKind;
|
||||
use clap::{Args, CommandFactory};
|
||||
|
||||
use crate::{ManagedEngineConfig, allocate_handshake_port};
|
||||
|
||||
/// Managed Python headless-engine CLI arguments.
|
||||
#[derive(Debug, Clone, Args, PartialEq, Eq)]
|
||||
pub struct ManagedEngineArgs {
|
||||
/// Python executable used to launch the managed headless vLLM engine.
|
||||
#[arg(long, env = "VLLM_RS_PYTHON", default_value = "python3")]
|
||||
pub python: String,
|
||||
/// Host/IP used both for the managed-engine handshake endpoint and the
|
||||
/// frontend-advertised input/output ZMQ socket addresses.
|
||||
#[arg(
|
||||
long = "data-parallel-address",
|
||||
visible_alias = "handshake-host",
|
||||
default_value = "127.0.0.1"
|
||||
)]
|
||||
pub handshake_host: String,
|
||||
/// Optional TCP port for the managed-engine handshake / data-parallel RPC
|
||||
/// endpoint.
|
||||
///
|
||||
/// When omitted, the CLI allocates an ephemeral port automatically.
|
||||
#[arg(
|
||||
long = "data-parallel-rpc-port",
|
||||
visible_alias = "handshake-port",
|
||||
value_parser = clap::value_parser!(u16).range(1..)
|
||||
)]
|
||||
pub handshake_port: Option<u16>,
|
||||
/// Number of data parallel replicas across the whole deployment.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub data_parallel_size: usize,
|
||||
/// Number of data parallel replicas to run on this node.
|
||||
#[arg(long)]
|
||||
pub data_parallel_size_local: Option<usize>,
|
||||
|
||||
/// Additional arguments forwarded to `python -m vllm.entrypoints.cli.main
|
||||
/// serve ...`.
|
||||
///
|
||||
/// Arguments after an explicit `--` are forwarded verbatim. Before `--`,
|
||||
/// `vllm-rs serve` automatically keeps recognized frontend options on
|
||||
/// the Rust side and forwards everything else to Python.
|
||||
#[arg(
|
||||
last = true,
|
||||
allow_hyphen_values = true,
|
||||
help_heading = "Passthrough arguments"
|
||||
)]
|
||||
pub python_args: Vec<String>,
|
||||
}
|
||||
|
||||
impl ManagedEngineArgs {
|
||||
/// Build the handshake address shared by the Rust frontend and managed
|
||||
/// Python engine.
|
||||
pub fn handshake_address(&self, handshake_port: u16) -> String {
|
||||
format!("tcp://{}:{}", self.handshake_host, handshake_port)
|
||||
}
|
||||
|
||||
/// Resolve the handshake port, either from the CLI argument (if specified)
|
||||
/// or by allocating a fresh port.
|
||||
pub fn resolve_handshake_port(&self) -> anyhow::Result<u16> {
|
||||
self.handshake_port
|
||||
.map(Ok)
|
||||
.unwrap_or_else(|| allocate_handshake_port(&self.handshake_host))
|
||||
}
|
||||
|
||||
/// Build the managed Python-engine spawn configuration.
|
||||
pub fn into_config(
|
||||
self,
|
||||
model: String,
|
||||
max_model_len: Option<u32>,
|
||||
handshake_port: u16,
|
||||
) -> ManagedEngineConfig {
|
||||
let mut python_args = self.python_args;
|
||||
// Manually forward some args to the Python engine.
|
||||
if let Some(max_model_len) = max_model_len {
|
||||
python_args.push("--max-model-len".to_string());
|
||||
python_args.push(max_model_len.to_string());
|
||||
}
|
||||
if let Some(data_parallel_size_local) = self.data_parallel_size_local {
|
||||
python_args.push("--data-parallel-size-local".to_string());
|
||||
python_args.push(data_parallel_size_local.to_string());
|
||||
}
|
||||
|
||||
ManagedEngineConfig {
|
||||
python: self.python,
|
||||
model,
|
||||
handshake_host: self.handshake_host,
|
||||
handshake_port,
|
||||
data_parallel_size: self.data_parallel_size,
|
||||
python_args,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the number of engines that the Rust frontend should expect to
|
||||
/// coordinate with.
|
||||
fn local_engine_count(&self) -> usize {
|
||||
self.data_parallel_size_local.unwrap_or(self.data_parallel_size)
|
||||
}
|
||||
|
||||
/// Return whether the managed Rust frontend only needs to communicate with
|
||||
/// colocated engines.
|
||||
pub fn frontend_local_only(&self) -> bool {
|
||||
self.data_parallel_size_local != Some(0)
|
||||
&& self.local_engine_count() == self.data_parallel_size
|
||||
}
|
||||
}
|
||||
|
||||
/// Python `argparse` accepts these multi-character single-dash aliases, but
|
||||
/// `clap` cannot model them directly.
|
||||
const PYTHON_MULTI_CHAR_ALIASES: &[(&str, &str)] = &[
|
||||
("-asc", "--api-server-count"),
|
||||
("-pp", "--pipeline-parallel-size"),
|
||||
("-tp", "--tensor-parallel-size"),
|
||||
("-dcp", "--decode-context-parallel-size"),
|
||||
("-pcp", "--prefill-context-parallel-size"),
|
||||
("-dp", "--data-parallel-size"),
|
||||
("-dpn", "--data-parallel-rank"),
|
||||
("-dpr", "--data-parallel-start-rank"),
|
||||
("-dpl", "--data-parallel-size-local"),
|
||||
("-dpa", "--data-parallel-address"),
|
||||
("-dpp", "--data-parallel-rpc-port"),
|
||||
("-dpb", "--data-parallel-backend"),
|
||||
("-dph", "--data-parallel-hybrid-lb"),
|
||||
("-dpe", "--data-parallel-external-lb"),
|
||||
("-ep", "--enable-expert-parallel"),
|
||||
("-cc", "--compilation-config"),
|
||||
("-ac", "--attention-config"),
|
||||
];
|
||||
|
||||
/// Repartition managed-engine argv so Rust-owned flags stay before `--`, while
|
||||
/// everything else is forwarded to Python.
|
||||
pub fn repartition_managed_engine_args<C>(
|
||||
args: &[OsString],
|
||||
subcommand: Option<&str>,
|
||||
) -> Result<Vec<OsString>, clap::Error>
|
||||
where
|
||||
C: CommandFactory,
|
||||
{
|
||||
let command = C::command();
|
||||
let (prefix, real_args, command) = match subcommand {
|
||||
Some(subcommand) => {
|
||||
if !matches_subcommand(args, subcommand) {
|
||||
return Ok(args.to_vec());
|
||||
};
|
||||
|
||||
let subcommand = command
|
||||
.find_subcommand(subcommand)
|
||||
.expect("managed-engine subcommand should exist");
|
||||
|
||||
(args[..2].to_vec(), &args[2..], subcommand)
|
||||
}
|
||||
None => {
|
||||
let Some(program) = args.first() else {
|
||||
return Ok(args.to_vec());
|
||||
};
|
||||
|
||||
(vec![program.clone()], &args[1..], &command)
|
||||
}
|
||||
};
|
||||
|
||||
let mut repartitioned = prefix;
|
||||
repartitioned.extend(repartition_real_managed_engine_args(real_args, command)?);
|
||||
Ok(repartitioned)
|
||||
}
|
||||
|
||||
fn repartition_real_managed_engine_args(
|
||||
args: &[OsString],
|
||||
command: &clap::Command,
|
||||
) -> Result<Vec<OsString>, clap::Error> {
|
||||
let Some(model) = args.first() else {
|
||||
return Ok(args.to_vec());
|
||||
};
|
||||
|
||||
let model = model.to_string_lossy();
|
||||
if is_help_flag(&model) {
|
||||
return Ok(args.to_vec());
|
||||
}
|
||||
if model == "--" || is_option_like(&model) {
|
||||
return Err(build_missing_model_error(command));
|
||||
}
|
||||
|
||||
let (long_flags, short_flags) = collect_option_names(command);
|
||||
let (front_args, explicit_passthrough, had_separator) = split_managed_engine_args(&args[1..]);
|
||||
let normalized_front_args = normalize_python_arg_aliases(front_args);
|
||||
|
||||
let mut frontend_chunks = Vec::new();
|
||||
let mut python_chunks = Vec::new();
|
||||
let mut current_chunk = Vec::new();
|
||||
|
||||
for arg in normalized_front_args {
|
||||
let text = arg.to_string_lossy();
|
||||
if is_option_like(&text) && !current_chunk.is_empty() {
|
||||
push_chunk(
|
||||
&mut frontend_chunks,
|
||||
&mut python_chunks,
|
||||
std::mem::take(&mut current_chunk),
|
||||
&long_flags,
|
||||
&short_flags,
|
||||
);
|
||||
}
|
||||
current_chunk.push(arg);
|
||||
}
|
||||
if !current_chunk.is_empty() {
|
||||
push_chunk(
|
||||
&mut frontend_chunks,
|
||||
&mut python_chunks,
|
||||
current_chunk,
|
||||
&long_flags,
|
||||
&short_flags,
|
||||
);
|
||||
}
|
||||
|
||||
let mut repartitioned = vec![args[0].clone()];
|
||||
repartitioned.extend(frontend_chunks);
|
||||
if had_separator || !python_chunks.is_empty() || !explicit_passthrough.is_empty() {
|
||||
repartitioned.push("--".into());
|
||||
repartitioned.extend(python_chunks);
|
||||
repartitioned.extend(explicit_passthrough.iter().cloned());
|
||||
}
|
||||
|
||||
Ok(repartitioned)
|
||||
}
|
||||
|
||||
fn matches_subcommand(args: &[OsString], subcommand: &str) -> bool {
|
||||
args.get(1)
|
||||
.and_then(|arg| arg.to_str())
|
||||
.is_some_and(|candidate| candidate == subcommand)
|
||||
}
|
||||
|
||||
fn split_managed_engine_args(args: &[OsString]) -> (&[OsString], &[OsString], bool) {
|
||||
if let Some(index) = args.iter().position(|arg| arg == "--") {
|
||||
(&args[..index], &args[index + 1..], true)
|
||||
} else {
|
||||
(args, &[], false)
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_python_arg_aliases(args: &[OsString]) -> Vec<OsString> {
|
||||
args.iter()
|
||||
.map(|arg| {
|
||||
let text = arg.to_string_lossy();
|
||||
normalize_python_multi_char_alias(&text)
|
||||
.map(Into::into)
|
||||
.unwrap_or_else(|| arg.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalize_python_multi_char_alias(arg: &str) -> Option<String> {
|
||||
find_python_multi_char_alias(arg).map(|canonical| match arg.split_once('=') {
|
||||
Some((_, value)) => format!("{canonical}={value}"),
|
||||
None => canonical.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn find_python_multi_char_alias(arg: &str) -> Option<&'static str> {
|
||||
PYTHON_MULTI_CHAR_ALIASES.iter().find_map(|&(alias, canonical)| {
|
||||
(arg == alias || arg.starts_with(&format!("{alias}="))).then_some(canonical)
|
||||
})
|
||||
}
|
||||
|
||||
fn push_chunk(
|
||||
frontend_chunks: &mut Vec<OsString>,
|
||||
python_chunks: &mut Vec<OsString>,
|
||||
chunk: Vec<OsString>,
|
||||
long_flags: &HashSet<String>,
|
||||
short_flags: &HashSet<char>,
|
||||
) {
|
||||
if chunk_head_is_frontend_owned(&chunk, long_flags, short_flags) {
|
||||
frontend_chunks.extend(chunk);
|
||||
} else {
|
||||
python_chunks.extend(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk_head_is_frontend_owned(
|
||||
chunk: &[OsString],
|
||||
long_flags: &HashSet<String>,
|
||||
short_flags: &HashSet<char>,
|
||||
) -> bool {
|
||||
let Some(head) = chunk.first() else {
|
||||
return false;
|
||||
};
|
||||
let head = head.to_string_lossy();
|
||||
|
||||
if let Some(rest) = head.strip_prefix("--") {
|
||||
let name = rest.split_once('=').map_or(rest, |(name, _)| name);
|
||||
return long_flags.contains(name);
|
||||
}
|
||||
|
||||
let Some(rest) = head.strip_prefix('-') else {
|
||||
return false;
|
||||
};
|
||||
let Some(short) = rest.chars().next() else {
|
||||
return false;
|
||||
};
|
||||
short_flags.contains(&short)
|
||||
}
|
||||
|
||||
fn collect_option_names(command: &clap::Command) -> (HashSet<String>, HashSet<char>) {
|
||||
let mut long_flags = HashSet::new();
|
||||
let mut short_flags = HashSet::new();
|
||||
for arg in command.get_arguments() {
|
||||
if let Some(names) = arg.get_long_and_visible_aliases() {
|
||||
long_flags.extend(names.into_iter().map(str::to_owned));
|
||||
}
|
||||
if let Some(short) = arg.get_short() {
|
||||
short_flags.insert(short);
|
||||
}
|
||||
if let Some(short_aliases) = arg.get_visible_short_aliases() {
|
||||
short_flags.extend(short_aliases);
|
||||
}
|
||||
}
|
||||
|
||||
long_flags.insert("help".to_string());
|
||||
short_flags.insert('h');
|
||||
|
||||
(long_flags, short_flags)
|
||||
}
|
||||
|
||||
fn is_option_like(arg: &str) -> bool {
|
||||
if arg == "--" {
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(rest) = arg.strip_prefix("--") {
|
||||
return rest.chars().next().is_some_and(char::is_alphabetic);
|
||||
}
|
||||
|
||||
if let Some(rest) = arg.strip_prefix('-') {
|
||||
return rest.chars().next().is_some_and(char::is_alphabetic);
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
fn is_help_flag(arg: &str) -> bool {
|
||||
arg == "-h" || arg == "--help"
|
||||
}
|
||||
|
||||
fn build_missing_model_error(command: &clap::Command) -> clap::Error {
|
||||
command.clone().error(
|
||||
ErrorKind::MissingRequiredArgument,
|
||||
"the model must appear immediately after the command",
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod cli;
|
||||
mod process;
|
||||
|
||||
pub use process::{ManagedEngineConfig, ManagedEngineHandle, allocate_handshake_port};
|
||||
@@ -0,0 +1,263 @@
|
||||
use std::io;
|
||||
use std::net::TcpListener;
|
||||
use std::process::{Command as StdCommand, ExitStatus, Stdio};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::interval;
|
||||
use tracing::info;
|
||||
|
||||
const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(200);
|
||||
const MIN_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Allocate one ephemeral TCP port for the managed headless-engine handshake on
|
||||
/// the given host.
|
||||
pub fn allocate_handshake_port(host: &str) -> Result<u16> {
|
||||
let listener = TcpListener::bind((host, 0)).context("failed to allocate handshake port")?;
|
||||
let port = listener
|
||||
.local_addr()
|
||||
.context("failed to inspect allocated handshake listener address")?
|
||||
.port();
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
/// Spawn configuration for one managed headless Python vLLM engine.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ManagedEngineConfig {
|
||||
/// Python executable used to launch `vllm.entrypoints.cli.main`.
|
||||
pub python: String,
|
||||
/// Model identifier passed to `vllm ... serve <model>`.
|
||||
pub model: String,
|
||||
/// Host portion of the headless-engine handshake endpoint.
|
||||
pub handshake_host: String,
|
||||
/// Port portion of the headless-engine handshake endpoint.
|
||||
pub handshake_port: u16,
|
||||
/// Number of data parallel replicas across the whole deployment.
|
||||
///
|
||||
/// The per-node replica count is forwarded separately in `python_args` as
|
||||
/// `--data-parallel-size-local`.
|
||||
pub data_parallel_size: usize,
|
||||
/// Extra CLI arguments forwarded verbatim to Python vLLM.
|
||||
pub python_args: Vec<String>,
|
||||
}
|
||||
|
||||
impl ManagedEngineConfig {
|
||||
/// Render the handshake address that the Rust frontend should dial.
|
||||
pub fn handshake_address(&self) -> String {
|
||||
format!("tcp://{}:{}", self.handshake_host, self.handshake_port)
|
||||
}
|
||||
|
||||
/// Build the concrete Python command line for the managed headless engine.
|
||||
pub fn to_command(&self) -> StdCommand {
|
||||
let mut command = StdCommand::new(&self.python);
|
||||
command
|
||||
.arg("-m")
|
||||
.arg("vllm.entrypoints.cli.main")
|
||||
.arg("serve")
|
||||
.arg(&self.model)
|
||||
.arg("--headless")
|
||||
.arg("--data-parallel-address")
|
||||
.arg(&self.handshake_host)
|
||||
.arg("--data-parallel-rpc-port")
|
||||
.arg(self.handshake_port.to_string())
|
||||
.arg("--data-parallel-size")
|
||||
.arg(self.data_parallel_size.to_string())
|
||||
.args(&self.python_args);
|
||||
command
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII-style handle for one managed Python headless engine subprocess.
|
||||
#[derive(Clone)]
|
||||
pub struct ManagedEngineHandle {
|
||||
child: Arc<Mutex<Child>>,
|
||||
shutdown_started: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ManagedEngineHandle {
|
||||
/// Spawn one managed Python headless engine and return a handle for
|
||||
/// monitoring it.
|
||||
pub async fn spawn(config: ManagedEngineConfig) -> Result<Self> {
|
||||
let command = config.to_command();
|
||||
info!(
|
||||
handshake_address = %config.handshake_address(),
|
||||
?command,
|
||||
"starting managed Python headless engine"
|
||||
);
|
||||
|
||||
let mut command = Command::from(command);
|
||||
command.stdin(Stdio::null()).stdout(Stdio::inherit()).stderr(Stdio::inherit());
|
||||
|
||||
process_group::configure(&mut command);
|
||||
|
||||
let child = command.spawn().context("failed to spawn managed engine")?;
|
||||
|
||||
Ok(Self {
|
||||
child: Arc::new(Mutex::new(child)),
|
||||
shutdown_started: Arc::new(AtomicBool::new(false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Poll whether the managed engine has exited yet.
|
||||
pub async fn try_wait(&self) -> Option<ExitStatus> {
|
||||
let mut child = self.child.lock().await;
|
||||
child.try_wait().expect("failed to poll the status of managed engine")
|
||||
}
|
||||
|
||||
/// Wait until the managed engine exits.
|
||||
pub async fn wait_for_exit(&self) -> ExitStatus {
|
||||
let mut interval = interval(CHILD_POLL_INTERVAL);
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Some(status) = self.try_wait().await {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Terminate the managed engine process group and wait for it to stop.
|
||||
pub async fn shutdown(&self, timeout: Duration) -> Result<()> {
|
||||
if self.shutdown_started.swap(true, Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(pid) = self.child.lock().await.id() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
// Enforce a minimum shutdown timeout to give the engine process enough time to
|
||||
// clean up.
|
||||
let shutdown_timeout = std::cmp::max(timeout, MIN_SHUTDOWN_TIMEOUT);
|
||||
|
||||
// First, try to gracefully terminate.
|
||||
info!(
|
||||
pid,
|
||||
?shutdown_timeout,
|
||||
"shutting down managed engine with SIGTERM"
|
||||
);
|
||||
process_group::terminate(pid)?;
|
||||
|
||||
// Wait for the process to exit on its own.
|
||||
if tokio::time::timeout(shutdown_timeout, self.wait_for_exit()).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// If it doesn't exit within the timeout, force kill it.
|
||||
info!(
|
||||
pid,
|
||||
"managed engine did not exit within timeout, sending SIGKILL"
|
||||
);
|
||||
process_group::kill(pid)?;
|
||||
|
||||
let _ = self.wait_for_exit().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Process group helper functions for managing the Python subprocess and its
|
||||
/// potential children in a platform-aware way.
|
||||
mod process_group {
|
||||
use super::*;
|
||||
|
||||
/// Place the Python child into its own process group so `serve` can tear
|
||||
/// down the whole subtree rather than just the immediate shell process.
|
||||
pub fn configure(command: &mut Command) {
|
||||
unsafe {
|
||||
command.pre_exec(|| {
|
||||
if libc::setpgid(0, 0) != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Send SIGTERM to the managed Python process group.
|
||||
pub fn terminate(pid: u32) -> Result<()> {
|
||||
signal(pid, libc::SIGTERM)
|
||||
}
|
||||
|
||||
/// Send SIGKILL to the managed Python process group.
|
||||
pub fn kill(pid: u32) -> Result<()> {
|
||||
signal(pid, libc::SIGKILL)
|
||||
}
|
||||
|
||||
/// Deliver one signal to the managed Python process group.
|
||||
fn signal(pid: u32, signal: i32) -> Result<()> {
|
||||
let rc = unsafe { libc::kill(-(pid as i32), signal) };
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let error = io::Error::last_os_error();
|
||||
if matches!(error.raw_os_error(), Some(code) if code == libc::ESRCH) {
|
||||
return Ok(());
|
||||
}
|
||||
Err(error).context("failed to signal managed engine process group")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use expect_test::expect;
|
||||
|
||||
use super::{ManagedEngineConfig, allocate_handshake_port};
|
||||
|
||||
#[test]
|
||||
fn command_snapshot() {
|
||||
let config = ManagedEngineConfig {
|
||||
python: "python3".to_string(),
|
||||
model: "Qwen/Qwen3-0.6B".to_string(),
|
||||
handshake_host: "127.0.0.1".to_string(),
|
||||
handshake_port: 62100,
|
||||
data_parallel_size: 4,
|
||||
python_args: vec![
|
||||
"--data-parallel-size-local".to_string(),
|
||||
"2".to_string(),
|
||||
"--data-parallel-start-rank".to_string(),
|
||||
"2".to_string(),
|
||||
"--dtype".to_string(),
|
||||
"float16".to_string(),
|
||||
"--max-model-len".to_string(),
|
||||
"512".to_string(),
|
||||
],
|
||||
};
|
||||
let command = config.to_command();
|
||||
let args = command.get_args().collect::<Vec<_>>();
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
"-m",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--headless",
|
||||
"--data-parallel-address",
|
||||
"127.0.0.1",
|
||||
"--data-parallel-rpc-port",
|
||||
"62100",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
"--data-parallel-size-local",
|
||||
"2",
|
||||
"--data-parallel-start-rank",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocate_handshake_port_returns_non_zero_port() {
|
||||
let port = allocate_handshake_port("127.0.0.1").unwrap();
|
||||
assert_ne!(port, 0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user