[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+39
View File
@@ -0,0 +1,39 @@
[package]
name = "vllm-cmd"
version.workspace = true
edition.workspace = true
license.workspace = true
[[bin]]
name = "vllm-rs"
path = "src/main.rs"
[features]
default = []
native-tls-vendored = ["dep:native-tls-vendored"]
[dependencies]
anyhow.workspace = true
clap.workspace = true
educe.workspace = true
itertools.workspace = true
native-tls-vendored = { workspace = true, optional = true }
serde.workspace = true
serde_json.workspace = true
serde_with.workspace = true
thiserror-ext.workspace = true
time.workspace = true
tokio = { workspace = true, features = ["signal"] }
tokio-util.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
vllm-engine-core-client.workspace = true
vllm-managed-engine.workspace = true
vllm-server.workspace = true
[dev-dependencies]
expect-test.workspace = true
[lints]
workspace = true
+44
View File
@@ -0,0 +1,44 @@
# `vllm-rs` CLI Quick Start
Start Qwen3 with one managed `vllm-rs serve` command from the repo root:
```bash
HF_HUB_OFFLINE=1 \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
cargo run --bin vllm-rs -- serve \
Qwen/Qwen3-0.6B \
--python ../vllm/.venv/bin/python \
--max-model-len 512 \
-- \
--dtype float16
```
This launches:
- a managed headless Python `vllm` engine
- the Rust OpenAI-compatible frontend on `127.0.0.1:8000`
All Python engine arguments must be placed after `--`. Arguments before `--` are parsed by the Rust
frontend itself.
You can then send OpenAI-style requests to the Rust frontend:
```bash
curl http://127.0.0.1:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "Qwen/Qwen3-0.6B",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"stream": true
}'
```
If you already started headless `vllm` yourself, use `frontend` instead:
```bash
cargo run --bin vllm-rs -- frontend \
--handshake-address tcp://127.0.0.1:62100 \
Qwen/Qwen3-0.6B
```
+434
View File
@@ -0,0 +1,434 @@
//! CLI argument definitions for the `vllm-rs` binary.
//!
//! Python vLLM references:
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
mod unsupported;
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::path::PathBuf;
use std::time::Duration;
use clap::{Args, Parser, Subcommand};
use educe::Educe;
use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
use thiserror_ext::AsReport as _;
use uuid::Uuid;
use vllm_engine_core_client::TransportMode;
use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
use vllm_server::{
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
RendererSelection,
};
use crate::cli::unsupported::UnsupportedArgs;
/// Top-level parser for the `vllm-rs` binary.
#[derive(Debug, Parser)]
#[command(
name = "vllm-rs",
about = "Rust frontend and managed-engine CLI for vLLM."
)]
pub struct Cli {
#[command(subcommand)]
pub command: Command,
}
impl Cli {
pub fn parse() -> Self {
Self::try_parse_from(std::env::args_os()).unwrap_or_else(|error| error.exit())
}
pub fn try_parse_from<I, T>(itr: I) -> Result<Self, clap::Error>
where
I: IntoIterator<Item = T>,
T: Into<OsString>,
{
let args: Vec<OsString> = itr.into_iter().map(Into::into).collect();
let repartitioned_args = repartition_managed_engine_args::<Self>(&args, Some("serve"))?;
<Self as Parser>::try_parse_from(&repartitioned_args).inspect(|cli| {
if let Command::Serve(serve) = &cli.command
&& serve.debug_cli
{
println!(
"Original CLI args: {}\n",
args.join(OsStr::new(" ")).display()
);
println!(
"Repartitioned CLI args: {}\n",
repartitioned_args.join(OsStr::new(" ")).display()
);
println!(
"Passthrough Python args: {}",
serve.managed_engine.python_args.join(" ")
);
std::process::exit(0);
}
})
}
}
/// Supported top-level CLI commands.
#[derive(Debug, Subcommand, PartialEq, Eq)]
pub enum Command {
/// Run the Rust OpenAI frontend as a Python-supervised worker.
Frontend(FrontendArgs),
/// Launch a managed Python headless engine, then run the Rust OpenAI
/// frontend.
Serve(ServeArgs),
}
/// Runtime arguments shared by the external-engine and managed-engine paths.
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
#[educe(Debug)]
pub struct SharedRuntimeArgs {
#[serde(rename = "model_tag")]
/// Model identifier or local model directory used for backend loading and
/// public model ID.
pub model: String,
/// Maximum time to wait for the expected engines to register on the
/// frontend transport.
#[arg(
long = "engine-ready-timeout-secs",
env = "VLLM_ENGINE_READY_TIMEOUT_S",
default_value_t = default_engine_ready_timeout_secs()
)]
#[serde(default = "default_engine_ready_timeout_secs")]
pub engine_ready_timeout_secs: u64,
/// Select the tool call parser depending on the model that you're using.
/// Use `auto` to infer from the model or `none` to disable parsing.
#[arg(long, default_value_t)]
#[serde(default)]
pub tool_call_parser: ParserSelection,
/// Select the reasoning parser depending on the model that you're using.
/// Use `auto` to infer from the model or `none` to disable parsing.
#[arg(long, default_value_t)]
#[serde(default)]
pub reasoning_parser: ParserSelection,
/// Select the chat renderer implementation.
#[arg(long = "tokenizer-mode", default_value_t)]
#[serde(default, rename = "tokenizer_mode")]
pub renderer: RendererSelection,
/// Override the maximum model context length. When set, the frontend uses
/// this value instead of the model's `max_position_embeddings` from
/// `config.json`.
#[arg(long)]
pub max_model_len: Option<u32>,
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
/// started.
#[arg(long)]
#[serde(default)]
pub grpc_port: Option<u16>,
/// Maximum time to wait for active requests to drain during shutdown.
#[arg(long, default_value_t = 0)]
#[serde(default)]
pub shutdown_timeout: u64,
/// The file path to the chat template, or the template in single-line form
/// for the specified model.
#[arg(long)]
#[serde(default)]
pub chat_template: Option<String>,
/// Default keyword arguments to pass to the chat template renderer.
///
/// These will be merged with request-level chat_template_kwargs, with
/// request values taking precedence. Useful for setting default
/// behavior for reasoning models.
///
/// Example: `{"enable_thinking": false}` to disable thinking mode by
/// default for Qwen3/DeepSeek models.
#[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
#[serde(default)]
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// The format to render message content within a chat template.
///
/// * "auto" detects the format from the template
/// * "string" renders content as a string. Example: `"Hello World"`
/// * "openai" renders content as a list of dictionaries, similar to OpenAI schema. Example:
/// `[{"type": "text", "text": "Hello world!"}]`
#[arg(long, default_value_t)]
#[serde(default)]
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Log a summary line for each completed request, including prompt/output
/// token counts and finish reason.
#[arg(long)]
#[serde(default)]
pub enable_log_requests: bool,
/// Disable periodic logging of engine statistics (throughput, queue depth,
/// cache usage).
#[arg(long)]
#[serde(default)]
pub disable_log_stats: bool,
/// The model name(s) used in the API. If multiple names are provided, the
/// server will respond to any of the provided names. The model name in the
/// model field of a response will be the first name in this list. If not
/// specified, the model name will be the same as the `--model` argument.
/// Noted that this name(s) will also be used in `model_name` tag
/// content of prometheus metrics, if multiple names provided, metrics
/// tag will take the first one.
#[arg(long, num_args = 0..)]
#[serde(default)]
pub served_model_name: Vec<String>,
/// Unsupported Python vLLM frontend arguments recognized but not yet
/// implemented in Rust.
#[educe(Debug(ignore))]
#[command(flatten)]
#[serde(default, flatten)]
pub unsupported: UnsupportedArgs,
}
impl SharedRuntimeArgs {
/// Maximum time to wait for the expected engines to register on the
/// frontend transport.
pub fn ready_timeout(&self) -> Duration {
Duration::from_secs(self.engine_ready_timeout_secs)
}
/// Maximum time to wait for active requests to drain during shutdown.
pub fn shutdown_timeout(&self) -> Duration {
Duration::from_secs(self.shutdown_timeout)
}
/// Build the OpenAI-server config for the Python-bootstrap worker contract.
///
/// The resulting config binds the Python-supplied transport addresses and
/// inherits an already open HTTP listener from the supervisor process.
fn into_bootstrapped_config(
self,
listen_fd: i32,
input_address: String,
output_address: String,
coordinator_address: Option<String>,
engine_count: usize,
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
Config {
transport_mode: TransportMode::Bootstrapped {
input_address,
output_address,
engine_count,
ready_timeout,
},
coordinator_mode: match coordinator_address {
Some(address) => CoordinatorMode::External { address },
None => CoordinatorMode::None,
},
model: self.model,
served_model_name: self.served_model_name,
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
renderer: self.renderer,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
enable_log_requests: self.enable_log_requests,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
}
}
/// Build the OpenAI-server config for the managed `serve` path that still
/// owns the startup handshake and binds its own HTTP listener.
fn into_managed_config(
self,
listener_mode: HttpListenerMode,
handshake_address: String,
advertised_host: String,
engine_count: usize,
local_input_address: Option<String>,
local_output_address: Option<String>,
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
Config {
transport_mode: TransportMode::HandshakeOwner {
handshake_address,
advertised_host,
engine_count,
ready_timeout,
local_input_address,
local_output_address,
},
coordinator_mode: CoordinatorMode::MaybeInProc,
model: self.model,
served_model_name: self.served_model_name,
listener_mode,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
renderer: self.renderer,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
enable_log_requests: self.enable_log_requests,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
}
}
}
fn default_engine_ready_timeout_secs() -> u64 {
600
}
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
}
fn parse_runtime_args_json(value: &str) -> Result<SharedRuntimeArgs, String> {
let args: SharedRuntimeArgs = serde_json::from_str(value)
.map_err(|e| format!("invalid JSON arguments: {}", e.as_report()))?;
args.unsupported.check()?;
Ok(args)
}
/// Arguments for running the Rust frontend as a Python-bootstrapped worker.
#[derive(Educe, Clone, Args, PartialEq, Eq)]
#[educe(Debug)]
pub struct FrontendArgs {
/// Inherited listening socket file descriptor passed by the Python
/// supervisor.
#[arg(long)]
pub listen_fd: i32,
/// Frontend input ROUTER socket address that the Python engines will
/// connect to.
#[arg(long)]
pub input_address: String,
/// Frontend output PULL socket address that the Python engines will push
/// responses to.
#[arg(long)]
pub output_address: String,
/// Optional Python-owned frontend-side DP coordinator socket address for
/// external coordinator mode in the bootstrapped frontend path, i.e.,
/// `stats_update_address`.
#[arg(long)]
pub coordinator_address: Option<String>,
/// Total number of data-parallel engines expected for this frontend.
#[arg(long, default_value_t = 1)]
pub engine_count: usize,
/// Shared frontend arguments as one JSON object.
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
pub runtime: SharedRuntimeArgs,
}
impl FrontendArgs {
/// Convert the CLI arguments into the OpenAI server's runtime config.
pub fn into_config(self) -> Config {
self.runtime.into_bootstrapped_config(
self.listen_fd,
self.input_address,
self.output_address,
self.coordinator_address,
self.engine_count,
)
}
}
/// Arguments for the managed-engine mode that spawns Python on behalf of the
/// user.
#[derive(Educe, Clone, Args, PartialEq, Eq)]
#[educe(Debug)]
#[command(override_usage = "vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]")]
pub struct ServeArgs {
/// Only launch the managed Python headless engine and do not start the Rust
/// frontend.
#[arg(long)]
pub headless: bool,
/// HTTP bind host for the OpenAI-compatible server.
#[arg(long, default_value = "127.0.0.1")]
pub host: String,
/// HTTP bind port for the OpenAI-compatible server.
#[arg(long, default_value_t = 8000)]
pub port: u16,
/// Unix domain socket path. If set, host and port arguments are ignored.
#[arg(long)]
pub uds: Option<String>,
/// Flag to print debug information about CLI argument parsing and exit.
#[educe(Debug(ignore))]
#[arg(long, hide = true, env = "VLLM_RS_DEBUG_CLI")]
pub debug_cli: bool,
/// Shared frontend arguments.
#[command(flatten)]
pub runtime: SharedRuntimeArgs,
/// Managed Python headless-engine arguments.
#[command(flatten)]
pub managed_engine: ManagedEngineArgs,
}
impl ServeArgs {
/// Build the OpenAI-server runtime config used after the managed Python
/// engine starts.
pub fn to_frontend_config(&self, handshake_address: String) -> Config {
// Prefer IPC sockets for local engine input/output.
let (local_input_address, local_output_address) =
self.managed_engine.frontend_local_only().then(frontend_ipc_addresses).unzip();
let listener_mode = match &self.uds {
Some(path) => HttpListenerMode::BindUnix { path: path.clone() },
None => HttpListenerMode::BindTcp {
host: self.host.clone(),
port: self.port,
},
};
self.runtime.clone().into_managed_config(
listener_mode,
handshake_address,
self.managed_engine.handshake_host.clone(),
self.managed_engine.data_parallel_size,
local_input_address,
local_output_address,
)
}
/// Build the managed Python-engine spawn configuration with the given
/// handshake port.
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
self.managed_engine.clone().into_config(
self.runtime.model.clone(),
self.runtime.max_model_len,
handshake_port,
)
}
}
/// Allocate fresh IPC endpoints for one managed frontend instance.
fn frontend_ipc_addresses() -> (String, String) {
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
.map(PathBuf::from)
.unwrap_or_else(std::env::temp_dir);
let input_name = format!("vllm-rs-i-{}", Uuid::new_v4().simple());
let output_name = format!("vllm-rs-o-{}", Uuid::new_v4().simple());
let input = preferred_base_path.join(input_name);
let output = preferred_base_path.join(output_name);
(
format!("ipc://{}", input.to_string_lossy()),
format!("ipc://{}", output.to_string_lossy()),
)
}
#[cfg(test)]
mod tests;
+905
View File
@@ -0,0 +1,905 @@
use expect_test::expect;
use vllm_engine_core_client::TransportMode;
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
use super::{Cli, Command};
#[test]
fn serve_args_forward_python_flags_with_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--python",
"../vllm/.venv/bin/python",
"--max-model-len",
"512",
"--",
"--dtype",
"float16",
])
.unwrap();
expect![[r#"
Cli {
command: Serve(
ServeArgs {
headless: false,
host: "127.0.0.1",
port: 8000,
uds: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
max_model_len: Some(
512,
),
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
served_model_name: [],
},
managed_engine: ManagedEngineArgs {
python: "../vllm/.venv/bin/python",
handshake_host: "127.0.0.1",
handshake_port: None,
data_parallel_size: 1,
data_parallel_size_local: None,
python_args: [
"--dtype",
"float16",
],
},
},
),
}
"#]]
.assert_debug_eq(&cli);
}
#[test]
fn serve_args_auto_forward_python_flags_without_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--python",
"python3",
"--quantization",
"awq",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["--quantization", "awq"]
);
}
#[test]
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["--tensor-parallel-size", "2"]
);
}
#[test]
fn serve_args_accept_explicit_deepseek_v32_renderer() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--tokenizer-mode",
"deepseek_v32",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
}
#[test]
fn serve_args_reject_unknown_renderer_value() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--tokenizer-mode",
"definitely_missing",
])
.unwrap_err();
expect![[r#"
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4)
For more information, try '--help'.
"#]]
.assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_unsupported_flag_arg() {
let error = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--allow-credentials"])
.unwrap_err();
expect![[r#"
error: invalid value 'true' for '--allow-credentials [<ALLOW_CREDENTIALS>]': argument is not implemented in Rust frontend yet
Remove this unsupported argument to continue.
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
For more information, try '--help'.
"#]]
.assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_unsupported_no_flag_alias() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--no-enable-log-deltas",
])
.unwrap_err();
expect![[r#"
error: invalid value 'true' for '--enable-log-deltas [<ENABLE_LOG_DELTAS>]': argument is not implemented in Rust frontend yet
Remove this unsupported argument to continue.
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
For more information, try '--help'.
"#]]
.assert_eq(&error.to_string());
}
#[test]
fn frontend_args_accept_json() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--coordinator-address",
"tcp://127.0.0.1:7000",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_count":2}"#,
])
.unwrap();
expect![[r#"
Cli {
command: Frontend(
FrontendArgs {
listen_fd: 3,
input_address: "ipc:///tmp/input.sock",
output_address: "ipc:///tmp/output.sock",
coordinator_address: Some(
"tcp://127.0.0.1:7000",
),
engine_count: 1,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
max_model_len: None,
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
served_model_name: [],
},
},
),
}
"#]]
.assert_debug_eq(&cli);
}
#[test]
fn frontend_args_json_applies_defaults() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
assert_eq!(args.runtime.max_model_len, None);
assert_eq!(args.runtime.shutdown_timeout, 0);
}
#[test]
fn frontend_args_json_accepts_supported_non_default_fields() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","max_model_len":8192,"shutdown_timeout":3}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
assert_eq!(
args.runtime.tool_call_parser,
ParserSelection::Explicit("hermes".to_string())
);
assert_eq!(
args.runtime.reasoning_parser,
ParserSelection::Explicit("qwen3_thinking".to_string())
);
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
assert_eq!(args.runtime.max_model_len, Some(8192));
assert_eq!(args.runtime.shutdown_timeout, 3);
}
#[test]
fn serve_args_accept_none_reasoning_parser() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--reasoning-parser",
"none",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
}
#[test]
fn frontend_args_json_ignores_unknown_fields() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","uds":"/tmp/vllm.sock","nested_unknown":{"x":1}}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
}
#[test]
fn frontend_args_json_accepts_noop_fields() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
}
#[test]
fn frontend_args_json_rejects_unsupported_fields() {
let error = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}"#,
])
.unwrap_err();
expect![[r#"
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}' for '--args-json <JSON>':
The following arguments are not implemented in Rust frontend yet:
- allow_credentials
Remove these arguments to continue.
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn frontend_args_json_aggregates_multiple_unsupported_fields() {
let error = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}"#,
])
.unwrap_err();
expect![[r#"
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}' for '--args-json <JSON>':
The following arguments are not implemented in Rust frontend yet:
- allow_credentials
- api_key
Remove these arguments to continue.
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn frontend_args_json_rejects_malformed_json() {
let error = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B""#,
])
.unwrap_err();
expect![[r#"
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B"' for '--args-json <JSON>': invalid JSON arguments: EOF while parsing an object at line 1 column 30
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_flags_before_model() {
let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
.unwrap_err();
expect![[r#"
error: the model must appear immediately after the command
Usage: vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]
For more information, try '--help'.
"#]]
.assert_eq(&error.to_string());
}
#[test]
fn serve_args_accept_headless_mode() {
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--headless"]).unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert!(args.headless);
}
#[test]
fn serve_args_keep_python_passthrough_flags_after_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--python",
"python3",
"--",
"--tensor-parallel-size",
"2",
"--dtype",
"float16",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["--tensor-parallel-size", "2", "--dtype", "float16"]
);
}
#[test]
fn serve_args_keep_python_multi_char_alias_after_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--python",
"python3",
"--",
"-tp",
"2",
"--dtype",
"float16",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["-tp", "2", "--dtype", "float16"]
);
}
#[test]
fn serve_args_keep_frontend_arg_after_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--",
"--uds",
"/tmp/vllm.sock",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["--uds", "/tmp/vllm.sock"]
);
}
#[test]
fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--",
"-dpr",
"1",
"-dpl",
"2",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["-dpr", "1", "-dpl", "2"]
);
}
#[test]
fn serve_args_auto_forward_unknown_flags_without_separator() {
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--foo", "bar"]).unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]);
}
#[test]
fn serve_args_auto_forward_negative_value_without_separator() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--num-gpu-blocks-override",
"-1",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(
args.managed_engine.python_args,
vec!["--num-gpu-blocks-override", "-1"]
);
}
#[test]
fn serve_args_accept_handshake_aliases() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--python",
"python3",
"--handshake-host",
"10.99.48.128",
"--handshake-port",
"13345",
"--data-parallel-size",
"4",
])
.unwrap();
expect![[r#"
Cli {
command: Serve(
ServeArgs {
headless: false,
host: "127.0.0.1",
port: 8000,
uds: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
max_model_len: None,
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
served_model_name: [],
},
managed_engine: ManagedEngineArgs {
python: "python3",
handshake_host: "10.99.48.128",
handshake_port: Some(
13345,
),
data_parallel_size: 4,
data_parallel_size_local: None,
python_args: [],
},
},
),
}
"#]]
.assert_debug_eq(&cli);
}
#[test]
fn serve_args_accept_data_parallel_primary_flags() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--data-parallel-address",
"10.99.48.128",
"--data-parallel-rpc-port",
"13345",
"--data-parallel-size",
"4",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert!(!args.headless);
assert_eq!(args.managed_engine.handshake_host, "10.99.48.128");
assert_eq!(args.managed_engine.handshake_port, Some(13345));
assert_eq!(args.managed_engine.data_parallel_size, 4);
}
#[test]
fn serve_frontend_config_uses_dp_address_as_advertised_host() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--handshake-host",
"10.99.48.128",
"--data-parallel-size",
"4",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
let TransportMode::HandshakeOwner {
handshake_address,
advertised_host,
engine_count,
ready_timeout,
local_input_address,
local_output_address,
} = &config.transport_mode
else {
panic!("expected handshake-owned transport");
};
assert_eq!(handshake_address, "tcp://10.99.48.128:29550");
assert_eq!(advertised_host, "10.99.48.128");
assert_eq!(*engine_count, 4);
assert!(
local_input_address
.as_deref()
.is_some_and(|address| address.starts_with("ipc://"))
);
assert!(
local_output_address
.as_deref()
.is_some_and(|address| address.starts_with("ipc://"))
);
assert_ne!(local_input_address, local_output_address);
expect![[r#"
Config {
transport_mode: HandshakeOwner {
handshake_address: "tcp://10.99.48.128:29550",
advertised_host: "10.99.48.128",
engine_count: 4,
ready_timeout: 600s,
local_input_address: Some(
"<ipc input>",
),
local_output_address: Some(
"<ipc output>",
),
},
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
listener_mode: BindTcp {
host: "127.0.0.1",
port: 8000,
},
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
}
"#]]
.assert_debug_eq(&Config {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: handshake_address.clone(),
advertised_host: advertised_host.clone(),
engine_count: *engine_count,
ready_timeout: *ready_timeout,
local_input_address: Some("<ipc input>".to_string()),
local_output_address: Some("<ipc output>".to_string()),
},
..config.clone()
});
}
#[test]
fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--data-parallel-address",
"10.99.48.128",
"--data-parallel-size",
"4",
"--data-parallel-size-local",
"2",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
expect![[r#"
Config {
transport_mode: HandshakeOwner {
handshake_address: "tcp://10.99.48.128:29550",
advertised_host: "10.99.48.128",
engine_count: 4,
ready_timeout: 600s,
local_input_address: None,
local_output_address: None,
},
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
listener_mode: BindTcp {
host: "127.0.0.1",
port: 8000,
},
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
}
"#]]
.assert_debug_eq(&config);
}
#[test]
fn frontend_args_reject_legacy_handshake_flags() {
let error = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
"--handshake-address",
"tcp://127.0.0.1:62100",
])
.unwrap_err();
assert!(error.to_string().contains("--handshake-address"));
}
#[test]
fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--coordinator-address",
"tcp://127.0.0.1:7000",
"--engine-count",
"2",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
let config = args.into_config();
expect![[r#"
Config {
transport_mode: Bootstrapped {
input_address: "ipc:///tmp/input.sock",
output_address: "ipc:///tmp/output.sock",
engine_count: 2,
ready_timeout: 600s,
},
coordinator_mode: External {
address: "tcp://127.0.0.1:7000",
},
model: "Qwen/Qwen3-0.6B",
served_model_name: [],
listener_mode: InheritedFd {
fd: 3,
},
tool_call_parser: Auto,
reasoning_parser: Auto,
renderer: Auto,
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
}
"#]]
.assert_debug_eq(&config);
}
#[test]
fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--uds",
"/tmp/vllm.sock",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
assert_eq!(
config.listener_mode,
HttpListenerMode::BindUnix {
path: "/tmp/vllm.sock".to_string(),
}
);
}
+660
View File
@@ -0,0 +1,660 @@
#![allow(clippy::doc_lazy_continuation)]
use std::fmt::Display;
use std::str::FromStr;
use clap::Args;
use clap::builder::{TypedValueParser, ValueParserFactory};
use itertools::Itertools;
use serde::{Deserialize, Deserializer, Serialize};
/// Marker type for frontend-owned `serve` arguments that `vllm-rs` recognizes
/// but does not support yet.
///
/// When passed as JSON args, it can be deserialized from any value, and
/// serializes back to the original value.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct Unsupported(pub serde_json::Value);
impl FromStr for Unsupported {
type Err = String;
fn from_str(_s: &str) -> Result<Self, Self::Err> {
Err("argument is not implemented in Rust frontend yet
Remove this unsupported argument to continue.
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument."
.to_string())
}
}
/// Marker type for no-op arguments that are accepted by the Rust frontend but
/// have no effect.
///
/// When passed as JSON args, it can be deserialized from any value, but always
/// serializes back to `null`.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct Noop;
impl<'de> Deserialize<'de> for Noop {
fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(Noop)
}
}
impl ValueParserFactory for Noop {
type Parser = NoopValueParser;
fn value_parser() -> Self::Parser {
NoopValueParser
}
}
#[derive(Copy, Clone, Debug)]
pub struct NoopValueParser;
#[track_caller]
fn noop_warn(arg: impl Display) {
tracing::warn!("argument '{arg}' currently has no effect in Rust frontend, ignoring");
}
impl TypedValueParser for NoopValueParser {
type Value = Noop;
fn parse_ref(
&self,
_cmd: &clap::Command,
arg: Option<&clap::Arg>,
_value: &std::ffi::OsStr,
) -> Result<Self::Value, clap::Error> {
if let Some(arg) = arg {
noop_warn(arg);
}
Ok(Noop)
}
}
/// Frontend-owned Python `serve` arguments that `vllm-rs` recognizes but does
/// not support yet.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
#[command(next_help_heading = "Options not implemented in Rust frontend yet")]
pub struct UnsupportedArgs {
#[command(flatten)]
#[serde(default, flatten)]
top_level: TopLevelUnsupportedArgs,
#[command(flatten)]
#[serde(default, flatten)]
engine: EngineUnsupportedArgs,
#[command(flatten)]
#[serde(default, flatten)]
server: ServerUnsupportedArgs,
}
impl UnsupportedArgs {
/// Check whether any unsupported arguments are set, and if so, return an
/// error listing them. Also warn about any no-op arguments that are set
/// but will be ignored.
pub(crate) fn check(&self) -> Result<(), String> {
let value = serde_json::to_value(self).unwrap();
let map = value.as_object().unwrap();
let mut unsupported = Vec::new();
for (key, value) in map {
if value.is_null() {
noop_warn(key);
} else {
unsupported.push(key.as_str());
}
}
if !unsupported.is_empty() {
unsupported.sort_unstable();
let bullets = unsupported.into_iter().map(|key| format!("- {key}")).join("\n");
return Err(format!(
"
The following arguments are not implemented in Rust frontend yet:
{bullets}
Remove these arguments to continue."
));
}
Ok(())
}
}
/// Frontend-owned Python `vllm serve` top-level arguments that `vllm-rs`
/// recognizes but does not support yet.
///
/// Source of truth in Python vLLM:
/// - `vllm.entrypoints.openai.cli_args.make_arg_parser(...)`
/// - `vllm.entrypoints.cli.serve.ServeSubcommand.subparser_init(...)`
///
/// These are not part of `EngineArgs`, `AsyncEngineArgs`, `BaseFrontendArgs`,
/// or `FrontendArgs`. They live on the `serve` command itself and control
/// managed-engine / multi-process orchestration rather than the shared frontend
/// runtime config.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
pub struct TopLevelUnsupportedArgs {
/// How many API server processes to run. Defaults to data_parallel_size if
/// not specified.
#[arg(long, hide = true)]
pub api_server_count: Option<Noop>,
/// Read CLI options from a config file. Must be a YAML with the following
/// options: https://docs.vllm.ai/en/latest/configuration/serve_args.html
#[arg(long)]
pub config: Option<Unsupported>,
/// Launch a gRPC server instead of the HTTP OpenAI-compatible server.
/// Requires: pip install vllm[grpc].
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
pub grpc: Option<Unsupported>,
}
/// Frontend-owned Python engine arguments that `vllm-rs` recognizes but does
/// not support yet.
///
/// Source of truth in Python vLLM:
/// - `vllm.engine.arg_utils.EngineArgs.add_cli_args(...)`
/// - `vllm.engine.arg_utils.AsyncEngineArgs.add_cli_args(...)`
///
/// These arguments are declared through the Python engine-args surface, but
/// they are still frontend-owned: the API server / AsyncLLM layer reads them
/// for tokenizer setup, request validation, routing, logging, and other
/// frontend behavior, so Rust must recognize them rather than treating them as
/// pure engine passthrough.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
pub struct EngineUnsupportedArgs {
/// Name or path of the Hugging Face tokenizer to use. If unspecified, model
/// name or path will be used.
#[arg(long)]
pub tokenizer: Option<Unsupported>,
/// Name or path of the Hugging Face config to use. If unspecified, model
/// name or path will be used.
#[arg(long)]
pub hf_config_path: Option<Unsupported>,
/// Allowing API requests to read local images or videos from directories
/// specified by the server file system. This is a security risk. Should
/// only be enabled in trusted environments.
#[arg(long)]
pub allowed_local_media_path: Option<Unsupported>,
/// If set, only media URLs that belong to this domain can be used for
/// multi-modal inputs.
#[arg(long)]
pub allowed_media_domains: Option<Unsupported>,
/// The specific revision to use for the tokenizer on the Hugging Face Hub.
/// It can be a branch name, a tag name, or a commit id. If unspecified,
/// will use the default version.
#[arg(long)]
pub tokenizer_revision: Option<Unsupported>,
/// Maximum number of log probabilities to return when `logprobs` is
/// specified in `SamplingParams`. The default value comes the default for
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
/// (output_length * vocab_size) logprobs are allowed to be returned and
/// it may cause OOM.
#[arg(long)]
pub max_logprobs: Option<Unsupported>,
/// Skip initialization of tokenizer and detokenizer. Expects valid
/// `prompt_token_ids` and `None` for prompt from the input. The generated
/// output will contain token ids.
#[arg(
long,
visible_alias = "no-skip-tokenizer-init",
default_missing_value = "true",
num_args = 0..=1
)]
pub skip_tokenizer_init: Option<Unsupported>,
/// If `True`, enables passing text embeddings as inputs via the
/// `prompt_embeds` key.
///
/// WARNING: The vLLM engine may crash if incorrect shape of embeddings is
/// passed. Only enable this flag for trusted users!
#[arg(
long,
visible_alias = "no-enable-prompt-embeds",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_prompt_embeds: Option<Unsupported>,
/// The token to use as HTTP bearer authorization for remote files. If
/// `True`, will use the token generated when running `hf auth login`
/// (stored in `~/.cache/huggingface/token`).
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
pub hf_token: Option<Unsupported>,
/// If a dictionary, contains arguments to be forwarded to the Hugging Face
/// config. If a callable, it is called to update the HuggingFace config.
#[arg(long)]
pub hf_overrides: Option<Unsupported>,
/// The folder path to the generation config. Defaults to `"auto"`, the
/// generation config will be loaded from model path. If set to `"vllm"`, no
/// generation config is loaded, vLLM defaults will be used. If set to a
/// folder path, the generation config will be loaded from the specified
/// folder path. If `max_new_tokens` is specified in generation config,
/// then it sets a server-wide limit on the number of output tokens for
/// all requests.
#[arg(long)]
pub generation_config: Option<Unsupported>,
/// IOProcessor plugin name to load at model startup
#[arg(long)]
pub io_processor_plugin: Option<Unsupported>,
/// Path to a dynamically reasoning parser plugin that can be dynamically
/// loaded and registered.
#[arg(long)]
pub reasoning_parser_plugin: Option<Unsupported>,
/// Rank of the data parallel group.
#[arg(long, env = "VLLM_DP_RANK")]
pub data_parallel_rank: Option<Unsupported>,
/// Whether to use "hybrid" DP LB mode. Applies only to online serving
/// and when data_parallel_size > 0. Enables running an AsyncLLM
/// and API server on a "per-node" basis where vLLM load balances
/// between local data parallel ranks, but an external LB balances
/// between vLLM nodes/replicas. Set explicitly in conjunction with
/// --data-parallel-start-rank.
#[arg(
long,
visible_alias = "no-data-parallel-hybrid-lb",
default_missing_value = "true",
num_args = 0..=1
)]
pub data_parallel_hybrid_lb: Option<Unsupported>,
/// Whether to use "external" DP LB mode. Applies only to online serving
/// and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"
/// wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank
/// is provided explicitly to vllm serve.
#[arg(
long,
visible_alias = "no-data-parallel-external-lb",
default_missing_value = "true",
num_args = 0..=1
)]
pub data_parallel_external_lb: Option<Unsupported>,
/// This feature is work in progress and no prefill optimization takes place
/// with this flag enabled currently.
#[arg(
long,
visible_alias = "no-kv-sharing-fast-prefill",
default_missing_value = "true",
num_args = 0..=1
)]
pub kv_sharing_fast_prefill: Option<Unsupported>,
/// The maximum number of input items and options allowed per
/// prompt for each modality.
#[arg(long)]
pub limit_mm_per_prompt: Option<Unsupported>,
/// Additional args passed to process media inputs, keyed by modalities.
#[arg(long)]
pub media_io_kwargs: Option<Unsupported>,
/// Arguments to be forwarded to the model's processor for multi-modal data,
/// e.g., image processor.
#[arg(long)]
pub mm_processor_kwargs: Option<Unsupported>,
/// The size (in GiB) of the multi-modal processor cache.
#[arg(long)]
pub mm_processor_cache_gb: Option<Unsupported>,
/// Type of cache to use for the multi-modal preprocessor/mapper.
#[arg(long)]
pub mm_processor_cache_type: Option<Unsupported>,
/// If True, enable handling of LoRA adapters.
#[arg(
long,
visible_alias = "no-enable-lora",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_lora: Option<Unsupported>,
/// Dictionary mapping specific modalities to LoRA model paths.
#[arg(long)]
pub default_mm_loras: Option<Unsupported>,
/// Target URL to which OpenTelemetry traces will be sent.
#[arg(long)]
pub otlp_traces_endpoint: Option<Unsupported>,
/// It makes sense to set this only if `--otlp-traces-endpoint` is set.
#[arg(long)]
pub collect_detailed_traces: Option<Unsupported>,
/// The interval (or buffer size) for streaming in terms of token length.
#[arg(long)]
pub stream_interval: Option<Unsupported>,
/// Structured outputs configuration.
#[arg(long)]
pub structured_outputs_config: Option<Noop>,
/// Log aggregate rather than per-engine statistics when using data
/// parallelism.
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
pub aggregate_engine_logging: Option<Unsupported>,
}
/// Frontend-owned Python OpenAI server arguments that `vllm-rs` recognizes but
/// does not support yet.
///
/// Source of truth in Python vLLM:
/// - `vllm.entrypoints.openai.cli_args.BaseFrontendArgs`
/// - `vllm.entrypoints.openai.cli_args.FrontendArgs`
///
/// These are not engine args. They belong to the Python OpenAI-compatible
/// frontend / API-server layer itself, for example chat-template configuration,
/// tool/frontend behavior, TLS / CORS / HTTP server settings, and other
/// northbound server knobs.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
pub struct ServerUnsupportedArgs {
/// LoRA modules configurations in either 'name=path' format or JSON format
/// or JSON list format. Example (old format): `'name=path'` Example (new
/// format): `{"name": "name", "path": "lora_path",
/// "base_model_name": "id"}`
#[arg(long)]
pub lora_modules: Option<Unsupported>,
/// Whether to trust the chat template provided in the request. If False,
/// the server will always use the chat template specified by
/// `--chat-template` or the ones from tokenizer.
#[arg(
long,
visible_alias = "no-trust-request-chat-template",
default_missing_value = "true",
num_args = 0..=1
)]
pub trust_request_chat_template: Option<Unsupported>,
/// The role name to return if `request.add_generation_prompt=true`.
#[arg(long)]
pub response_role: Option<Unsupported>,
/// When `--max-logprobs` is specified, represents single tokens as
/// strings of the form 'token_id:{token_id}' so that tokens that are not
/// JSON-encodable can be identified.
#[arg(
long,
visible_alias = "no-return-tokens-as-token-ids",
default_missing_value = "true",
num_args = 0..=1
)]
pub return_tokens_as_token_ids: Option<Unsupported>,
/// Enable auto tool choice for supported models. Use `--tool-call-parser`
/// to specify which parser to use.
#[arg(
long,
visible_alias = "no-enable-auto-tool-choice",
default_missing_value = "true",
num_args = 0..=1,
hide = true
)]
pub enable_auto_tool_choice: Option<Noop>,
/// If specified, exclude tool definitions in prompts when
/// tool_choice='none'.
#[arg(
long,
visible_alias = "no-exclude-tools-when-tool-choice-none",
default_missing_value = "true",
num_args = 0..=1
)]
pub exclude_tools_when_tool_choice_none: Option<Unsupported>,
/// Special the tool parser plugin write to parse the model-generated tool
/// into OpenAI API format, the name register in this plugin can be used in
/// `--tool-call-parser`.
#[arg(long)]
pub tool_parser_plugin: Option<Unsupported>,
/// Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
/// Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
/// built-in demo tools (browser and Python code interpreter). WARNING:
/// The `demo` Python tool executes model-generated code in Docker without
/// network isolation by default. See the security guide for more
/// information.
#[arg(long)]
pub tool_server: Option<Unsupported>,
/// Path to logging config JSON file for both vllm and uvicorn
#[arg(long, /* env = "VLLM_LOGGING_CONFIG_PATH" */)]
pub log_config_file: Option<Unsupported>,
/// Max number of prompt characters or prompt ID numbers being printed in
/// log. The default of None means unlimited.
#[arg(long)]
pub max_log_len: Option<Unsupported>,
/// If set to True, enable prompt_tokens_details in usage.
#[arg(
long,
visible_alias = "no-enable-prompt-tokens-details",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_prompt_tokens_details: Option<Unsupported>,
/// If set to True, enable tracking server_load_metrics in the app state.
#[arg(
long,
visible_alias = "no-enable-server-load-tracking",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_server_load_tracking: Option<Noop>,
/// If set to True, including usage on every request.
#[arg(
long,
visible_alias = "no-enable-force-include-usage",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_force_include_usage: Option<Unsupported>,
/// Enable the `/tokenizer_info` endpoint. May expose chat
/// templates and other tokenizer configuration.
#[arg(
long,
visible_alias = "no-enable-tokenizer-info-endpoint",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_tokenizer_info_endpoint: Option<Unsupported>,
/// If set to True, log model outputs (generations).
/// Requires `--enable-log-requests`. As with `--enable-log-requests`,
/// information is only logged at INFO level at maximum.
#[arg(
long,
visible_alias = "no-enable-log-outputs",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_log_outputs: Option<Unsupported>,
/// If set to False, output deltas will not be logged. Relevant only if
/// --enable-log-outputs is set.
#[arg(
long,
visible_alias = "no-enable-log-deltas",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_log_deltas: Option<Unsupported>,
/// If set to True, log the stack trace of error responses
#[arg(
long,
// env = "VLLM_SERVER_DEV_MODE",
visible_alias = "no-log-error-stack",
default_missing_value = "true",
num_args = 0..=1
)]
pub log_error_stack: Option<Unsupported>,
/// If set to True, only enable the Tokens In<>Out endpoint.
/// This is intended for use in a Disaggregated Everything setup.
#[arg(
long,
visible_alias = "no-tokens-only",
default_missing_value = "true",
num_args = 0..=1
)]
pub tokens_only: Option<Unsupported>,
/// Log level for uvicorn.
#[arg(long)]
pub uvicorn_log_level: Option<Unsupported>,
/// Disable uvicorn access log.
#[arg(
long,
visible_alias = "no-disable-uvicorn-access-log",
default_missing_value = "true",
num_args = 0..=1
)]
pub disable_uvicorn_access_log: Option<Noop>,
/// Comma-separated list of endpoint paths to exclude from uvicorn access
/// logs. This is useful to reduce log noise from high-frequency endpoints
/// like health checks. Example: "/health,/metrics,/ping".
/// When set, access logs for requests to these paths will be suppressed
/// while keeping logs for other endpoints.
#[arg(long)]
pub disable_access_log_for_endpoints: Option<Noop>,
/// Allow credentials.
#[arg(
long,
visible_alias = "no-allow-credentials",
default_missing_value = "true",
num_args = 0..=1
)]
pub allow_credentials: Option<Unsupported>,
/// Allowed origins.
#[arg(long)]
pub allowed_origins: Option<Unsupported>,
/// Allowed methods.
#[arg(long)]
pub allowed_methods: Option<Unsupported>,
/// Allowed headers.
#[arg(long)]
pub allowed_headers: Option<Unsupported>,
/// If provided, the server will require one of these keys to be presented
/// in the header.
#[arg(long)]
pub api_key: Option<Unsupported>,
/// The file path to the SSL key file.
#[arg(long)]
pub ssl_keyfile: Option<Unsupported>,
/// The file path to the SSL cert file.
#[arg(long)]
pub ssl_certfile: Option<Unsupported>,
/// The CA certificates file.
#[arg(long)]
pub ssl_ca_certs: Option<Unsupported>,
/// Refresh SSL Context when SSL certificate files change
#[arg(
long,
visible_alias = "no-enable-ssl-refresh",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_ssl_refresh: Option<Unsupported>,
/// Whether client certificate is required (see stdlib ssl module's).
#[arg(long)]
pub ssl_cert_reqs: Option<Unsupported>,
/// SSL cipher suites for HTTPS (TLS 1.2 and below only).
/// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
#[arg(long)]
pub ssl_ciphers: Option<Unsupported>,
/// FastAPI root_path when app is behind a path based routing proxy.
#[arg(long)]
pub root_path: Option<Unsupported>,
/// Additional ASGI middleware to apply to the app. We accept multiple
/// --middleware arguments. The value should be an import path. If a
/// function is provided, vLLM will add it to the server using
/// `@app.middleware('http')`. If a class is provided, vLLM will
/// add it to the server using `app.add_middleware()`.
#[arg(long)]
pub middleware: Option<Unsupported>,
/// If specified, API server will add X-Request-Id header to responses.
#[arg(
long,
visible_alias = "no-enable-request-id-headers",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_request_id_headers: Option<Unsupported>,
/// Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint.
#[arg(
long,
visible_alias = "no-disable-fastapi-docs",
default_missing_value = "true",
num_args = 0..=1
)]
pub disable_fastapi_docs: Option<Unsupported>,
/// Maximum size (bytes) of an incomplete HTTP event (header or body) for
/// h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB).
#[arg(long)]
pub h11_max_incomplete_event_size: Option<Unsupported>,
/// Maximum number of HTTP headers allowed in a request for h11 parser.
/// Helps mitigate header abuse. Default: 256.
#[arg(long)]
pub h11_max_header_count: Option<Unsupported>,
/// Enable offline FastAPI documentation for air-gapped environments.
/// Uses vendored static assets bundled with vLLM.
#[arg(
long,
visible_alias = "no-enable-offline-docs",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_offline_docs: Option<Unsupported>,
}
+328
View File
@@ -0,0 +1,328 @@
use std::{env, fmt, process};
use time::UtcOffset;
use time::macros::format_description;
use tracing::level_filters::LevelFilter;
use tracing::{Event, Level, Subscriber};
use tracing_subscriber::Layer as _;
use tracing_subscriber::filter::Targets;
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
use tracing_subscriber::fmt::time::FormatTime;
use tracing_subscriber::fmt::{FmtContext, FormattedFields};
use tracing_subscriber::layer::SubscriberExt as _;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::util::SubscriberInitExt as _;
const CYAN: &str = "\x1b[0;36m";
const GREY: &str = "\x1b[90m";
const GREEN: &str = "\x1b[32m";
const YELLOW: &str = "\x1b[33m";
const RED: &str = "\x1b[31m";
const WHITE: &str = "\x1b[37m";
const RESET: &str = "\x1b[0m";
const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
format_description!("[month]-[day] [hour]:[minute]:[second]");
const PROCESS_LABEL: &str = "RustFrontend";
/// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
pub(crate) fn init_tracing() {
let filter = build_targets_filter(
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
env::var("RUST_LOG").ok().as_deref(),
);
let formatter = VllmEventFormatter::new();
let _ = tracing_subscriber::registry()
.with(tracing_subscriber::fmt::layer().event_format(formatter).with_filter(filter))
.try_init();
}
/// Build the CLI log filter by merging the vLLM-style default level with
/// Rust-style target overrides.
///
/// Precedence:
/// - Start from `VLLM_LOGGING_LEVEL` as the default level for all targets.
/// - If `RUST_LOG` contains a global default level such as `warn`, it overrides
/// `VLLM_LOGGING_LEVEL`.
/// - Any explicit target directives in `RUST_LOG`, such as `hyper=info`, override whichever default
/// level is active for those targets only.
fn build_targets_filter(vllm_logging_level: Option<&str>, rust_log: Option<&str>) -> Targets {
let mut filter =
Targets::new().with_default(map_python_log_level(vllm_logging_level.unwrap_or("INFO")));
if let Some(rust_log) = rust_log
&& !rust_log.is_empty()
{
let rust_log_targets: Targets = rust_log.parse().expect("failed to parse `RUST_LOG`");
if let Some(default_level) = rust_log_targets.default_level() {
filter = filter.with_default(default_level);
}
filter = filter.with_targets(rust_log_targets);
}
filter
}
#[derive(Debug, Clone, Copy)]
struct VllmLocalTimer {
local_offset: UtcOffset,
}
impl Default for VllmLocalTimer {
fn default() -> Self {
let local_offset = UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC);
Self { local_offset }
}
}
impl FormatTime for VllmLocalTimer {
fn format_time(&self, w: &mut Writer<'_>) -> fmt::Result {
let now = time::OffsetDateTime::now_utc().to_offset(self.local_offset);
let formatted = now.format(VLLM_TIME_FORMAT).map_err(|_| fmt::Error)?;
w.write_str(&formatted)
}
}
#[derive(Debug, Clone)]
struct VllmEventFormatter {
prefix: String,
timer: VllmLocalTimer,
}
impl VllmEventFormatter {
fn new() -> Self {
Self {
prefix: format!("({} pid={})", PROCESS_LABEL, process::id()),
timer: VllmLocalTimer::default(),
}
}
fn write_process_prefix(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
write_colored(writer, ansi, Some(CYAN), &self.prefix)?;
writer.write_char(' ')
}
fn write_level(&self, writer: &mut Writer<'_>, level: &Level, ansi: bool) -> fmt::Result {
let (text, color) = match *level {
Level::TRACE => ("TRACE", WHITE),
Level::DEBUG => ("DEBUG", WHITE),
Level::INFO => ("INFO", GREEN),
Level::WARN => ("WARNING", YELLOW),
Level::ERROR => ("ERROR", RED),
};
write_colored(writer, ansi, Some(color), text)
}
fn write_timestamp(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
if ansi {
writer.write_str(GREY)?;
}
if self.timer.format_time(writer).is_err() {
writer.write_str("<unknown time>")?;
}
if ansi {
writer.write_str(RESET)?;
}
Ok(())
}
fn write_location(
&self,
writer: &mut Writer<'_>,
file: Option<&str>,
line: Option<u32>,
full_path: bool,
ansi: bool,
) -> fmt::Result {
let Some(file) = file else {
return Ok(());
};
let file = if full_path {
file
} else {
shorten_file_path(file)
};
if ansi {
writer.write_str(GREY)?;
}
match line {
Some(line) => write!(writer, "[{file}:{line}]")?,
None => write!(writer, "[{file}]")?,
}
if ansi {
writer.write_str(RESET)?;
}
Ok(())
}
fn write_scope<S, N>(&self, ctx: &FmtContext<'_, S, N>, writer: &mut Writer<'_>) -> fmt::Result
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
N: for<'writer> FormatFields<'writer> + 'static,
{
let Some(scope) = ctx.event_scope() else {
return Ok(());
};
let mut seen = false;
for span in scope.from_root() {
if seen {
writer.write_str(":")?;
}
seen = true;
writer.write_str(span.metadata().name())?;
let ext = span.extensions();
if let Some(fields) = ext.get::<FormattedFields<N>>()
&& !fields.is_empty()
{
write!(writer, "{{{fields}}}")?;
}
}
if seen {
writer.write_str(": ")?;
}
Ok(())
}
}
impl<S, N> FormatEvent<S, N> for VllmEventFormatter
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
N: for<'writer> FormatFields<'writer> + 'static,
{
fn format_event(
&self,
ctx: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &Event<'_>,
) -> fmt::Result {
let meta = event.metadata();
let ansi = writer.has_ansi_escapes();
self.write_process_prefix(&mut writer, ansi)?;
self.write_level(&mut writer, meta.level(), ansi)?;
writer.write_char(' ')?;
self.write_timestamp(&mut writer, ansi)?;
writer.write_char(' ')?;
// Use the full file path only when DEBUG (or more verbose) is enabled anywhere,
// independent of the level of this particular event. Filenames alone are often
// ambiguous, but full paths are too noisy for normal INFO-level operation.
let full_path = LevelFilter::current() >= LevelFilter::DEBUG;
self.write_location(&mut writer, meta.file(), meta.line(), full_path, ansi)?;
writer.write_char(' ')?;
self.write_scope(ctx, &mut writer)?;
ctx.format_fields(writer.by_ref(), event)?;
writer.write_char('\n')
}
}
/// Shorten a source file path for log output while preserving enough context
/// for common Rust entrypoint and module filenames.
///
/// - For `mod.rs`, keep the parent directory as `parent/mod.rs`.
/// - For `src/lib.rs` and `src/main.rs`, keep one additional component as `crate/src/lib.rs` or
/// `crate/src/main.rs` when available.
/// - Other files are displayed as just the basename.
fn shorten_file_path(file: &str) -> &str {
let mut parts = file.rsplit('/');
let name = parts.next().unwrap_or(file);
let parent = parts.next();
let grandparent = parts.next();
let Some(parent) = parent else {
return file;
};
if name == "mod.rs" {
return &file[file.len() - parent.len() - 1 - name.len()..];
}
if !matches!(name, "lib.rs" | "main.rs") || parent != "src" {
return name;
}
let Some(grandparent) = grandparent else {
return file;
};
&file[file.len() - grandparent.len() - 1 - parent.len() - 1 - name.len()..]
}
fn write_colored(
writer: &mut Writer<'_>,
ansi: bool,
color: Option<&str>,
text: &str,
) -> fmt::Result {
if ansi {
if let Some(color) = color {
writer.write_str(color)?;
}
writer.write_str(text)?;
if color.is_some() {
writer.write_str(RESET)?;
}
return Ok(());
}
writer.write_str(text)
}
/// Map a Python logging level name to the corresponding Rust tracing level.
fn map_python_log_level(level: &str) -> LevelFilter {
match level.to_ascii_uppercase().as_str() {
"CRITICAL" | "FATAL" => LevelFilter::ERROR,
"ERROR" => LevelFilter::ERROR,
"WARNING" | "WARN" => LevelFilter::WARN,
"INFO" => LevelFilter::INFO,
"DEBUG" => LevelFilter::DEBUG,
"NOTSET" => LevelFilter::TRACE,
_ => LevelFilter::INFO,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rust_log_target_overrides_are_merged_with_vllm_default_level() {
let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error"));
assert_eq!(filter.to_string(), "tower=error,hyper=warn,debug");
}
#[test]
fn rust_log_default_level_overrides_vllm_default_level() {
let filter = build_targets_filter(Some("DEBUG"), Some("warn,hyper=info"));
assert_eq!(filter.to_string(), "hyper=info,warn");
}
#[test]
fn invalid_vllm_level_falls_back_to_info() {
let filter = build_targets_filter(Some("bogus"), None);
assert_eq!(filter.to_string(), "info");
}
#[test]
fn location_path_uses_filename_for_non_ambiguous_files() {
assert_eq!(shorten_file_path("src/cmd/src/logging.rs"), "logging.rs");
assert_eq!(shorten_file_path("src/chat/lib.rs"), "lib.rs");
assert_eq!(shorten_file_path("src/chat/main.rs"), "main.rs");
assert_eq!(shorten_file_path("src/chat/src/xmod.rs"), "xmod.rs");
}
#[test]
fn location_path_keeps_more_context_for_common_entrypoint_filenames() {
assert_eq!(shorten_file_path("src/lib.rs"), "src/lib.rs");
assert_eq!(shorten_file_path("src/chat/src/lib.rs"), "chat/src/lib.rs");
assert_eq!(shorten_file_path("src/cmd/src/main.rs"), "cmd/src/main.rs");
assert_eq!(shorten_file_path("mod.rs"), "mod.rs");
assert_eq!(shorten_file_path("src/chat/src/tool/mod.rs"), "tool/mod.rs");
}
}
+189
View File
@@ -0,0 +1,189 @@
mod cli;
mod logging;
use std::env;
use std::process::ExitStatus;
use anyhow::{Context, Result, anyhow, bail};
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use vllm_managed_engine::ManagedEngineHandle;
use crate::cli::{Cli, Command};
const TOKIO_WORKER_THREADS_ENV: &str = "TOKIO_WORKER_THREADS";
const DEFAULT_MAX_TOKIO_WORKER_THREADS: usize = 32;
/// Cap the default number of Tokio worker threads if the user did not
/// explicitly set `TOKIO_WORKER_THREADS` to avoid spawning too many threads on
/// machines with a large number of CPUs, which may lead to excessive context
/// switching and degraded performance.
fn tokio_worker_threads() -> Option<usize> {
if env::var_os(TOKIO_WORKER_THREADS_ENV).is_some() {
return None;
}
std::thread::available_parallelism()
.map(|parallelism| {
let available = parallelism.get();
let worker_threads = available.min(DEFAULT_MAX_TOKIO_WORKER_THREADS);
if worker_threads < available {
info!(
available_parallelism = available,
capped_worker_threads = worker_threads,
"capping tokio worker threads, set {TOKIO_WORKER_THREADS_ENV} to override"
);
}
worker_threads
})
.ok()
}
/// Reason that caused a managed `serve` session to stop.
#[derive(Debug)]
enum ShutdownReason {
Signal,
Server(anyhow::Error),
EngineExited(ExitStatus),
}
/// Cancellation token tripped by Ctrl-C or SIGTERM.
fn shutdown_signal() -> CancellationToken {
let token = CancellationToken::new();
let shutdown = token.clone();
tokio::spawn(async move {
let ctrl_c = async {
tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
};
let sigterm = async {
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.expect("failed to install SIGTERM signal handler")
.recv()
.await;
};
tokio::select! {
_ = ctrl_c => info!("received shutdown signal (Ctrl-C), shutting down..."),
_ = sigterm => info!("received shutdown signal (SIGTERM), shutting down..."),
}
shutdown.cancel();
});
token
}
fn main() -> Result<()> {
logging::init_tracing();
let cli = Cli::parse();
let mut runtime = tokio::runtime::Builder::new_multi_thread();
runtime.enable_all();
if let Some(worker_threads) = tokio_worker_threads() {
runtime.worker_threads(worker_threads);
}
runtime
.build()
.context("failed to build Tokio runtime")?
.block_on(async_main(cli))
}
async fn async_main(cli: Cli) -> Result<()> {
match cli.command {
Command::Frontend(args) => vllm_server::serve(args.into_config(), shutdown_signal()).await,
Command::Serve(args) => {
let handshake_port = args.managed_engine.resolve_handshake_port()?;
if args.managed_engine.data_parallel_size_local == Some(0) {
if args.headless {
bail!("cannot combine `--headless` with `--data-parallel-size-local 0`");
}
let handshake_address = args.managed_engine.handshake_address(handshake_port);
info!(
%handshake_address,
engine_count = args.managed_engine.data_parallel_size,
"running Rust frontend without a managed local Python engine"
);
let config = args.to_frontend_config(handshake_address);
return vllm_server::serve(config, shutdown_signal()).await;
}
let shutdown_timeout = args.runtime.shutdown_timeout();
let engine_config = args.to_managed_engine_config(handshake_port);
let handshake_address = engine_config.handshake_address();
let engine = ManagedEngineHandle::spawn(engine_config)
.await
.context("failed to start managed Python headless engine")?;
let shutdown = shutdown_signal();
let mut serve_task = if args.headless {
info!("running managed Python headless engine without Rust frontend");
let shutdown = shutdown.clone();
tokio::spawn(async move {
shutdown.cancelled().await;
Ok(())
})
} else {
let config = args.to_frontend_config(handshake_address);
let shutdown = shutdown.clone();
tokio::spawn(async move {
let result = vllm_server::serve(config, shutdown).await;
if result.is_ok() {
info!("OpenAI server shut down gracefully");
}
result
})
};
let shutdown_reason = tokio::select! {
biased;
// Received shutdown signal via Ctrl-C or SIGTERM.
_ = shutdown.cancelled() => ShutdownReason::Signal,
// Engine process exited unexpectedly.
status = engine.wait_for_exit() => {
warn!(%status, "managed Python headless engine exited, shutting down...");
ShutdownReason::EngineExited(status)
}
// Serve task exited unexpectedly.
serve_result = &mut serve_task => {
let serve_result = serve_result.context("serve task join failed")?;
match serve_result {
Ok(()) => ShutdownReason::Server(anyhow!("OpenAI server shut down unexpectedly without error")),
Err(error) => ShutdownReason::Server(error),
}
}
};
// Regardless of the shutdown reason, broadcast shutdown signal here to ensure
// that all serving tasks are notified.
shutdown.cancel();
// Shutdown begins. Terminate the managed engine first.
engine.shutdown(shutdown_timeout).await?;
info!("managed engine shut down gracefully");
// Wait for the API server to shut down gracefully by draining in-flight
// requests.
if !matches!(shutdown_reason, ShutdownReason::Server(_)) {
serve_task.await.context("serve task join failed")??;
}
match shutdown_reason {
ShutdownReason::Signal => Ok(()),
ShutdownReason::Server(error) => {
Err(error.context("OpenAI server shut down unexpectedly"))
}
ShutdownReason::EngineExited(status) => Err(anyhow!(
"managed Python headless engine exited unexpectedly with status {status}"
)),
}
}
}
}