[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "vllm-cmd"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "vllm-rs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
native-tls-vendored = ["dep:native-tls-vendored"]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
educe.workspace = true
|
||||
itertools.workspace = true
|
||||
native-tls-vendored = { workspace = true, optional = true }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["signal"] }
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-server.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
expect-test.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,44 @@
|
||||
# `vllm-rs` CLI Quick Start
|
||||
|
||||
Start Qwen3 with one managed `vllm-rs serve` command from the repo root:
|
||||
|
||||
```bash
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
cargo run --bin vllm-rs -- serve \
|
||||
Qwen/Qwen3-0.6B \
|
||||
--python ../vllm/.venv/bin/python \
|
||||
--max-model-len 512 \
|
||||
-- \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
This launches:
|
||||
|
||||
- a managed headless Python `vllm` engine
|
||||
- the Rust OpenAI-compatible frontend on `127.0.0.1:8000`
|
||||
|
||||
All Python engine arguments must be placed after `--`. Arguments before `--` are parsed by the Rust
|
||||
frontend itself.
|
||||
|
||||
You can then send OpenAI-style requests to the Rust frontend:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
If you already started headless `vllm` yourself, use `frontend` instead:
|
||||
|
||||
```bash
|
||||
cargo run --bin vllm-rs -- frontend \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
Qwen/Qwen3-0.6B
|
||||
```
|
||||
@@ -0,0 +1,434 @@
|
||||
//! CLI argument definitions for the `vllm-rs` binary.
|
||||
//!
|
||||
//! Python vLLM references:
|
||||
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
|
||||
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
|
||||
|
||||
mod unsupported;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use educe::Educe;
|
||||
use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
|
||||
RendererSelection,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
|
||||
/// Top-level parser for the `vllm-rs` binary.
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
name = "vllm-rs",
|
||||
about = "Rust frontend and managed-engine CLI for vLLM."
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[command(subcommand)]
|
||||
pub command: Command,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub fn parse() -> Self {
|
||||
Self::try_parse_from(std::env::args_os()).unwrap_or_else(|error| error.exit())
|
||||
}
|
||||
|
||||
pub fn try_parse_from<I, T>(itr: I) -> Result<Self, clap::Error>
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<OsString>,
|
||||
{
|
||||
let args: Vec<OsString> = itr.into_iter().map(Into::into).collect();
|
||||
let repartitioned_args = repartition_managed_engine_args::<Self>(&args, Some("serve"))?;
|
||||
<Self as Parser>::try_parse_from(&repartitioned_args).inspect(|cli| {
|
||||
if let Command::Serve(serve) = &cli.command
|
||||
&& serve.debug_cli
|
||||
{
|
||||
println!(
|
||||
"Original CLI args: {}\n",
|
||||
args.join(OsStr::new(" ")).display()
|
||||
);
|
||||
println!(
|
||||
"Repartitioned CLI args: {}\n",
|
||||
repartitioned_args.join(OsStr::new(" ")).display()
|
||||
);
|
||||
println!(
|
||||
"Passthrough Python args: {}",
|
||||
serve.managed_engine.python_args.join(" ")
|
||||
);
|
||||
std::process::exit(0);
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Supported top-level CLI commands.
|
||||
#[derive(Debug, Subcommand, PartialEq, Eq)]
|
||||
pub enum Command {
|
||||
/// Run the Rust OpenAI frontend as a Python-supervised worker.
|
||||
Frontend(FrontendArgs),
|
||||
/// Launch a managed Python headless engine, then run the Rust OpenAI
|
||||
/// frontend.
|
||||
Serve(ServeArgs),
|
||||
}
|
||||
|
||||
/// Runtime arguments shared by the external-engine and managed-engine paths.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
||||
#[educe(Debug)]
|
||||
pub struct SharedRuntimeArgs {
|
||||
#[serde(rename = "model_tag")]
|
||||
/// Model identifier or local model directory used for backend loading and
|
||||
/// public model ID.
|
||||
pub model: String,
|
||||
|
||||
/// Maximum time to wait for the expected engines to register on the
|
||||
/// frontend transport.
|
||||
#[arg(
|
||||
long = "engine-ready-timeout-secs",
|
||||
env = "VLLM_ENGINE_READY_TIMEOUT_S",
|
||||
default_value_t = default_engine_ready_timeout_secs()
|
||||
)]
|
||||
#[serde(default = "default_engine_ready_timeout_secs")]
|
||||
pub engine_ready_timeout_secs: u64,
|
||||
|
||||
/// Select the tool call parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Select the reasoning parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
#[serde(default, rename = "tokenizer_mode")]
|
||||
pub renderer: RendererSelection,
|
||||
/// Override the maximum model context length. When set, the frontend uses
|
||||
/// this value instead of the model's `max_position_embeddings` from
|
||||
/// `config.json`.
|
||||
#[arg(long)]
|
||||
pub max_model_len: Option<u32>,
|
||||
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
|
||||
/// started.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
pub shutdown_timeout: u64,
|
||||
|
||||
/// The file path to the chat template, or the template in single-line form
|
||||
/// for the specified model.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Default keyword arguments to pass to the chat template renderer.
|
||||
///
|
||||
/// These will be merged with request-level chat_template_kwargs, with
|
||||
/// request values taking precedence. Useful for setting default
|
||||
/// behavior for reasoning models.
|
||||
///
|
||||
/// Example: `{"enable_thinking": false}` to disable thinking mode by
|
||||
/// default for Qwen3/DeepSeek models.
|
||||
#[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
|
||||
#[serde(default)]
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// The format to render message content within a chat template.
|
||||
///
|
||||
/// * "auto" detects the format from the template
|
||||
/// * "string" renders content as a string. Example: `"Hello World"`
|
||||
/// * "openai" renders content as a list of dictionaries, similar to OpenAI schema. Example:
|
||||
/// `[{"type": "text", "text": "Hello world!"}]`
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
|
||||
/// Log a summary line for each completed request, including prompt/output
|
||||
/// token counts and finish reason.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub enable_log_requests: bool,
|
||||
|
||||
/// Disable periodic logging of engine statistics (throughput, queue depth,
|
||||
/// cache usage).
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub disable_log_stats: bool,
|
||||
|
||||
/// The model name(s) used in the API. If multiple names are provided, the
|
||||
/// server will respond to any of the provided names. The model name in the
|
||||
/// model field of a response will be the first name in this list. If not
|
||||
/// specified, the model name will be the same as the `--model` argument.
|
||||
/// Noted that this name(s) will also be used in `model_name` tag
|
||||
/// content of prometheus metrics, if multiple names provided, metrics
|
||||
/// tag will take the first one.
|
||||
#[arg(long, num_args = 0..)]
|
||||
#[serde(default)]
|
||||
pub served_model_name: Vec<String>,
|
||||
|
||||
/// Unsupported Python vLLM frontend arguments recognized but not yet
|
||||
/// implemented in Rust.
|
||||
#[educe(Debug(ignore))]
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
pub unsupported: UnsupportedArgs,
|
||||
}
|
||||
|
||||
impl SharedRuntimeArgs {
|
||||
/// Maximum time to wait for the expected engines to register on the
|
||||
/// frontend transport.
|
||||
pub fn ready_timeout(&self) -> Duration {
|
||||
Duration::from_secs(self.engine_ready_timeout_secs)
|
||||
}
|
||||
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
pub fn shutdown_timeout(&self) -> Duration {
|
||||
Duration::from_secs(self.shutdown_timeout)
|
||||
}
|
||||
|
||||
/// Build the OpenAI-server config for the Python-bootstrap worker contract.
|
||||
///
|
||||
/// The resulting config binds the Python-supplied transport addresses and
|
||||
/// inherits an already open HTTP listener from the supervisor process.
|
||||
fn into_bootstrapped_config(
|
||||
self,
|
||||
listen_fd: i32,
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
coordinator_address: Option<String>,
|
||||
engine_count: usize,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
input_address,
|
||||
output_address,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
},
|
||||
coordinator_mode: match coordinator_address {
|
||||
Some(address) => CoordinatorMode::External { address },
|
||||
None => CoordinatorMode::None,
|
||||
},
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the OpenAI-server config for the managed `serve` path that still
|
||||
/// owns the startup handshake and binds its own HTTP listener.
|
||||
fn into_managed_config(
|
||||
self,
|
||||
listener_mode: HttpListenerMode,
|
||||
handshake_address: String,
|
||||
advertised_host: String,
|
||||
engine_count: usize,
|
||||
local_input_address: Option<String>,
|
||||
local_output_address: Option<String>,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_engine_ready_timeout_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||
}
|
||||
|
||||
fn parse_runtime_args_json(value: &str) -> Result<SharedRuntimeArgs, String> {
|
||||
let args: SharedRuntimeArgs = serde_json::from_str(value)
|
||||
.map_err(|e| format!("invalid JSON arguments: {}", e.as_report()))?;
|
||||
args.unsupported.check()?;
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
/// Arguments for running the Rust frontend as a Python-bootstrapped worker.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq)]
|
||||
#[educe(Debug)]
|
||||
pub struct FrontendArgs {
|
||||
/// Inherited listening socket file descriptor passed by the Python
|
||||
/// supervisor.
|
||||
#[arg(long)]
|
||||
pub listen_fd: i32,
|
||||
/// Frontend input ROUTER socket address that the Python engines will
|
||||
/// connect to.
|
||||
#[arg(long)]
|
||||
pub input_address: String,
|
||||
/// Frontend output PULL socket address that the Python engines will push
|
||||
/// responses to.
|
||||
#[arg(long)]
|
||||
pub output_address: String,
|
||||
/// Optional Python-owned frontend-side DP coordinator socket address for
|
||||
/// external coordinator mode in the bootstrapped frontend path, i.e.,
|
||||
/// `stats_update_address`.
|
||||
#[arg(long)]
|
||||
pub coordinator_address: Option<String>,
|
||||
/// Total number of data-parallel engines expected for this frontend.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub engine_count: usize,
|
||||
|
||||
/// Shared frontend arguments as one JSON object.
|
||||
#[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
|
||||
pub runtime: SharedRuntimeArgs,
|
||||
}
|
||||
|
||||
impl FrontendArgs {
|
||||
/// Convert the CLI arguments into the OpenAI server's runtime config.
|
||||
pub fn into_config(self) -> Config {
|
||||
self.runtime.into_bootstrapped_config(
|
||||
self.listen_fd,
|
||||
self.input_address,
|
||||
self.output_address,
|
||||
self.coordinator_address,
|
||||
self.engine_count,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Arguments for the managed-engine mode that spawns Python on behalf of the
|
||||
/// user.
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq)]
|
||||
#[educe(Debug)]
|
||||
#[command(override_usage = "vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]")]
|
||||
pub struct ServeArgs {
|
||||
/// Only launch the managed Python headless engine and do not start the Rust
|
||||
/// frontend.
|
||||
#[arg(long)]
|
||||
pub headless: bool,
|
||||
/// HTTP bind host for the OpenAI-compatible server.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
pub host: String,
|
||||
/// HTTP bind port for the OpenAI-compatible server.
|
||||
#[arg(long, default_value_t = 8000)]
|
||||
pub port: u16,
|
||||
/// Unix domain socket path. If set, host and port arguments are ignored.
|
||||
#[arg(long)]
|
||||
pub uds: Option<String>,
|
||||
|
||||
/// Flag to print debug information about CLI argument parsing and exit.
|
||||
#[educe(Debug(ignore))]
|
||||
#[arg(long, hide = true, env = "VLLM_RS_DEBUG_CLI")]
|
||||
pub debug_cli: bool,
|
||||
|
||||
/// Shared frontend arguments.
|
||||
#[command(flatten)]
|
||||
pub runtime: SharedRuntimeArgs,
|
||||
|
||||
/// Managed Python headless-engine arguments.
|
||||
#[command(flatten)]
|
||||
pub managed_engine: ManagedEngineArgs,
|
||||
}
|
||||
|
||||
impl ServeArgs {
|
||||
/// Build the OpenAI-server runtime config used after the managed Python
|
||||
/// engine starts.
|
||||
pub fn to_frontend_config(&self, handshake_address: String) -> Config {
|
||||
// Prefer IPC sockets for local engine input/output.
|
||||
let (local_input_address, local_output_address) =
|
||||
self.managed_engine.frontend_local_only().then(frontend_ipc_addresses).unzip();
|
||||
let listener_mode = match &self.uds {
|
||||
Some(path) => HttpListenerMode::BindUnix { path: path.clone() },
|
||||
None => HttpListenerMode::BindTcp {
|
||||
host: self.host.clone(),
|
||||
port: self.port,
|
||||
},
|
||||
};
|
||||
|
||||
self.runtime.clone().into_managed_config(
|
||||
listener_mode,
|
||||
handshake_address,
|
||||
self.managed_engine.handshake_host.clone(),
|
||||
self.managed_engine.data_parallel_size,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the managed Python-engine spawn configuration with the given
|
||||
/// handshake port.
|
||||
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
handshake_port,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
||||
fn frontend_ipc_addresses() -> (String, String) {
|
||||
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(std::env::temp_dir);
|
||||
let input_name = format!("vllm-rs-i-{}", Uuid::new_v4().simple());
|
||||
let output_name = format!("vllm-rs-o-{}", Uuid::new_v4().simple());
|
||||
|
||||
let input = preferred_base_path.join(input_name);
|
||||
let output = preferred_base_path.join(output_name);
|
||||
|
||||
(
|
||||
format!("ipc://{}", input.to_string_lossy()),
|
||||
format!("ipc://{}", output.to_string_lossy()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,905 @@
|
||||
use expect_test::expect;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
|
||||
|
||||
use super::{Cli, Command};
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_python_flags_with_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"../vllm/.venv/bin/python",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Serve(
|
||||
ServeArgs {
|
||||
headless: false,
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: Some(
|
||||
512,
|
||||
),
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "../vllm/.venv/bin/python",
|
||||
handshake_host: "127.0.0.1",
|
||||
handshake_port: None,
|
||||
data_parallel_size: 1,
|
||||
data_parallel_size_local: None,
|
||||
python_args: [
|
||||
"--dtype",
|
||||
"float16",
|
||||
],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_flags_without_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--quantization",
|
||||
"awq",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--quantization", "awq"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_explicit_deepseek_v32_renderer() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--tokenizer-mode",
|
||||
"deepseek_v32",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unknown_renderer_value() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--tokenizer-mode",
|
||||
"definitely_missing",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_flag_arg() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--allow-credentials"])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'true' for '--allow-credentials [<ALLOW_CREDENTIALS>]': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_no_flag_alias() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--no-enable-log-deltas",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'true' for '--enable-log-deltas [<ENABLE_LOG_DELTAS>]': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_accept_json() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--coordinator-address",
|
||||
"tcp://127.0.0.1:7000",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_count":2}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Frontend(
|
||||
FrontendArgs {
|
||||
listen_fd: 3,
|
||||
input_address: "ipc:///tmp/input.sock",
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
coordinator_address: Some(
|
||||
"tcp://127.0.0.1:7000",
|
||||
),
|
||||
engine_count: 1,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_applies_defaults() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_model_len, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","max_model_len":8192,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
|
||||
assert_eq!(
|
||||
args.runtime.tool_call_parser,
|
||||
ParserSelection::Explicit("hermes".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
args.runtime.reasoning_parser,
|
||||
ParserSelection::Explicit("qwen3_thinking".to_string())
|
||||
);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
assert_eq!(args.runtime.max_model_len, Some(8192));
|
||||
assert_eq!(args.runtime.shutdown_timeout, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_none_reasoning_parser() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--reasoning-parser",
|
||||
"none",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_ignores_unknown_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","uds":"/tmp/vllm.sock","nested_unknown":{"x":1}}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_noop_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_unsupported_fields() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_aggregates_multiple_unsupported_fields() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"api_key":"secret"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
- api_key
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_malformed_json() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B""#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B"' for '--args-json <JSON>': invalid JSON arguments: EOF while parsing an object at line 1 column 30
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_flags_before_model() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: the model must appear immediately after the command
|
||||
|
||||
Usage: vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_headless_mode() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--headless"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert!(args.headless);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_passthrough_flags_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2", "--dtype", "float16"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_multi_char_alias_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--",
|
||||
"-tp",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-tp", "2", "--dtype", "float16"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_frontend_arg_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--",
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--uds", "/tmp/vllm.sock"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--",
|
||||
"-dpr",
|
||||
"1",
|
||||
"-dpl",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-dpr", "1", "-dpl", "2"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_unknown_flags_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--foo", "bar"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_negative_value_without_separator() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--num-gpu-blocks-override",
|
||||
"-1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--num-gpu-blocks-override", "-1"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_handshake_aliases() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--python",
|
||||
"python3",
|
||||
"--handshake-host",
|
||||
"10.99.48.128",
|
||||
"--handshake-port",
|
||||
"13345",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
Cli {
|
||||
command: Serve(
|
||||
ServeArgs {
|
||||
headless: false,
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
max_model_len: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "python3",
|
||||
handshake_host: "10.99.48.128",
|
||||
handshake_port: Some(
|
||||
13345,
|
||||
),
|
||||
data_parallel_size: 4,
|
||||
data_parallel_size_local: None,
|
||||
python_args: [],
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&cli);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_accept_data_parallel_primary_flags() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--data-parallel-address",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-rpc-port",
|
||||
"13345",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert!(!args.headless);
|
||||
assert_eq!(args.managed_engine.handshake_host, "10.99.48.128");
|
||||
assert_eq!(args.managed_engine.handshake_port, Some(13345));
|
||||
assert_eq!(args.managed_engine.data_parallel_size, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--handshake-host",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
|
||||
|
||||
let TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
} = &config.transport_mode
|
||||
else {
|
||||
panic!("expected handshake-owned transport");
|
||||
};
|
||||
|
||||
assert_eq!(handshake_address, "tcp://10.99.48.128:29550");
|
||||
assert_eq!(advertised_host, "10.99.48.128");
|
||||
assert_eq!(*engine_count, 4);
|
||||
assert!(
|
||||
local_input_address
|
||||
.as_deref()
|
||||
.is_some_and(|address| address.starts_with("ipc://"))
|
||||
);
|
||||
assert!(
|
||||
local_output_address
|
||||
.as_deref()
|
||||
.is_some_and(|address| address.starts_with("ipc://"))
|
||||
);
|
||||
assert_ne!(local_input_address, local_output_address);
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: HandshakeOwner {
|
||||
handshake_address: "tcp://10.99.48.128:29550",
|
||||
advertised_host: "10.99.48.128",
|
||||
engine_count: 4,
|
||||
ready_timeout: 600s,
|
||||
local_input_address: Some(
|
||||
"<ipc input>",
|
||||
),
|
||||
local_output_address: Some(
|
||||
"<ipc output>",
|
||||
),
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: handshake_address.clone(),
|
||||
advertised_host: advertised_host.clone(),
|
||||
engine_count: *engine_count,
|
||||
ready_timeout: *ready_timeout,
|
||||
local_input_address: Some("<ipc input>".to_string()),
|
||||
local_output_address: Some("<ipc output>".to_string()),
|
||||
},
|
||||
..config.clone()
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--data-parallel-address",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-size",
|
||||
"4",
|
||||
"--data-parallel-size-local",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: HandshakeOwner {
|
||||
handshake_address: "tcp://10.99.48.128:29550",
|
||||
advertised_host: "10.99.48.128",
|
||||
engine_count: 4,
|
||||
ready_timeout: 600s,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_reject_legacy_handshake_flags() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
"--handshake-address",
|
||||
"tcp://127.0.0.1:62100",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("--handshake-address"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--coordinator-address",
|
||||
"tcp://127.0.0.1:7000",
|
||||
"--engine-count",
|
||||
"2",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
|
||||
expect![[r#"
|
||||
Config {
|
||||
transport_mode: Bootstrapped {
|
||||
input_address: "ipc:///tmp/input.sock",
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
engine_count: 2,
|
||||
ready_timeout: 600s,
|
||||
},
|
||||
coordinator_mode: External {
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
fd: 3,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
renderer: Auto,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
|
||||
|
||||
assert_eq!(
|
||||
config.listener_mode,
|
||||
HttpListenerMode::BindUnix {
|
||||
path: "/tmp/vllm.sock".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,660 @@
|
||||
#![allow(clippy::doc_lazy_continuation)]
|
||||
|
||||
use std::fmt::Display;
|
||||
use std::str::FromStr;
|
||||
|
||||
use clap::Args;
|
||||
use clap::builder::{TypedValueParser, ValueParserFactory};
|
||||
use itertools::Itertools;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
/// Marker type for frontend-owned `serve` arguments that `vllm-rs` recognizes
|
||||
/// but does not support yet.
|
||||
///
|
||||
/// When passed as JSON args, it can be deserialized from any value, and
|
||||
/// serializes back to the original value.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct Unsupported(pub serde_json::Value);
|
||||
|
||||
impl FromStr for Unsupported {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(_s: &str) -> Result<Self, Self::Err> {
|
||||
Err("argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument."
|
||||
.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Marker type for no-op arguments that are accepted by the Rust frontend but
|
||||
/// have no effect.
|
||||
///
|
||||
/// When passed as JSON args, it can be deserialized from any value, but always
|
||||
/// serializes back to `null`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Noop;
|
||||
|
||||
impl<'de> Deserialize<'de> for Noop {
|
||||
fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Ok(Noop)
|
||||
}
|
||||
}
|
||||
|
||||
impl ValueParserFactory for Noop {
|
||||
type Parser = NoopValueParser;
|
||||
|
||||
fn value_parser() -> Self::Parser {
|
||||
NoopValueParser
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct NoopValueParser;
|
||||
|
||||
#[track_caller]
|
||||
fn noop_warn(arg: impl Display) {
|
||||
tracing::warn!("argument '{arg}' currently has no effect in Rust frontend, ignoring");
|
||||
}
|
||||
|
||||
impl TypedValueParser for NoopValueParser {
|
||||
type Value = Noop;
|
||||
|
||||
fn parse_ref(
|
||||
&self,
|
||||
_cmd: &clap::Command,
|
||||
arg: Option<&clap::Arg>,
|
||||
_value: &std::ffi::OsStr,
|
||||
) -> Result<Self::Value, clap::Error> {
|
||||
if let Some(arg) = arg {
|
||||
noop_warn(arg);
|
||||
}
|
||||
Ok(Noop)
|
||||
}
|
||||
}
|
||||
|
||||
/// Frontend-owned Python `serve` arguments that `vllm-rs` recognizes but does
|
||||
/// not support yet.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
#[command(next_help_heading = "Options not implemented in Rust frontend yet")]
|
||||
pub struct UnsupportedArgs {
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
top_level: TopLevelUnsupportedArgs,
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
engine: EngineUnsupportedArgs,
|
||||
#[command(flatten)]
|
||||
#[serde(default, flatten)]
|
||||
server: ServerUnsupportedArgs,
|
||||
}
|
||||
|
||||
impl UnsupportedArgs {
|
||||
/// Check whether any unsupported arguments are set, and if so, return an
|
||||
/// error listing them. Also warn about any no-op arguments that are set
|
||||
/// but will be ignored.
|
||||
pub(crate) fn check(&self) -> Result<(), String> {
|
||||
let value = serde_json::to_value(self).unwrap();
|
||||
let map = value.as_object().unwrap();
|
||||
let mut unsupported = Vec::new();
|
||||
|
||||
for (key, value) in map {
|
||||
if value.is_null() {
|
||||
noop_warn(key);
|
||||
} else {
|
||||
unsupported.push(key.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
if !unsupported.is_empty() {
|
||||
unsupported.sort_unstable();
|
||||
let bullets = unsupported.into_iter().map(|key| format!("- {key}")).join("\n");
|
||||
return Err(format!(
|
||||
"
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
{bullets}
|
||||
|
||||
Remove these arguments to continue."
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Frontend-owned Python `vllm serve` top-level arguments that `vllm-rs`
|
||||
/// recognizes but does not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.entrypoints.openai.cli_args.make_arg_parser(...)`
|
||||
/// - `vllm.entrypoints.cli.serve.ServeSubcommand.subparser_init(...)`
|
||||
///
|
||||
/// These are not part of `EngineArgs`, `AsyncEngineArgs`, `BaseFrontendArgs`,
|
||||
/// or `FrontendArgs`. They live on the `serve` command itself and control
|
||||
/// managed-engine / multi-process orchestration rather than the shared frontend
|
||||
/// runtime config.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct TopLevelUnsupportedArgs {
|
||||
/// How many API server processes to run. Defaults to data_parallel_size if
|
||||
/// not specified.
|
||||
#[arg(long, hide = true)]
|
||||
pub api_server_count: Option<Noop>,
|
||||
|
||||
/// Read CLI options from a config file. Must be a YAML with the following
|
||||
/// options: https://docs.vllm.ai/en/latest/configuration/serve_args.html
|
||||
#[arg(long)]
|
||||
pub config: Option<Unsupported>,
|
||||
|
||||
/// Launch a gRPC server instead of the HTTP OpenAI-compatible server.
|
||||
/// Requires: pip install vllm[grpc].
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub grpc: Option<Unsupported>,
|
||||
}
|
||||
|
||||
/// Frontend-owned Python engine arguments that `vllm-rs` recognizes but does
|
||||
/// not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.engine.arg_utils.EngineArgs.add_cli_args(...)`
|
||||
/// - `vllm.engine.arg_utils.AsyncEngineArgs.add_cli_args(...)`
|
||||
///
|
||||
/// These arguments are declared through the Python engine-args surface, but
|
||||
/// they are still frontend-owned: the API server / AsyncLLM layer reads them
|
||||
/// for tokenizer setup, request validation, routing, logging, and other
|
||||
/// frontend behavior, so Rust must recognize them rather than treating them as
|
||||
/// pure engine passthrough.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct EngineUnsupportedArgs {
|
||||
/// Name or path of the Hugging Face tokenizer to use. If unspecified, model
|
||||
/// name or path will be used.
|
||||
#[arg(long)]
|
||||
pub tokenizer: Option<Unsupported>,
|
||||
|
||||
/// Name or path of the Hugging Face config to use. If unspecified, model
|
||||
/// name or path will be used.
|
||||
#[arg(long)]
|
||||
pub hf_config_path: Option<Unsupported>,
|
||||
|
||||
/// Allowing API requests to read local images or videos from directories
|
||||
/// specified by the server file system. This is a security risk. Should
|
||||
/// only be enabled in trusted environments.
|
||||
#[arg(long)]
|
||||
pub allowed_local_media_path: Option<Unsupported>,
|
||||
|
||||
/// If set, only media URLs that belong to this domain can be used for
|
||||
/// multi-modal inputs.
|
||||
#[arg(long)]
|
||||
pub allowed_media_domains: Option<Unsupported>,
|
||||
|
||||
/// The specific revision to use for the tokenizer on the Hugging Face Hub.
|
||||
/// It can be a branch name, a tag name, or a commit id. If unspecified,
|
||||
/// will use the default version.
|
||||
#[arg(long)]
|
||||
pub tokenizer_revision: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in `SamplingParams`. The default value comes the default for
|
||||
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
|
||||
/// (output_length * vocab_size) logprobs are allowed to be returned and
|
||||
/// it may cause OOM.
|
||||
#[arg(long)]
|
||||
pub max_logprobs: Option<Unsupported>,
|
||||
|
||||
/// Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
/// `prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
/// output will contain token ids.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-skip-tokenizer-init",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub skip_tokenizer_init: Option<Unsupported>,
|
||||
|
||||
/// If `True`, enables passing text embeddings as inputs via the
|
||||
/// `prompt_embeds` key.
|
||||
///
|
||||
/// WARNING: The vLLM engine may crash if incorrect shape of embeddings is
|
||||
/// passed. Only enable this flag for trusted users!
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-prompt-embeds",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_prompt_embeds: Option<Unsupported>,
|
||||
|
||||
/// The token to use as HTTP bearer authorization for remote files. If
|
||||
/// `True`, will use the token generated when running `hf auth login`
|
||||
/// (stored in `~/.cache/huggingface/token`).
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub hf_token: Option<Unsupported>,
|
||||
|
||||
/// If a dictionary, contains arguments to be forwarded to the Hugging Face
|
||||
/// config. If a callable, it is called to update the HuggingFace config.
|
||||
#[arg(long)]
|
||||
pub hf_overrides: Option<Unsupported>,
|
||||
|
||||
/// The folder path to the generation config. Defaults to `"auto"`, the
|
||||
/// generation config will be loaded from model path. If set to `"vllm"`, no
|
||||
/// generation config is loaded, vLLM defaults will be used. If set to a
|
||||
/// folder path, the generation config will be loaded from the specified
|
||||
/// folder path. If `max_new_tokens` is specified in generation config,
|
||||
/// then it sets a server-wide limit on the number of output tokens for
|
||||
/// all requests.
|
||||
#[arg(long)]
|
||||
pub generation_config: Option<Unsupported>,
|
||||
|
||||
/// IOProcessor plugin name to load at model startup
|
||||
#[arg(long)]
|
||||
pub io_processor_plugin: Option<Unsupported>,
|
||||
|
||||
/// Path to a dynamically reasoning parser plugin that can be dynamically
|
||||
/// loaded and registered.
|
||||
#[arg(long)]
|
||||
pub reasoning_parser_plugin: Option<Unsupported>,
|
||||
|
||||
/// Rank of the data parallel group.
|
||||
#[arg(long, env = "VLLM_DP_RANK")]
|
||||
pub data_parallel_rank: Option<Unsupported>,
|
||||
|
||||
/// Whether to use "hybrid" DP LB mode. Applies only to online serving
|
||||
/// and when data_parallel_size > 0. Enables running an AsyncLLM
|
||||
/// and API server on a "per-node" basis where vLLM load balances
|
||||
/// between local data parallel ranks, but an external LB balances
|
||||
/// between vLLM nodes/replicas. Set explicitly in conjunction with
|
||||
/// --data-parallel-start-rank.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-data-parallel-hybrid-lb",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub data_parallel_hybrid_lb: Option<Unsupported>,
|
||||
|
||||
/// Whether to use "external" DP LB mode. Applies only to online serving
|
||||
/// and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"
|
||||
/// wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank
|
||||
/// is provided explicitly to vllm serve.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-data-parallel-external-lb",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub data_parallel_external_lb: Option<Unsupported>,
|
||||
|
||||
/// This feature is work in progress and no prefill optimization takes place
|
||||
/// with this flag enabled currently.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-kv-sharing-fast-prefill",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub kv_sharing_fast_prefill: Option<Unsupported>,
|
||||
|
||||
/// The maximum number of input items and options allowed per
|
||||
/// prompt for each modality.
|
||||
#[arg(long)]
|
||||
pub limit_mm_per_prompt: Option<Unsupported>,
|
||||
|
||||
/// Additional args passed to process media inputs, keyed by modalities.
|
||||
#[arg(long)]
|
||||
pub media_io_kwargs: Option<Unsupported>,
|
||||
|
||||
/// Arguments to be forwarded to the model's processor for multi-modal data,
|
||||
/// e.g., image processor.
|
||||
#[arg(long)]
|
||||
pub mm_processor_kwargs: Option<Unsupported>,
|
||||
|
||||
/// The size (in GiB) of the multi-modal processor cache.
|
||||
#[arg(long)]
|
||||
pub mm_processor_cache_gb: Option<Unsupported>,
|
||||
|
||||
/// Type of cache to use for the multi-modal preprocessor/mapper.
|
||||
#[arg(long)]
|
||||
pub mm_processor_cache_type: Option<Unsupported>,
|
||||
|
||||
/// If True, enable handling of LoRA adapters.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-lora",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_lora: Option<Unsupported>,
|
||||
|
||||
/// Dictionary mapping specific modalities to LoRA model paths.
|
||||
#[arg(long)]
|
||||
pub default_mm_loras: Option<Unsupported>,
|
||||
|
||||
/// Target URL to which OpenTelemetry traces will be sent.
|
||||
#[arg(long)]
|
||||
pub otlp_traces_endpoint: Option<Unsupported>,
|
||||
|
||||
/// It makes sense to set this only if `--otlp-traces-endpoint` is set.
|
||||
#[arg(long)]
|
||||
pub collect_detailed_traces: Option<Unsupported>,
|
||||
|
||||
/// The interval (or buffer size) for streaming in terms of token length.
|
||||
#[arg(long)]
|
||||
pub stream_interval: Option<Unsupported>,
|
||||
|
||||
/// Structured outputs configuration.
|
||||
#[arg(long)]
|
||||
pub structured_outputs_config: Option<Noop>,
|
||||
|
||||
/// Log aggregate rather than per-engine statistics when using data
|
||||
/// parallelism.
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub aggregate_engine_logging: Option<Unsupported>,
|
||||
}
|
||||
|
||||
/// Frontend-owned Python OpenAI server arguments that `vllm-rs` recognizes but
|
||||
/// does not support yet.
|
||||
///
|
||||
/// Source of truth in Python vLLM:
|
||||
/// - `vllm.entrypoints.openai.cli_args.BaseFrontendArgs`
|
||||
/// - `vllm.entrypoints.openai.cli_args.FrontendArgs`
|
||||
///
|
||||
/// These are not engine args. They belong to the Python OpenAI-compatible
|
||||
/// frontend / API-server layer itself, for example chat-template configuration,
|
||||
/// tool/frontend behavior, TLS / CORS / HTTP server settings, and other
|
||||
/// northbound server knobs.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||
pub struct ServerUnsupportedArgs {
|
||||
/// LoRA modules configurations in either 'name=path' format or JSON format
|
||||
/// or JSON list format. Example (old format): `'name=path'` Example (new
|
||||
/// format): `{"name": "name", "path": "lora_path",
|
||||
/// "base_model_name": "id"}`
|
||||
#[arg(long)]
|
||||
pub lora_modules: Option<Unsupported>,
|
||||
|
||||
/// Whether to trust the chat template provided in the request. If False,
|
||||
/// the server will always use the chat template specified by
|
||||
/// `--chat-template` or the ones from tokenizer.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-trust-request-chat-template",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub trust_request_chat_template: Option<Unsupported>,
|
||||
|
||||
/// The role name to return if `request.add_generation_prompt=true`.
|
||||
#[arg(long)]
|
||||
pub response_role: Option<Unsupported>,
|
||||
|
||||
/// When `--max-logprobs` is specified, represents single tokens as
|
||||
/// strings of the form 'token_id:{token_id}' so that tokens that are not
|
||||
/// JSON-encodable can be identified.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-return-tokens-as-token-ids",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub return_tokens_as_token_ids: Option<Unsupported>,
|
||||
|
||||
/// Enable auto tool choice for supported models. Use `--tool-call-parser`
|
||||
/// to specify which parser to use.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-auto-tool-choice",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1,
|
||||
hide = true
|
||||
)]
|
||||
pub enable_auto_tool_choice: Option<Noop>,
|
||||
|
||||
/// If specified, exclude tool definitions in prompts when
|
||||
/// tool_choice='none'.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-exclude-tools-when-tool-choice-none",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub exclude_tools_when_tool_choice_none: Option<Unsupported>,
|
||||
|
||||
/// Special the tool parser plugin write to parse the model-generated tool
|
||||
/// into OpenAI API format, the name register in this plugin can be used in
|
||||
/// `--tool-call-parser`.
|
||||
#[arg(long)]
|
||||
pub tool_parser_plugin: Option<Unsupported>,
|
||||
|
||||
/// Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
/// Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
/// built-in demo tools (browser and Python code interpreter). WARNING:
|
||||
/// The `demo` Python tool executes model-generated code in Docker without
|
||||
/// network isolation by default. See the security guide for more
|
||||
/// information.
|
||||
#[arg(long)]
|
||||
pub tool_server: Option<Unsupported>,
|
||||
|
||||
/// Path to logging config JSON file for both vllm and uvicorn
|
||||
#[arg(long, /* env = "VLLM_LOGGING_CONFIG_PATH" */)]
|
||||
pub log_config_file: Option<Unsupported>,
|
||||
|
||||
/// Max number of prompt characters or prompt ID numbers being printed in
|
||||
/// log. The default of None means unlimited.
|
||||
#[arg(long)]
|
||||
pub max_log_len: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable prompt_tokens_details in usage.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-prompt-tokens-details",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_prompt_tokens_details: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable tracking server_load_metrics in the app state.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-server-load-tracking",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_server_load_tracking: Option<Noop>,
|
||||
|
||||
/// If set to True, including usage on every request.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-force-include-usage",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_force_include_usage: Option<Unsupported>,
|
||||
|
||||
/// Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
/// templates and other tokenizer configuration.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-tokenizer-info-endpoint",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_tokenizer_info_endpoint: Option<Unsupported>,
|
||||
|
||||
/// If set to True, log model outputs (generations).
|
||||
/// Requires `--enable-log-requests`. As with `--enable-log-requests`,
|
||||
/// information is only logged at INFO level at maximum.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-log-outputs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_log_outputs: Option<Unsupported>,
|
||||
|
||||
/// If set to False, output deltas will not be logged. Relevant only if
|
||||
/// --enable-log-outputs is set.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-log-deltas",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_log_deltas: Option<Unsupported>,
|
||||
|
||||
/// If set to True, log the stack trace of error responses
|
||||
#[arg(
|
||||
long,
|
||||
// env = "VLLM_SERVER_DEV_MODE",
|
||||
visible_alias = "no-log-error-stack",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub log_error_stack: Option<Unsupported>,
|
||||
|
||||
/// If set to True, only enable the Tokens In<>Out endpoint.
|
||||
/// This is intended for use in a Disaggregated Everything setup.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-tokens-only",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub tokens_only: Option<Unsupported>,
|
||||
|
||||
/// Log level for uvicorn.
|
||||
#[arg(long)]
|
||||
pub uvicorn_log_level: Option<Unsupported>,
|
||||
|
||||
/// Disable uvicorn access log.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-disable-uvicorn-access-log",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub disable_uvicorn_access_log: Option<Noop>,
|
||||
|
||||
/// Comma-separated list of endpoint paths to exclude from uvicorn access
|
||||
/// logs. This is useful to reduce log noise from high-frequency endpoints
|
||||
/// like health checks. Example: "/health,/metrics,/ping".
|
||||
/// When set, access logs for requests to these paths will be suppressed
|
||||
/// while keeping logs for other endpoints.
|
||||
#[arg(long)]
|
||||
pub disable_access_log_for_endpoints: Option<Noop>,
|
||||
|
||||
/// Allow credentials.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-allow-credentials",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub allow_credentials: Option<Unsupported>,
|
||||
|
||||
/// Allowed origins.
|
||||
#[arg(long)]
|
||||
pub allowed_origins: Option<Unsupported>,
|
||||
|
||||
/// Allowed methods.
|
||||
#[arg(long)]
|
||||
pub allowed_methods: Option<Unsupported>,
|
||||
|
||||
/// Allowed headers.
|
||||
#[arg(long)]
|
||||
pub allowed_headers: Option<Unsupported>,
|
||||
|
||||
/// If provided, the server will require one of these keys to be presented
|
||||
/// in the header.
|
||||
#[arg(long)]
|
||||
pub api_key: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL key file.
|
||||
#[arg(long)]
|
||||
pub ssl_keyfile: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL cert file.
|
||||
#[arg(long)]
|
||||
pub ssl_certfile: Option<Unsupported>,
|
||||
|
||||
/// The CA certificates file.
|
||||
#[arg(long)]
|
||||
pub ssl_ca_certs: Option<Unsupported>,
|
||||
|
||||
/// Refresh SSL Context when SSL certificate files change
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-ssl-refresh",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_ssl_refresh: Option<Unsupported>,
|
||||
|
||||
/// Whether client certificate is required (see stdlib ssl module's).
|
||||
#[arg(long)]
|
||||
pub ssl_cert_reqs: Option<Unsupported>,
|
||||
|
||||
/// SSL cipher suites for HTTPS (TLS 1.2 and below only).
|
||||
/// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
|
||||
#[arg(long)]
|
||||
pub ssl_ciphers: Option<Unsupported>,
|
||||
|
||||
/// FastAPI root_path when app is behind a path based routing proxy.
|
||||
#[arg(long)]
|
||||
pub root_path: Option<Unsupported>,
|
||||
|
||||
/// Additional ASGI middleware to apply to the app. We accept multiple
|
||||
/// --middleware arguments. The value should be an import path. If a
|
||||
/// function is provided, vLLM will add it to the server using
|
||||
/// `@app.middleware('http')`. If a class is provided, vLLM will
|
||||
/// add it to the server using `app.add_middleware()`.
|
||||
#[arg(long)]
|
||||
pub middleware: Option<Unsupported>,
|
||||
|
||||
/// If specified, API server will add X-Request-Id header to responses.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-request-id-headers",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_request_id_headers: Option<Unsupported>,
|
||||
|
||||
/// Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-disable-fastapi-docs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub disable_fastapi_docs: Option<Unsupported>,
|
||||
|
||||
/// Maximum size (bytes) of an incomplete HTTP event (header or body) for
|
||||
/// h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB).
|
||||
#[arg(long)]
|
||||
pub h11_max_incomplete_event_size: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of HTTP headers allowed in a request for h11 parser.
|
||||
/// Helps mitigate header abuse. Default: 256.
|
||||
#[arg(long)]
|
||||
pub h11_max_header_count: Option<Unsupported>,
|
||||
|
||||
/// Enable offline FastAPI documentation for air-gapped environments.
|
||||
/// Uses vendored static assets bundled with vLLM.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-offline-docs",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_offline_docs: Option<Unsupported>,
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
use std::{env, fmt, process};
|
||||
|
||||
use time::UtcOffset;
|
||||
use time::macros::format_description;
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::Layer as _;
|
||||
use tracing_subscriber::filter::Targets;
|
||||
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
|
||||
use tracing_subscriber::fmt::time::FormatTime;
|
||||
use tracing_subscriber::fmt::{FmtContext, FormattedFields};
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
use tracing_subscriber::util::SubscriberInitExt as _;
|
||||
|
||||
const CYAN: &str = "\x1b[0;36m";
|
||||
const GREY: &str = "\x1b[90m";
|
||||
const GREEN: &str = "\x1b[32m";
|
||||
const YELLOW: &str = "\x1b[33m";
|
||||
const RED: &str = "\x1b[31m";
|
||||
const WHITE: &str = "\x1b[37m";
|
||||
const RESET: &str = "\x1b[0m";
|
||||
const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
|
||||
format_description!("[month]-[day] [hour]:[minute]:[second]");
|
||||
|
||||
const PROCESS_LABEL: &str = "RustFrontend";
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
|
||||
pub(crate) fn init_tracing() {
|
||||
let filter = build_targets_filter(
|
||||
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
|
||||
env::var("RUST_LOG").ok().as_deref(),
|
||||
);
|
||||
let formatter = VllmEventFormatter::new();
|
||||
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().event_format(formatter).with_filter(filter))
|
||||
.try_init();
|
||||
}
|
||||
|
||||
/// Build the CLI log filter by merging the vLLM-style default level with
|
||||
/// Rust-style target overrides.
|
||||
///
|
||||
/// Precedence:
|
||||
/// - Start from `VLLM_LOGGING_LEVEL` as the default level for all targets.
|
||||
/// - If `RUST_LOG` contains a global default level such as `warn`, it overrides
|
||||
/// `VLLM_LOGGING_LEVEL`.
|
||||
/// - Any explicit target directives in `RUST_LOG`, such as `hyper=info`, override whichever default
|
||||
/// level is active for those targets only.
|
||||
fn build_targets_filter(vllm_logging_level: Option<&str>, rust_log: Option<&str>) -> Targets {
|
||||
let mut filter =
|
||||
Targets::new().with_default(map_python_log_level(vllm_logging_level.unwrap_or("INFO")));
|
||||
|
||||
if let Some(rust_log) = rust_log
|
||||
&& !rust_log.is_empty()
|
||||
{
|
||||
let rust_log_targets: Targets = rust_log.parse().expect("failed to parse `RUST_LOG`");
|
||||
if let Some(default_level) = rust_log_targets.default_level() {
|
||||
filter = filter.with_default(default_level);
|
||||
}
|
||||
filter = filter.with_targets(rust_log_targets);
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct VllmLocalTimer {
|
||||
local_offset: UtcOffset,
|
||||
}
|
||||
|
||||
impl Default for VllmLocalTimer {
|
||||
fn default() -> Self {
|
||||
let local_offset = UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC);
|
||||
Self { local_offset }
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatTime for VllmLocalTimer {
|
||||
fn format_time(&self, w: &mut Writer<'_>) -> fmt::Result {
|
||||
let now = time::OffsetDateTime::now_utc().to_offset(self.local_offset);
|
||||
let formatted = now.format(VLLM_TIME_FORMAT).map_err(|_| fmt::Error)?;
|
||||
w.write_str(&formatted)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VllmEventFormatter {
|
||||
prefix: String,
|
||||
timer: VllmLocalTimer,
|
||||
}
|
||||
|
||||
impl VllmEventFormatter {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
prefix: format!("({} pid={})", PROCESS_LABEL, process::id()),
|
||||
timer: VllmLocalTimer::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_process_prefix(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
|
||||
write_colored(writer, ansi, Some(CYAN), &self.prefix)?;
|
||||
writer.write_char(' ')
|
||||
}
|
||||
|
||||
fn write_level(&self, writer: &mut Writer<'_>, level: &Level, ansi: bool) -> fmt::Result {
|
||||
let (text, color) = match *level {
|
||||
Level::TRACE => ("TRACE", WHITE),
|
||||
Level::DEBUG => ("DEBUG", WHITE),
|
||||
Level::INFO => ("INFO", GREEN),
|
||||
Level::WARN => ("WARNING", YELLOW),
|
||||
Level::ERROR => ("ERROR", RED),
|
||||
};
|
||||
write_colored(writer, ansi, Some(color), text)
|
||||
}
|
||||
|
||||
fn write_timestamp(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
|
||||
if ansi {
|
||||
writer.write_str(GREY)?;
|
||||
}
|
||||
if self.timer.format_time(writer).is_err() {
|
||||
writer.write_str("<unknown time>")?;
|
||||
}
|
||||
if ansi {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_location(
|
||||
&self,
|
||||
writer: &mut Writer<'_>,
|
||||
file: Option<&str>,
|
||||
line: Option<u32>,
|
||||
full_path: bool,
|
||||
ansi: bool,
|
||||
) -> fmt::Result {
|
||||
let Some(file) = file else {
|
||||
return Ok(());
|
||||
};
|
||||
let file = if full_path {
|
||||
file
|
||||
} else {
|
||||
shorten_file_path(file)
|
||||
};
|
||||
if ansi {
|
||||
writer.write_str(GREY)?;
|
||||
}
|
||||
match line {
|
||||
Some(line) => write!(writer, "[{file}:{line}]")?,
|
||||
None => write!(writer, "[{file}]")?,
|
||||
}
|
||||
if ansi {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_scope<S, N>(&self, ctx: &FmtContext<'_, S, N>, writer: &mut Writer<'_>) -> fmt::Result
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
N: for<'writer> FormatFields<'writer> + 'static,
|
||||
{
|
||||
let Some(scope) = ctx.event_scope() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut seen = false;
|
||||
for span in scope.from_root() {
|
||||
if seen {
|
||||
writer.write_str(":")?;
|
||||
}
|
||||
seen = true;
|
||||
writer.write_str(span.metadata().name())?;
|
||||
|
||||
let ext = span.extensions();
|
||||
if let Some(fields) = ext.get::<FormattedFields<N>>()
|
||||
&& !fields.is_empty()
|
||||
{
|
||||
write!(writer, "{{{fields}}}")?;
|
||||
}
|
||||
}
|
||||
|
||||
if seen {
|
||||
writer.write_str(": ")?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for VllmEventFormatter
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
N: for<'writer> FormatFields<'writer> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
let meta = event.metadata();
|
||||
let ansi = writer.has_ansi_escapes();
|
||||
|
||||
self.write_process_prefix(&mut writer, ansi)?;
|
||||
self.write_level(&mut writer, meta.level(), ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
self.write_timestamp(&mut writer, ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
// Use the full file path only when DEBUG (or more verbose) is enabled anywhere,
|
||||
// independent of the level of this particular event. Filenames alone are often
|
||||
// ambiguous, but full paths are too noisy for normal INFO-level operation.
|
||||
let full_path = LevelFilter::current() >= LevelFilter::DEBUG;
|
||||
self.write_location(&mut writer, meta.file(), meta.line(), full_path, ansi)?;
|
||||
writer.write_char(' ')?;
|
||||
self.write_scope(ctx, &mut writer)?;
|
||||
ctx.format_fields(writer.by_ref(), event)?;
|
||||
writer.write_char('\n')
|
||||
}
|
||||
}
|
||||
|
||||
/// Shorten a source file path for log output while preserving enough context
|
||||
/// for common Rust entrypoint and module filenames.
|
||||
///
|
||||
/// - For `mod.rs`, keep the parent directory as `parent/mod.rs`.
|
||||
/// - For `src/lib.rs` and `src/main.rs`, keep one additional component as `crate/src/lib.rs` or
|
||||
/// `crate/src/main.rs` when available.
|
||||
/// - Other files are displayed as just the basename.
|
||||
fn shorten_file_path(file: &str) -> &str {
|
||||
let mut parts = file.rsplit('/');
|
||||
let name = parts.next().unwrap_or(file);
|
||||
let parent = parts.next();
|
||||
let grandparent = parts.next();
|
||||
|
||||
let Some(parent) = parent else {
|
||||
return file;
|
||||
};
|
||||
|
||||
if name == "mod.rs" {
|
||||
return &file[file.len() - parent.len() - 1 - name.len()..];
|
||||
}
|
||||
|
||||
if !matches!(name, "lib.rs" | "main.rs") || parent != "src" {
|
||||
return name;
|
||||
}
|
||||
let Some(grandparent) = grandparent else {
|
||||
return file;
|
||||
};
|
||||
|
||||
&file[file.len() - grandparent.len() - 1 - parent.len() - 1 - name.len()..]
|
||||
}
|
||||
|
||||
fn write_colored(
|
||||
writer: &mut Writer<'_>,
|
||||
ansi: bool,
|
||||
color: Option<&str>,
|
||||
text: &str,
|
||||
) -> fmt::Result {
|
||||
if ansi {
|
||||
if let Some(color) = color {
|
||||
writer.write_str(color)?;
|
||||
}
|
||||
writer.write_str(text)?;
|
||||
if color.is_some() {
|
||||
writer.write_str(RESET)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
writer.write_str(text)
|
||||
}
|
||||
|
||||
/// Map a Python logging level name to the corresponding Rust tracing level.
|
||||
fn map_python_log_level(level: &str) -> LevelFilter {
|
||||
match level.to_ascii_uppercase().as_str() {
|
||||
"CRITICAL" | "FATAL" => LevelFilter::ERROR,
|
||||
"ERROR" => LevelFilter::ERROR,
|
||||
"WARNING" | "WARN" => LevelFilter::WARN,
|
||||
"INFO" => LevelFilter::INFO,
|
||||
"DEBUG" => LevelFilter::DEBUG,
|
||||
"NOTSET" => LevelFilter::TRACE,
|
||||
_ => LevelFilter::INFO,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rust_log_target_overrides_are_merged_with_vllm_default_level() {
|
||||
let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error"));
|
||||
|
||||
assert_eq!(filter.to_string(), "tower=error,hyper=warn,debug");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rust_log_default_level_overrides_vllm_default_level() {
|
||||
let filter = build_targets_filter(Some("DEBUG"), Some("warn,hyper=info"));
|
||||
|
||||
assert_eq!(filter.to_string(), "hyper=info,warn");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_vllm_level_falls_back_to_info() {
|
||||
let filter = build_targets_filter(Some("bogus"), None);
|
||||
|
||||
assert_eq!(filter.to_string(), "info");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_path_uses_filename_for_non_ambiguous_files() {
|
||||
assert_eq!(shorten_file_path("src/cmd/src/logging.rs"), "logging.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/lib.rs"), "lib.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/main.rs"), "main.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/xmod.rs"), "xmod.rs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_path_keeps_more_context_for_common_entrypoint_filenames() {
|
||||
assert_eq!(shorten_file_path("src/lib.rs"), "src/lib.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/lib.rs"), "chat/src/lib.rs");
|
||||
assert_eq!(shorten_file_path("src/cmd/src/main.rs"), "cmd/src/main.rs");
|
||||
assert_eq!(shorten_file_path("mod.rs"), "mod.rs");
|
||||
assert_eq!(shorten_file_path("src/chat/src/tool/mod.rs"), "tool/mod.rs");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
mod cli;
|
||||
mod logging;
|
||||
|
||||
use std::env;
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use vllm_managed_engine::ManagedEngineHandle;
|
||||
|
||||
use crate::cli::{Cli, Command};
|
||||
|
||||
const TOKIO_WORKER_THREADS_ENV: &str = "TOKIO_WORKER_THREADS";
|
||||
const DEFAULT_MAX_TOKIO_WORKER_THREADS: usize = 32;
|
||||
|
||||
/// Cap the default number of Tokio worker threads if the user did not
|
||||
/// explicitly set `TOKIO_WORKER_THREADS` to avoid spawning too many threads on
|
||||
/// machines with a large number of CPUs, which may lead to excessive context
|
||||
/// switching and degraded performance.
|
||||
fn tokio_worker_threads() -> Option<usize> {
|
||||
if env::var_os(TOKIO_WORKER_THREADS_ENV).is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
std::thread::available_parallelism()
|
||||
.map(|parallelism| {
|
||||
let available = parallelism.get();
|
||||
let worker_threads = available.min(DEFAULT_MAX_TOKIO_WORKER_THREADS);
|
||||
if worker_threads < available {
|
||||
info!(
|
||||
available_parallelism = available,
|
||||
capped_worker_threads = worker_threads,
|
||||
"capping tokio worker threads, set {TOKIO_WORKER_THREADS_ENV} to override"
|
||||
);
|
||||
}
|
||||
worker_threads
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Reason that caused a managed `serve` session to stop.
|
||||
#[derive(Debug)]
|
||||
enum ShutdownReason {
|
||||
Signal,
|
||||
Server(anyhow::Error),
|
||||
EngineExited(ExitStatus),
|
||||
}
|
||||
|
||||
/// Cancellation token tripped by Ctrl-C or SIGTERM.
|
||||
fn shutdown_signal() -> CancellationToken {
|
||||
let token = CancellationToken::new();
|
||||
let shutdown = token.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let ctrl_c = async {
|
||||
tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
|
||||
};
|
||||
|
||||
let sigterm = async {
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("failed to install SIGTERM signal handler")
|
||||
.recv()
|
||||
.await;
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
_ = ctrl_c => info!("received shutdown signal (Ctrl-C), shutting down..."),
|
||||
_ = sigterm => info!("received shutdown signal (SIGTERM), shutting down..."),
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
});
|
||||
|
||||
token
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
logging::init_tracing();
|
||||
let cli = Cli::parse();
|
||||
|
||||
let mut runtime = tokio::runtime::Builder::new_multi_thread();
|
||||
runtime.enable_all();
|
||||
if let Some(worker_threads) = tokio_worker_threads() {
|
||||
runtime.worker_threads(worker_threads);
|
||||
}
|
||||
|
||||
runtime
|
||||
.build()
|
||||
.context("failed to build Tokio runtime")?
|
||||
.block_on(async_main(cli))
|
||||
}
|
||||
|
||||
async fn async_main(cli: Cli) -> Result<()> {
|
||||
match cli.command {
|
||||
Command::Frontend(args) => vllm_server::serve(args.into_config(), shutdown_signal()).await,
|
||||
Command::Serve(args) => {
|
||||
let handshake_port = args.managed_engine.resolve_handshake_port()?;
|
||||
|
||||
if args.managed_engine.data_parallel_size_local == Some(0) {
|
||||
if args.headless {
|
||||
bail!("cannot combine `--headless` with `--data-parallel-size-local 0`");
|
||||
}
|
||||
|
||||
let handshake_address = args.managed_engine.handshake_address(handshake_port);
|
||||
info!(
|
||||
%handshake_address,
|
||||
engine_count = args.managed_engine.data_parallel_size,
|
||||
"running Rust frontend without a managed local Python engine"
|
||||
);
|
||||
let config = args.to_frontend_config(handshake_address);
|
||||
return vllm_server::serve(config, shutdown_signal()).await;
|
||||
}
|
||||
|
||||
let shutdown_timeout = args.runtime.shutdown_timeout();
|
||||
let engine_config = args.to_managed_engine_config(handshake_port);
|
||||
let handshake_address = engine_config.handshake_address();
|
||||
|
||||
let engine = ManagedEngineHandle::spawn(engine_config)
|
||||
.await
|
||||
.context("failed to start managed Python headless engine")?;
|
||||
|
||||
let shutdown = shutdown_signal();
|
||||
|
||||
let mut serve_task = if args.headless {
|
||||
info!("running managed Python headless engine without Rust frontend");
|
||||
let shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
shutdown.cancelled().await;
|
||||
Ok(())
|
||||
})
|
||||
} else {
|
||||
let config = args.to_frontend_config(handshake_address);
|
||||
let shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = vllm_server::serve(config, shutdown).await;
|
||||
if result.is_ok() {
|
||||
info!("OpenAI server shut down gracefully");
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
let shutdown_reason = tokio::select! {
|
||||
biased;
|
||||
|
||||
// Received shutdown signal via Ctrl-C or SIGTERM.
|
||||
_ = shutdown.cancelled() => ShutdownReason::Signal,
|
||||
|
||||
// Engine process exited unexpectedly.
|
||||
status = engine.wait_for_exit() => {
|
||||
warn!(%status, "managed Python headless engine exited, shutting down...");
|
||||
ShutdownReason::EngineExited(status)
|
||||
}
|
||||
|
||||
// Serve task exited unexpectedly.
|
||||
serve_result = &mut serve_task => {
|
||||
let serve_result = serve_result.context("serve task join failed")?;
|
||||
match serve_result {
|
||||
Ok(()) => ShutdownReason::Server(anyhow!("OpenAI server shut down unexpectedly without error")),
|
||||
Err(error) => ShutdownReason::Server(error),
|
||||
}
|
||||
}
|
||||
};
|
||||
// Regardless of the shutdown reason, broadcast shutdown signal here to ensure
|
||||
// that all serving tasks are notified.
|
||||
shutdown.cancel();
|
||||
|
||||
// Shutdown begins. Terminate the managed engine first.
|
||||
engine.shutdown(shutdown_timeout).await?;
|
||||
info!("managed engine shut down gracefully");
|
||||
// Wait for the API server to shut down gracefully by draining in-flight
|
||||
// requests.
|
||||
if !matches!(shutdown_reason, ShutdownReason::Server(_)) {
|
||||
serve_task.await.context("serve task join failed")??;
|
||||
}
|
||||
|
||||
match shutdown_reason {
|
||||
ShutdownReason::Signal => Ok(()),
|
||||
ShutdownReason::Server(error) => {
|
||||
Err(error.context("OpenAI server shut down unexpectedly"))
|
||||
}
|
||||
ShutdownReason::EngineExited(status) => Err(anyhow!(
|
||||
"managed Python headless engine exited unexpectedly with status {status}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user