[Rust Frontend] Add /server_info to Rust frontend (#43942)

Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Xunzhuo
2026-06-03 04:30:47 -07:00
committed by GitHub
co-authored by Bugen Zhao
parent 3d76f395e3
commit 309385a359
11 changed files with 274 additions and 15 deletions
+2 -2
View File
@@ -6,10 +6,10 @@ use std::convert::Infallible;
use std::fmt;
use std::str::FromStr;
use serde_with::DeserializeFromStr;
use serde_with::{DeserializeFromStr, SerializeDisplay};
/// Specify which reasoning or tool-call parser implementation to use.
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)]
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
pub enum ParserSelection {
/// Use model-based auto-detection.
#[default]
+2 -2
View File
@@ -5,7 +5,7 @@ use std::str::FromStr;
use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt};
use minijinja::machinery::{WhitespaceConfig, parse};
use minijinja::syntax::SyntaxConfig;
use serde_with::DeserializeFromStr;
use serde_with::{DeserializeFromStr, SerializeDisplay};
/// Chat template content format.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -18,7 +18,7 @@ pub enum ChatTemplateContentFormat {
}
/// Configurable chat-template content format selection.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
pub enum ChatTemplateContentFormatOption {
/// Detect the format from the template source.
#[default]
+2 -2
View File
@@ -1,10 +1,10 @@
use std::fmt;
use std::str::FromStr;
use serde_with::DeserializeFromStr;
use serde_with::{DeserializeFromStr, SerializeDisplay};
/// Specify which chat renderer implementation to use.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
pub enum RendererSelection {
/// Use model-based auto-detection.
#[default]
+2 -1
View File
@@ -2,6 +2,7 @@ use std::sync::Arc;
use std::time::Duration;
use futures::future::{join_all, try_join_all};
use serde::Serialize;
use tokio::sync::mpsc;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, info, trace};
@@ -23,7 +24,7 @@ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
/// How the frontend acquires its request/response transport with Python
/// `EngineCoreProc`s.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum TransportMode {
/// The Rust process owns the startup handshake and allocates or binds the
/// frontend transport addresses itself before replying to engine
+4 -3
View File
@@ -2,12 +2,13 @@ use std::collections::HashMap;
use std::time::Duration;
use anyhow::Result;
use serde::Serialize;
use serde_json::Value;
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
/// How the HTTP server obtains its listening socket.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum HttpListenerMode {
/// Bind a fresh TCP listener on the given host/port.
BindTcp { host: String, port: u16 },
@@ -20,7 +21,7 @@ pub enum HttpListenerMode {
/// Which coordinator implementation should be active when one is present for a
/// frontend client.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum CoordinatorMode {
/// Do not run a coordinator at all.
None,
@@ -32,7 +33,7 @@ pub enum CoordinatorMode {
}
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Config {
/// Frontend-to-engine transport setup.
pub transport_mode: TransportMode,
+4 -1
View File
@@ -7,6 +7,7 @@ mod listener;
mod lora;
mod middleware;
mod routes;
mod server_info;
mod state;
mod utils;
@@ -30,6 +31,7 @@ use vllm_text::TextLlm;
use crate::listener::Listener;
use crate::routes::build_router;
use crate::server_info::ServerInfoSnapshot;
use crate::state::AppState;
/// Build the shared application state for one configured model and one engine
@@ -88,7 +90,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
Ok(Arc::new(
AppState::new(served_model_names, chat)
.with_log_requests(config.enable_log_requests)
.with_request_id_headers(config.enable_request_id_headers),
.with_request_id_headers(config.enable_request_id_headers)
.with_server_info(ServerInfoSnapshot::from_config(config)),
))
}
+2
View File
@@ -6,6 +6,7 @@ mod load;
mod lora;
mod metrics;
pub(crate) mod openai;
mod server_info;
mod sleep;
mod version;
@@ -89,6 +90,7 @@ fn build_router_with_options(
.route("/sleep", post(sleep::sleep))
.route("/wake_up", post(sleep::wake_up))
.route("/is_sleeping", get(sleep::is_sleeping))
.route("/server_info", get(server_info::server_info))
}
let enable_request_id_headers = state.enable_request_id_headers;
+47
View File
@@ -0,0 +1,47 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use serde::Deserialize;
use crate::server_info::ServerInfoConfigFormat;
use crate::state::AppState;
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(rename_all = "lowercase")]
enum ConfigFormat {
Text,
Json,
}
impl From<ConfigFormat> for ServerInfoConfigFormat {
fn from(value: ConfigFormat) -> Self {
match value {
ConfigFormat::Text => Self::Text,
ConfigFormat::Json => Self::Json,
}
}
}
fn default_config_format() -> ConfigFormat {
ConfigFormat::Text
}
#[derive(Debug, Deserialize)]
pub(crate) struct ServerInfoParams {
#[serde(default = "default_config_format")]
config_format: ConfigFormat,
}
/// Get server configuration and environment metadata.
pub async fn server_info(
State(state): State<Arc<AppState>>,
Query(params): Query<ServerInfoParams>,
) -> Response {
match state.server_info_response(params.config_format.into()) {
Some(response) => Json(response).into_response(),
None => StatusCode::NOT_FOUND.into_response(),
}
}
+45 -4
View File
@@ -742,16 +742,23 @@ async fn test_chat_with_engine_outputs(
}
async fn test_app() -> axum::Router {
test_app_with_dev_mode(false).await
}
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai",
default_stream_output_specs(),
Arc::new(FakeChatBackend::new()),
)
.await;
build_router(Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)))
build_router_with_dev_mode(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
dev_mode_enabled,
)
}
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
@@ -1087,6 +1094,23 @@ async fn version_returns_engine_vllm_version() {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn server_info_endpoint_is_dev_mode_only() {
let mut app = test_app().await;
let response = app
.call(
Request::builder()
.uri("/server_info")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
@@ -1302,6 +1326,23 @@ async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn server_info_endpoint_returns_not_found_without_snapshot() {
let mut app = test_app_with_dev_mode(true).await;
let response = app
.call(
Request::builder()
.uri("/server_info")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
+144
View File
@@ -0,0 +1,144 @@
use std::collections::BTreeMap;
use serde_json::{Value, json};
use crate::config::Config;
const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] =
&["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ServerInfoConfigFormat {
Text,
Json,
}
/// Snapshot returned by `/server_info`.
#[derive(Debug, Clone)]
pub(crate) struct ServerInfoSnapshot {
vllm_config_text: String,
vllm_config_json: Value,
vllm_env: BTreeMap<String, String>,
system_env: BTreeMap<String, String>,
}
impl ServerInfoSnapshot {
/// Capture the runtime configuration fields available to the Rust frontend.
pub(crate) fn from_config(config: &Config) -> Self {
let vllm_config_json =
serde_json::to_value(config).expect("server info value must serialize");
Self {
vllm_config_text: render_config_text(&vllm_config_json),
vllm_config_json,
vllm_env: collect_vllm_env(),
system_env: collect_system_env(),
}
}
pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value {
let vllm_config = match config_format {
ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()),
ServerInfoConfigFormat::Json => self.vllm_config_json.clone(),
};
json!({
"vllm_config": vllm_config,
"vllm_env": self.vllm_env.clone(),
"system_env": self.system_env.clone(),
})
}
}
fn render_config_text(config: &Value) -> String {
match config {
Value::Object(fields) => fields
.iter()
.map(|(key, value)| format!("{key}={}", render_config_text_value(value)))
.collect::<Vec<_>>()
.join("\n"),
_ => render_config_text_value(config),
}
}
fn render_config_text_value(value: &Value) -> String {
match value {
Value::Null => "None".to_string(),
Value::String(value) => value.clone(),
_ => value.to_string(),
}
}
fn collect_vllm_env() -> BTreeMap<String, String> {
std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect()
}
fn is_public_vllm_env_key(key: &str) -> bool {
let key = key.to_ascii_uppercase();
key.starts_with("VLLM_")
&& !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern))
}
fn collect_system_env() -> BTreeMap<String, String> {
BTreeMap::from([
("arch".to_string(), std::env::consts::ARCH.to_string()),
("family".to_string(), std::env::consts::FAMILY.to_string()),
("os".to_string(), std::env::consts::OS.to_string()),
])
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use serde_json::{Value, json};
use super::{is_public_vllm_env_key, render_config_text};
#[test]
fn render_config_text_formats_config_snapshot() {
let rendered = render_config_text(&json!({
"model": "test-model",
"served_model_name": ["served-model"],
"chat_template": null,
"enable_log_requests": true,
}));
let lines = rendered.lines().collect::<BTreeSet<_>>();
assert_eq!(
lines,
BTreeSet::from([
"chat_template=None",
"enable_log_requests=true",
"model=test-model",
"served_model_name=[\"served-model\"]",
])
);
assert_eq!(
render_config_text(&Value::String("inline".to_string())),
"inline"
);
assert_eq!(render_config_text(&Value::Null), "None");
}
#[test]
fn server_info_env_filter_excludes_sensitive_vllm_keys() {
for key in [
"VLLM_API_KEY",
"VLLM_AUTH_TOKEN",
"VLLM_SECRET",
"VLLM_PASSWORD",
"VLLM_CREDENTIAL_FILE",
"vllm_token",
] {
assert!(!is_public_vllm_env_key(key), "{key}");
}
}
#[test]
fn server_info_env_filter_includes_public_vllm_keys() {
assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL"));
assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE"));
assert!(!is_public_vllm_env_key("OTHER_ENV"));
}
}
+20
View File
@@ -1,6 +1,7 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use serde_json::Value;
use tokio::time::{Duration, Instant, sleep_until};
use tracing::warn;
use vllm_chat::ChatLlm;
@@ -9,6 +10,8 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// Shared router state for the minimal single-model OpenAI server.
@@ -22,6 +25,8 @@ pub struct AppState {
pub enable_log_requests: bool,
/// Whether to set X-Request-Id on every HTTP response.
pub enable_request_id_headers: bool,
/// Runtime server information returned by `/server_info`, when available.
server_info: Option<ServerInfoSnapshot>,
/// Number of in-flight inference requests currently owned by this frontend.
server_load: AtomicU64,
/// Dynamic LoRA adapter registry.
@@ -47,6 +52,7 @@ impl AppState {
chat,
enable_log_requests: false,
enable_request_id_headers: false,
server_info: None,
server_load: AtomicU64::new(0),
lora_manager: LoraManager::new(),
}
@@ -64,6 +70,20 @@ impl AppState {
self
}
/// Attach the runtime server information snapshot used by `/server_info`.
pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
self.server_info = Some(server_info);
self
}
/// Build a `/server_info` response payload.
pub(crate) fn server_info_response(
&self,
config_format: ServerInfoConfigFormat,
) -> Option<Value> {
self.server_info.as_ref().map(|server_info| server_info.response(config_format))
}
/// The primary model name echoed back in API responses (the first served
/// name).
pub fn primary_model_name(&self) -> &str {