[Rust Frontend] Add /server_info to Rust frontend (#43942)
Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -6,10 +6,10 @@ use std::convert::Infallible;
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde_with::DeserializeFromStr;
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
|
||||
/// Specify which reasoning or tool-call parser implementation to use.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
|
||||
pub enum ParserSelection {
|
||||
/// Use model-based auto-detection.
|
||||
#[default]
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::str::FromStr;
|
||||
use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt};
|
||||
use minijinja::machinery::{WhitespaceConfig, parse};
|
||||
use minijinja::syntax::SyntaxConfig;
|
||||
use serde_with::DeserializeFromStr;
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
|
||||
/// Chat template content format.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -18,7 +18,7 @@ pub enum ChatTemplateContentFormat {
|
||||
}
|
||||
|
||||
/// Configurable chat-template content format selection.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
|
||||
pub enum ChatTemplateContentFormatOption {
|
||||
/// Detect the format from the template source.
|
||||
#[default]
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde_with::DeserializeFromStr;
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
|
||||
/// Specify which chat renderer implementation to use.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
|
||||
pub enum RendererSelection {
|
||||
/// Use model-based auto-detection.
|
||||
#[default]
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{join_all, try_join_all};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, info, trace};
|
||||
@@ -23,7 +24,7 @@ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
|
||||
|
||||
/// How the frontend acquires its request/response transport with Python
|
||||
/// `EngineCoreProc`s.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub enum TransportMode {
|
||||
/// The Rust process owns the startup handshake and allocates or binds the
|
||||
/// frontend transport addresses itself before replying to engine
|
||||
|
||||
@@ -2,12 +2,13 @@ use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
|
||||
/// How the HTTP server obtains its listening socket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub enum HttpListenerMode {
|
||||
/// Bind a fresh TCP listener on the given host/port.
|
||||
BindTcp { host: String, port: u16 },
|
||||
@@ -20,7 +21,7 @@ pub enum HttpListenerMode {
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub enum CoordinatorMode {
|
||||
/// Do not run a coordinator at all.
|
||||
None,
|
||||
@@ -32,7 +33,7 @@ pub enum CoordinatorMode {
|
||||
}
|
||||
|
||||
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct Config {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
|
||||
@@ -7,6 +7,7 @@ mod listener;
|
||||
mod lora;
|
||||
mod middleware;
|
||||
mod routes;
|
||||
mod server_info;
|
||||
mod state;
|
||||
mod utils;
|
||||
|
||||
@@ -30,6 +31,7 @@ use vllm_text::TextLlm;
|
||||
|
||||
use crate::listener::Listener;
|
||||
use crate::routes::build_router;
|
||||
use crate::server_info::ServerInfoSnapshot;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Build the shared application state for one configured model and one engine
|
||||
@@ -88,7 +90,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
Ok(Arc::new(
|
||||
AppState::new(served_model_names, chat)
|
||||
.with_log_requests(config.enable_log_requests)
|
||||
.with_request_id_headers(config.enable_request_id_headers),
|
||||
.with_request_id_headers(config.enable_request_id_headers)
|
||||
.with_server_info(ServerInfoSnapshot::from_config(config)),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ mod load;
|
||||
mod lora;
|
||||
mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod server_info;
|
||||
mod sleep;
|
||||
mod version;
|
||||
|
||||
@@ -89,6 +90,7 @@ fn build_router_with_options(
|
||||
.route("/sleep", post(sleep::sleep))
|
||||
.route("/wake_up", post(sleep::wake_up))
|
||||
.route("/is_sleeping", get(sleep::is_sleeping))
|
||||
.route("/server_info", get(server_info::server_info))
|
||||
}
|
||||
|
||||
let enable_request_id_headers = state.enable_request_id_headers;
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::server_info::ServerInfoConfigFormat;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum ConfigFormat {
|
||||
Text,
|
||||
Json,
|
||||
}
|
||||
|
||||
impl From<ConfigFormat> for ServerInfoConfigFormat {
|
||||
fn from(value: ConfigFormat) -> Self {
|
||||
match value {
|
||||
ConfigFormat::Text => Self::Text,
|
||||
ConfigFormat::Json => Self::Json,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_config_format() -> ConfigFormat {
|
||||
ConfigFormat::Text
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ServerInfoParams {
|
||||
#[serde(default = "default_config_format")]
|
||||
config_format: ConfigFormat,
|
||||
}
|
||||
|
||||
/// Get server configuration and environment metadata.
|
||||
pub async fn server_info(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<ServerInfoParams>,
|
||||
) -> Response {
|
||||
match state.server_info_response(params.config_format.into()) {
|
||||
Some(response) => Json(response).into_response(),
|
||||
None => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
@@ -742,16 +742,23 @@ async fn test_chat_with_engine_outputs(
|
||||
}
|
||||
|
||||
async fn test_app() -> axum::Router {
|
||||
test_app_with_dev_mode(false).await
|
||||
}
|
||||
|
||||
async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)))
|
||||
build_router_with_dev_mode(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
dev_mode_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
|
||||
@@ -1087,6 +1094,23 @@ async fn version_returns_engine_vllm_version() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn server_info_endpoint_is_dev_mode_only() {
|
||||
let mut app = test_app().await;
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/server_info")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
|
||||
@@ -1302,6 +1326,23 @@ async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn server_info_endpoint_returns_not_found_without_snapshot() {
|
||||
let mut app = test_app_with_dev_mode(true).await;
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/server_info")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::config::Config;
|
||||
|
||||
const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] =
|
||||
&["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"];
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ServerInfoConfigFormat {
|
||||
Text,
|
||||
Json,
|
||||
}
|
||||
|
||||
/// Snapshot returned by `/server_info`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ServerInfoSnapshot {
|
||||
vllm_config_text: String,
|
||||
vllm_config_json: Value,
|
||||
vllm_env: BTreeMap<String, String>,
|
||||
system_env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl ServerInfoSnapshot {
|
||||
/// Capture the runtime configuration fields available to the Rust frontend.
|
||||
pub(crate) fn from_config(config: &Config) -> Self {
|
||||
let vllm_config_json =
|
||||
serde_json::to_value(config).expect("server info value must serialize");
|
||||
|
||||
Self {
|
||||
vllm_config_text: render_config_text(&vllm_config_json),
|
||||
vllm_config_json,
|
||||
vllm_env: collect_vllm_env(),
|
||||
system_env: collect_system_env(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value {
|
||||
let vllm_config = match config_format {
|
||||
ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()),
|
||||
ServerInfoConfigFormat::Json => self.vllm_config_json.clone(),
|
||||
};
|
||||
|
||||
json!({
|
||||
"vllm_config": vllm_config,
|
||||
"vllm_env": self.vllm_env.clone(),
|
||||
"system_env": self.system_env.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn render_config_text(config: &Value) -> String {
|
||||
match config {
|
||||
Value::Object(fields) => fields
|
||||
.iter()
|
||||
.map(|(key, value)| format!("{key}={}", render_config_text_value(value)))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
_ => render_config_text_value(config),
|
||||
}
|
||||
}
|
||||
|
||||
fn render_config_text_value(value: &Value) -> String {
|
||||
match value {
|
||||
Value::Null => "None".to_string(),
|
||||
Value::String(value) => value.clone(),
|
||||
_ => value.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_vllm_env() -> BTreeMap<String, String> {
|
||||
std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect()
|
||||
}
|
||||
|
||||
fn is_public_vllm_env_key(key: &str) -> bool {
|
||||
let key = key.to_ascii_uppercase();
|
||||
key.starts_with("VLLM_")
|
||||
&& !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern))
|
||||
}
|
||||
|
||||
fn collect_system_env() -> BTreeMap<String, String> {
|
||||
BTreeMap::from([
|
||||
("arch".to_string(), std::env::consts::ARCH.to_string()),
|
||||
("family".to_string(), std::env::consts::FAMILY.to_string()),
|
||||
("os".to_string(), std::env::consts::OS.to_string()),
|
||||
])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{is_public_vllm_env_key, render_config_text};
|
||||
|
||||
#[test]
|
||||
fn render_config_text_formats_config_snapshot() {
|
||||
let rendered = render_config_text(&json!({
|
||||
"model": "test-model",
|
||||
"served_model_name": ["served-model"],
|
||||
"chat_template": null,
|
||||
"enable_log_requests": true,
|
||||
}));
|
||||
let lines = rendered.lines().collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(
|
||||
lines,
|
||||
BTreeSet::from([
|
||||
"chat_template=None",
|
||||
"enable_log_requests=true",
|
||||
"model=test-model",
|
||||
"served_model_name=[\"served-model\"]",
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
render_config_text(&Value::String("inline".to_string())),
|
||||
"inline"
|
||||
);
|
||||
assert_eq!(render_config_text(&Value::Null), "None");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_env_filter_excludes_sensitive_vllm_keys() {
|
||||
for key in [
|
||||
"VLLM_API_KEY",
|
||||
"VLLM_AUTH_TOKEN",
|
||||
"VLLM_SECRET",
|
||||
"VLLM_PASSWORD",
|
||||
"VLLM_CREDENTIAL_FILE",
|
||||
"vllm_token",
|
||||
] {
|
||||
assert!(!is_public_vllm_env_key(key), "{key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_env_filter_includes_public_vllm_keys() {
|
||||
assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL"));
|
||||
assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE"));
|
||||
assert!(!is_public_vllm_env_key("OTHER_ENV"));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use serde_json::Value;
|
||||
use tokio::time::{Duration, Instant, sleep_until};
|
||||
use tracing::warn;
|
||||
use vllm_chat::ChatLlm;
|
||||
@@ -9,6 +10,8 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
|
||||
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
|
||||
|
||||
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
|
||||
|
||||
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Shared router state for the minimal single-model OpenAI server.
|
||||
@@ -22,6 +25,8 @@ pub struct AppState {
|
||||
pub enable_log_requests: bool,
|
||||
/// Whether to set X-Request-Id on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
/// Runtime server information returned by `/server_info`, when available.
|
||||
server_info: Option<ServerInfoSnapshot>,
|
||||
/// Number of in-flight inference requests currently owned by this frontend.
|
||||
server_load: AtomicU64,
|
||||
/// Dynamic LoRA adapter registry.
|
||||
@@ -47,6 +52,7 @@ impl AppState {
|
||||
chat,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
server_info: None,
|
||||
server_load: AtomicU64::new(0),
|
||||
lora_manager: LoraManager::new(),
|
||||
}
|
||||
@@ -64,6 +70,20 @@ impl AppState {
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the runtime server information snapshot used by `/server_info`.
|
||||
pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
|
||||
self.server_info = Some(server_info);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build a `/server_info` response payload.
|
||||
pub(crate) fn server_info_response(
|
||||
&self,
|
||||
config_format: ServerInfoConfigFormat,
|
||||
) -> Option<Value> {
|
||||
self.server_info.as_ref().map(|server_info| server_info.response(config_format))
|
||||
}
|
||||
|
||||
/// The primary model name echoed back in API responses (the first served
|
||||
/// name).
|
||||
pub fn primary_model_name(&self) -> &str {
|
||||
|
||||
Reference in New Issue
Block a user