From 7299e6509ef8b9d27e86c4f2315e1ec5628ca426 Mon Sep 17 00:00:00 2001 From: Tahsin Tunan Date: Thu, 18 Jun 2026 16:29:21 +0600 Subject: [PATCH] [Rust Frontend] Return model metadata fields in /v1/models (#45950) Signed-off-by: Tahsin Tunan --- rust/Cargo.lock | 1 + rust/src/server/Cargo.toml | 1 + rust/src/server/src/lib.rs | 1 + rust/src/server/src/lora.rs | 19 ++-- rust/src/server/src/routes/openai/models.rs | 43 ++++++--- .../server/src/routes/openai/utils/types.rs | 6 ++ rust/src/server/src/routes/tests.rs | 87 +++++++++++++++++++ rust/src/server/src/state.rs | 21 ++++- 8 files changed, 153 insertions(+), 26 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0369dc8d94..60aa6c1241 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5880,6 +5880,7 @@ dependencies = [ "expect-test", "futures", "http-body", + "indexmap 2.13.0", "itertools 0.14.0", "libc", "llm-multimodal", diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index cb62f3376b..40f59675a6 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -11,6 +11,7 @@ axum.workspace = true educe.workspace = true futures.workspace = true http-body.workspace = true +indexmap.workspace = true itertools.workspace = true libc.workspace = true llm-multimodal.workspace = true diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index a2df7795be..5f135e0ed5 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -98,6 +98,7 @@ async fn build_state(config: &Config) -> Result> { Ok(Arc::new( AppState::new(served_model_names, chat) + .with_model_path(config.model.clone()) .with_api_server_options(config.api_server_options) .with_server_info(ServerInfoSnapshot::from_config(config)) .with_api_keys(config.api_keys.clone()) diff --git a/rust/src/server/src/lora.rs b/rust/src/server/src/lora.rs index d58a61df86..e92c663419 100644 --- a/rust/src/server/src/lora.rs +++ b/rust/src/server/src/lora.rs @@ -1,6 +1,6 @@ -use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; +use indexmap::IndexMap; use tokio::sync::{Mutex, RwLock}; use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::lora::LoraRequest; @@ -15,8 +15,8 @@ pub(crate) struct LoraModelResolution { /// Runtime registry for dynamically loaded LoRA adapters. pub(crate) struct LoraManager { - /// Dynamically loaded LoRA adapters keyed by public model name. - requests: RwLock>, + /// Dynamically loaded LoRA adapters keyed by public model name, in load order. + requests: RwLock>, /// Monotonic adapter id allocator. LoRA ids are one-indexed. id_counter: AtomicU64, /// Serialize dynamic LoRA registry updates around engine utility calls. @@ -51,18 +51,15 @@ pub(crate) enum UnloadLoraError { impl LoraManager { pub fn new() -> Self { Self { - requests: RwLock::new(BTreeMap::new()), + requests: RwLock::new(IndexMap::new()), id_counter: AtomicU64::new(0), update_lock: Mutex::new(()), } } - /// Return base served model names plus dynamically loaded LoRA adapter - /// names. - pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec { - let mut names = base_model_names.to_vec(); - names.extend(self.requests.read().await.keys().cloned()); - names + /// Snapshot loaded LoRA adapters in load order. + pub async fn served_lora_requests(&self) -> Vec { + self.requests.read().await.values().cloned().collect() } /// Resolve the requested model against one consistent LoRA registry @@ -163,6 +160,6 @@ impl LoraManager { }); } - Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request)) + Ok(self.requests.write().await.shift_remove(lora_name).unwrap_or(lora_request)) } } diff --git a/rust/src/server/src/routes/openai/models.rs b/rust/src/server/src/routes/openai/models.rs index 42efd259e1..b06e2dc693 100644 --- a/rust/src/server/src/routes/openai/models.rs +++ b/rust/src/server/src/routes/openai/models.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use axum::Json; use axum::extract::State; @@ -6,19 +7,39 @@ use axum::extract::State; use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject}; use crate::state::AppState; -/// Return all configured served model names in OpenAI `list models` format. +// Frontend marker; Python uses "vllm". +const OWNED_BY: &str = "vllm-frontend-rs"; + +/// Base cards carry `max_model_len` and `root` = model path; LoRA cards carry +/// `root` = adapter path and `parent` = base model. LoRA cards follow load order. pub async fn list_models(State(state): State>) -> Json { - let model_names = state.served_model_names_with_loras().await; + let created = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() as i64; + let max_model_len = state.chat.engine_core_client().max_model_len(); + let model_path = state.model_path().map(str::to_string); + + let base_cards = state.served_model_names().iter().map(|name| ModelObject { + id: name.clone(), + object: "model".to_string(), + created, + owned_by: OWNED_BY.to_string(), + root: Some(model_path.clone().unwrap_or_else(|| name.clone())), + parent: None, + max_model_len: Some(max_model_len), + }); + + let primary = state.primary_model_name().to_string(); + let lora_cards = state.served_lora_requests().await.into_iter().map(|lora| ModelObject { + id: lora.lora_name, + object: "model".to_string(), + created, + owned_by: OWNED_BY.to_string(), + root: Some(lora.lora_path), + parent: Some(lora.base_model_name.unwrap_or_else(|| primary.clone())), + max_model_len: None, + }); + Json(ListModelsResponse { object: "list".to_string(), - data: model_names - .into_iter() - .map(|name| ModelObject { - id: name, - object: "model".to_string(), - created: 0, - owned_by: "vllm-frontend-rs".to_string(), - }) - .collect(), + data: base_cards.chain(lora_cards).collect(), }) } diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 95d16b83b3..8b079bbcc1 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -457,6 +457,12 @@ pub struct ModelObject { pub object: String, pub created: i64, pub owned_by: String, + /// Backend model path (base cards) or adapter path (LoRA cards). + pub root: Option, + /// Base model a LoRA adapter derives from; `null` for base models. + pub parent: Option, + /// Maximum context length; `null` for LoRA adapter cards. + pub max_model_len: Option, } /// Response body for `GET /v1/models`. diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 5eb65a4985..9d05b1c8b8 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -1107,6 +1107,93 @@ async fn list_models_returns_configured_model() { let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); assert_eq!(json["data"][0]["id"], "Qwen/Qwen1.5-0.5B-Chat"); + // No model path configured: `root` falls back to the served name. + assert_eq!(json["data"][0]["root"], "Qwen/Qwen1.5-0.5B-Chat"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn list_models_base_card_includes_metadata() { + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( + b"engine-openai-models-meta", + default_stream_output_specs(), + Arc::new(FakeChatBackend::new()), + ) + .await; + // `id` is the served alias; `root` is the underlying model path. + let mut app = build_router(Arc::new( + AppState::new(vec!["public-alias".to_string()], chat) + .with_model_path("org/backend-model".to_string()), + )); + + let response = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + + let card = json["data"][0].as_object().expect("card object"); + assert_eq!(card["id"], "public-alias"); + assert_eq!(card["owned_by"], "vllm-frontend-rs"); + assert_eq!(card["root"], "org/backend-model"); + assert!(card["max_model_len"].as_u64().expect("max_model_len") > 0); + assert!(card["created"].as_i64().expect("created") > 0); + // `parent` must be emitted as null, not omitted. + assert!(card.contains_key("parent") && card["parent"].is_null()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn list_models_lists_loras_in_load_order() { + // Load out of lexicographic order; the list must preserve load order, not sort. + let (mut app, _engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + for _ in 0..2 { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let call_id = + payload.as_array().expect("utility array")[1].as_u64().expect("call id"); + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + } + }) + }) + .await; + + for name in ["zebra", "alpha"] { + let path = format!("org/{name}"); + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/load_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ "lora_name": name, "lora_path": path }).to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + } + + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + + assert_eq!(json["data"][0]["id"], "Qwen/Qwen1.5-0.5B-Chat"); + assert_eq!(json["data"][1]["id"], "zebra"); + assert_eq!(json["data"][2]["id"], "alpha"); + // `max_model_len` must be emitted as null on LoRA cards, not omitted. + let lora_card = json["data"][1].as_object().expect("lora card object"); + assert_eq!(lora_card["root"], "org/zebra"); + assert_eq!(lora_card["parent"], "Qwen/Qwen1.5-0.5B-Chat"); + assert!(lora_card.contains_key("max_model_len") && lora_card["max_model_len"].is_null()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 0be074dff8..01b5b78962 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -40,6 +40,8 @@ pub struct AppState { server_load: AtomicU64, /// Dynamic LoRA adapter registry. lora_manager: LoraManager, + /// Backend model path reported as `root` for base-model cards. + model_path: Option, } impl AppState { @@ -65,6 +67,7 @@ impl AppState { api_key_hashes: Vec::new(), server_load: AtomicU64::new(0), lora_manager: LoraManager::new(), + model_path: None, } } @@ -80,6 +83,12 @@ impl AppState { self } + /// Set the backend model path reported as `root` for base-model cards. + pub fn with_model_path(mut self, model_path: String) -> Self { + self.model_path = Some(model_path); + self + } + /// Attach the runtime server information snapshot used by `/server_info`. pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self { self.server_info = Some(server_info); @@ -123,10 +132,14 @@ impl AppState { &self.served_model_names } - /// Return base served model names plus dynamically loaded LoRA adapter - /// names. - pub async fn served_model_names_with_loras(&self) -> Vec { - self.lora_manager.served_model_names(&self.served_model_names).await + /// Backend model path reported as `root` for base-model cards, if known. + pub fn model_path(&self) -> Option<&str> { + self.model_path.as_deref() + } + + /// Snapshot the loaded LoRA adapters in load order, for `/v1/models` cards. + pub async fn served_lora_requests(&self) -> Vec { + self.lora_manager.served_lora_requests().await } /// Resolve the requested model against one dynamic LoRA registry snapshot.