diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 0669af8daf..5b6f66cf41 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -189,6 +189,7 @@ impl ChatLlm { cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: request.data_parallel_rank, + lora_request: request.lora_request, }; let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed(); diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index c1cb83b8dc..842c941a6c 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use llm_multimodal::ImageDetail; use serde::{Deserialize, Serialize}; use serde_json::Value; +use vllm_engine_core_client::protocol::lora::LoraRequest; pub use vllm_text::SamplingParams; use vllm_text::TextDecodeOptions; pub use vllm_tool_parser::Tool as ChatTool; @@ -426,6 +427,9 @@ pub struct ChatRequest { /// Override data parallel rank. #[serde(default)] pub data_parallel_rank: Option, + /// LoRA adapter selected for this request. + #[serde(default)] + pub lora_request: Option, } impl ChatRequest { @@ -445,6 +449,7 @@ impl ChatRequest { cache_salt: None, add_special_tokens: false, data_parallel_rank: None, + lora_request: None, } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 914498c48e..ea867e4673 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -87,6 +87,17 @@ fn serve_args_auto_forward_python_flags_without_separator() { ); } +#[test] +fn serve_args_auto_forward_enable_lora_to_python() { + let cli = + Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--enable-lora"]).unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]); +} + #[test] fn serve_args_auto_forward_python_multi_char_alias_without_separator() { let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap(); diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index c2f1087fc2..8bd972ae17 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -326,15 +326,6 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub mm_processor_cache_type: Option, - /// If True, enable handling of LoRA adapters. - #[arg( - long, - visible_alias = "no-enable-lora", - default_missing_value = "true", - num_args = 0..=1 - )] - pub enable_lora: Option, - /// Dictionary mapping specific modalities to LoRA model paths. #[arg(long)] pub default_mm_loras: Option, diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 94d5ab1c62..1e6968555e 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -10,6 +10,7 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop use crate::coordinator::CoordinatorHandle; use crate::error::{Error, Result}; use crate::protocol::handshake::EngineCoreReadyResponse; +use crate::protocol::lora::LoraRequest; use crate::protocol::utility::EngineCoreUtilityRequest; use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype}; use crate::transport::{self, ConnectedEngine}; @@ -659,6 +660,24 @@ impl EngineCoreClient { Ok(results.into_iter().all(|ok| ok)) } + /// Load or refresh one LoRA adapter on every connected engine. + pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result { + Ok(self + .call_utility::("add_lora", (lora_request,)) + .await? + .into_iter() + .all(|loaded| loaded)) + } + + /// Remove one LoRA adapter from every connected engine. + pub async fn remove_lora(&self, lora_id: u64) -> Result { + Ok(self + .call_utility::("remove_lora", (lora_id,)) + .await? + .into_iter() + .all(|removed| removed)) + } + /// Put the engine to sleep. pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> { self.call_utility::<(), _>("sleep", (level, mode)).await?; diff --git a/rust/src/engine-core-client/src/protocol/lora.rs b/rust/src/engine-core-client/src/protocol/lora.rs new file mode 100644 index 0000000000..27b74cf991 --- /dev/null +++ b/rust/src/engine-core-client/src/protocol/lora.rs @@ -0,0 +1,42 @@ +use serde_tuple::{Deserialize_tuple, Serialize_tuple}; + +use crate::protocol::OpaqueValue; + +/// Request for a LoRA adapter. +/// +/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec +/// `array_like=True` struct. Keep the field order aligned with Python. +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct LoraRequest { + pub lora_name: String, + pub lora_int_id: u64, + pub lora_path: String, + #[serde(default)] + pub base_model_name: Option, + #[serde(default)] + pub tensorizer_config_dict: Option, + #[serde(default)] + pub load_inplace: bool, + #[serde(default)] + pub is_3d_lora_weight: bool, +} + +impl LoraRequest { + pub fn new( + lora_name: String, + lora_int_id: u64, + lora_path: String, + load_inplace: bool, + is_3d_lora_weight: bool, + ) -> Self { + Self { + lora_name, + lora_int_id, + lora_path, + base_model_name: None, + tensorizer_config_dict: None, + load_inplace, + is_3d_lora_weight, + } + } +} diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 4a00d9d31c..e87bc334fd 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -48,6 +48,7 @@ mod classified_outputs; pub mod dtype; pub mod handshake; pub mod logprobs; +pub mod lora; pub mod multimodal; pub mod stats; pub mod tensor; @@ -349,7 +350,7 @@ pub struct EngineCoreRequest { pub pooling_params: Option, pub arrival_time: f64, #[serde(default)] - pub lora_request: Option, + pub lora_request: Option, #[serde(default)] pub cache_salt: Option, #[serde(default)] diff --git a/rust/src/llm/src/request.rs b/rust/src/llm/src/request.rs index b17035b051..af5d257774 100644 --- a/rust/src/llm/src/request.rs +++ b/rust/src/llm/src/request.rs @@ -2,8 +2,9 @@ use std::collections::BTreeMap; use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; +use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; -use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams, OpaqueValue}; +use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams}; use crate::error::{Error, Result}; @@ -34,7 +35,7 @@ pub struct GenerateRequest { pub priority: i32, pub data_parallel_rank: Option, pub reasoning_ended: Option, - pub lora_request: Option, + pub lora_request: Option, } #[derive(Debug)] diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index ed21dd3d33..0246064b48 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -91,6 +91,7 @@ pub fn to_text_request( cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(), add_special_tokens: true, data_parallel_rank: None, + lora_request: None, }) } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index d051c6db24..adfcc60229 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -4,6 +4,7 @@ mod config; mod error; mod grpc; mod listener; +mod lora; mod middleware; mod routes; mod state; diff --git a/rust/src/server/src/lora.rs b/rust/src/server/src/lora.rs new file mode 100644 index 0000000000..d58a61df86 --- /dev/null +++ b/rust/src/server/src/lora.rs @@ -0,0 +1,168 @@ +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tokio::sync::{Mutex, RwLock}; +use vllm_engine_core_client::EngineCoreClient; +use vllm_engine_core_client::protocol::lora::LoraRequest; + +/// Snapshot of the currently served model names plus the requested LoRA, if +/// the model name resolves to a dynamic adapter. +#[derive(Debug, Clone)] +pub(crate) struct LoraModelResolution { + pub model_names: Vec, + pub lora_request: Option, +} + +/// Runtime registry for dynamically loaded LoRA adapters. +pub(crate) struct LoraManager { + /// Dynamically loaded LoRA adapters keyed by public model name. + requests: RwLock>, + /// Monotonic adapter id allocator. LoRA ids are one-indexed. + id_counter: AtomicU64, + /// Serialize dynamic LoRA registry updates around engine utility calls. + update_lock: Mutex<()>, +} + +#[derive(Debug)] +pub(crate) enum LoadLoraError { + AlreadyLoaded { lora_name: String }, + BaseModelName { lora_name: String }, + Engine(vllm_engine_core_client::Error), + NotLoaded { lora_name: String }, +} + +#[derive(Debug)] +pub(crate) enum UnloadLoraError { + NotFound { + lora_name: String, + }, + IntIdMismatch { + lora_name: String, + expected: u64, + actual: u64, + }, + Engine(vllm_engine_core_client::Error), + NotRemoved { + lora_name: String, + lora_int_id: u64, + }, +} + +impl LoraManager { + pub fn new() -> Self { + Self { + requests: RwLock::new(BTreeMap::new()), + id_counter: AtomicU64::new(0), + update_lock: Mutex::new(()), + } + } + + /// Return base served model names plus dynamically loaded LoRA adapter + /// names. + pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec { + let mut names = base_model_names.to_vec(); + names.extend(self.requests.read().await.keys().cloned()); + names + } + + /// Resolve the requested model against one consistent LoRA registry + /// snapshot. + pub async fn resolve_model( + &self, + base_model_names: &[String], + model_name: Option<&str>, + ) -> LoraModelResolution { + let requests = self.requests.read().await; + let mut model_names = base_model_names.to_vec(); + model_names.extend(requests.keys().cloned()); + let lora_request = model_name.and_then(|name| requests.get(name).cloned()); + + LoraModelResolution { + model_names, + lora_request, + } + } + + /// Load one dynamic LoRA adapter and register it as a public model name. + pub async fn load_lora( + &self, + engine_core_client: &EngineCoreClient, + base_model_names: &[String], + lora_name: String, + lora_path: String, + load_inplace: bool, + is_3d_lora_weight: bool, + ) -> Result { + let _guard = self.update_lock.lock().await; + if base_model_names.iter().any(|name| name == &lora_name) { + return Err(LoadLoraError::BaseModelName { lora_name }); + } + if !load_inplace && self.requests.read().await.contains_key(&lora_name) { + return Err(LoadLoraError::AlreadyLoaded { lora_name }); + } + + let lora_int_id = self + .requests + .read() + .await + .get(&lora_name) + .map(|request| request.lora_int_id) + .unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1); + let lora_request = LoraRequest::new( + lora_name.clone(), + lora_int_id, + lora_path, + load_inplace, + is_3d_lora_weight, + ); + + let loaded = engine_core_client + .add_lora(&lora_request) + .await + .map_err(LoadLoraError::Engine)?; + if !loaded { + return Err(LoadLoraError::NotLoaded { lora_name }); + } + self.requests.write().await.insert(lora_name, lora_request.clone()); + Ok(lora_request) + } + + /// Remove one dynamic LoRA adapter from the engine and public model + /// registry. + pub async fn unload_lora( + &self, + engine_core_client: &EngineCoreClient, + lora_name: &str, + requested_lora_int_id: Option, + ) -> Result { + let _guard = self.update_lock.lock().await; + let lora_request = self.requests.read().await.get(lora_name).cloned().ok_or_else(|| { + UnloadLoraError::NotFound { + lora_name: lora_name.to_string(), + } + })?; + + if let Some(actual) = requested_lora_int_id + && actual != lora_request.lora_int_id + { + return Err(UnloadLoraError::IntIdMismatch { + lora_name: lora_name.to_string(), + expected: lora_request.lora_int_id, + actual, + }); + } + + let removed = engine_core_client + .remove_lora(lora_request.lora_int_id) + .await + .map_err(UnloadLoraError::Engine)?; + if !removed { + return Err(UnloadLoraError::NotRemoved { + lora_name: lora_request.lora_name, + lora_int_id: lora_request.lora_int_id, + }); + } + + Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request)) + } +} diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index b6085ad2be..b708b43e66 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -3,6 +3,7 @@ mod collective_rpc; mod health; mod inference; mod load; +mod lora; mod metrics; pub(crate) mod openai; mod sleep; @@ -25,12 +26,40 @@ fn server_dev_mode_enabled() -> bool { .is_some_and(|value| value != 0) } -/// Build the minimal OpenAI-compatible router for one configured model. -pub fn build_router(state: Arc) -> Router { - build_router_with_dev_mode(state, server_dev_mode_enabled()) +fn runtime_lora_updating_enabled() -> bool { + std::env::var("VLLM_ALLOW_RUNTIME_LORA_UPDATING") + .ok() + .is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true")) } +/// Build the minimal OpenAI-compatible router for one configured model. +pub fn build_router(state: Arc) -> Router { + build_router_with_options( + state, + server_dev_mode_enabled(), + runtime_lora_updating_enabled(), + ) +} + +#[cfg(test)] fn build_router_with_dev_mode(state: Arc, dev_mode_enabled: bool) -> Router { + build_router_with_dev_mode_and_lora(state, dev_mode_enabled, false) +} + +#[cfg(test)] +fn build_router_with_dev_mode_and_lora( + state: Arc, + dev_mode_enabled: bool, + runtime_lora_updating_enabled: bool, +) -> Router { + build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled) +} + +fn build_router_with_options( + state: Arc, + dev_mode_enabled: bool, + runtime_lora_updating_enabled: bool, +) -> Router { let mut router = Router::new() // Health & monitoring .route("/health", get(health::health)) @@ -44,6 +73,12 @@ fn build_router_with_dev_mode(state: Arc, dev_mode_enabled: bool) -> R // vLLM specific inference endpoints .route("/inference/v1/generate", post(inference::generate)); + if runtime_lora_updating_enabled { + router = router + .route("/v1/load_lora_adapter", post(lora::load_lora_adapter)) + .route("/v1/unload_lora_adapter", post(lora::unload_lora_adapter)); + } + if dev_mode_enabled { // Development-only router = router diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index b256b7721f..f15f757c09 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -42,8 +42,8 @@ pub async fn generate( ValidatedJson(body): ValidatedJson, ) -> Response { let request_context = resolve_request_context(&headers, body.request_id.as_deref()); - let prepared = match prepare_generate_request(body, state.served_model_names(), request_context) - { + let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await; + let prepared = match prepare_generate_request(body, &lora_resolution, request_context) { Ok(prepared) => prepared, Err(error) => return error.into_response(), }; diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 844a606bcf..f87ff403a7 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -3,6 +3,7 @@ use vllm_text::{Prompt, TextDecodeOptions, TextRequest}; use super::types::GenerateRequest; use super::validate; use crate::error::ApiError; +use crate::lora::LoraModelResolution; use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params}; /// Lowered generate request plus the response request ID. @@ -21,10 +22,10 @@ pub struct PreparedRequest { /// text-generation format. pub fn prepare_generate_request( request: GenerateRequest, - served_model_names: &[String], + lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, ) -> Result { - validate::validate_request_compat(&request, served_model_names)?; + validate::validate_request_compat(&request, &lora_resolution.model_names)?; let stream = request.stream; let include_usage = request @@ -57,6 +58,7 @@ pub fn prepare_generate_request( cache_salt: request.cache_salt, add_special_tokens: false, data_parallel_rank: ctx.data_parallel_rank, + lora_request: lora_resolution.lora_request.clone(), }; Ok(PreparedRequest { @@ -76,9 +78,17 @@ mod tests { use vllm_text::Prompt; use super::prepare_generate_request; + use crate::lora::LoraModelResolution; use crate::routes::inference::generate::types::GenerateRequest; use crate::utils::ResolvedRequestContext; + fn served(names: &[&str]) -> LoraModelResolution { + LoraModelResolution { + model_names: names.iter().map(|s| s.to_string()).collect(), + lora_request: None, + } + } + #[test] fn prepare_generate_request_maps_token_prompt_and_sampling_params() { let request: GenerateRequest = serde_json::from_value(json!({ @@ -100,7 +110,7 @@ mod tests { let prepared = prepare_generate_request( request, - &["Qwen/Qwen1.5-0.5B-Chat".to_string()], + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), ) .expect("prepare"); @@ -143,7 +153,7 @@ mod tests { let prepared = prepare_generate_request( request, - &["Qwen/Qwen1.5-0.5B-Chat".to_string()], + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), ) .expect("prepare"); diff --git a/rust/src/server/src/routes/lora.rs b/rust/src/server/src/routes/lora.rs new file mode 100644 index 0000000000..99f1c0fe32 --- /dev/null +++ b/rust/src/server/src/routes/lora.rs @@ -0,0 +1,296 @@ +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +use axum::extract::State; +use serde::Deserialize; +use thiserror_ext::AsReport; +use validator::Validate; + +use crate::error::ApiError; +use crate::lora::{LoadLoraError, UnloadLoraError}; +use crate::routes::openai::utils::types::Normalizable; +use crate::routes::openai::utils::validated_json::ValidatedJson; +use crate::state::AppState; + +const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES"; + +#[derive(Debug, Deserialize, Validate)] +pub(crate) struct LoadLoraAdapterRequest { + lora_name: String, + lora_path: String, + #[serde(default)] + load_inplace: bool, + #[serde(default)] + is_3d_lora_weight: bool, +} + +impl Normalizable for LoadLoraAdapterRequest {} + +#[derive(Debug, Deserialize, Validate)] +pub(crate) struct UnloadLoraAdapterRequest { + lora_name: String, + #[serde(default)] + lora_int_id: Option, +} + +impl Normalizable for UnloadLoraAdapterRequest {} + +fn runtime_lora_allowed_path_prefixes() -> Option> { + let prefixes = std::env::var_os(RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV)?; + let prefixes: Vec<_> = std::env::split_paths(&prefixes) + .filter(|path| !path.as_os_str().is_empty()) + .collect(); + (!prefixes.is_empty()).then_some(prefixes) +} + +fn looks_like_local_lora_path(lora_path: &str) -> bool { + let path = Path::new(lora_path); + path.is_absolute() + || lora_path.starts_with('~') + || lora_path.starts_with('.') + || path.components().any(|component| matches!(component, Component::ParentDir)) +} + +fn validate_lora_path_access( + lora_path: &str, + allowed_prefixes: Option<&[PathBuf]>, +) -> Result, ApiError> { + let path = Path::new(lora_path); + if !looks_like_local_lora_path(lora_path) && !path.exists() { + return Ok(None); + } + + let Some(allowed_prefixes) = allowed_prefixes else { + return Err(ApiError::invalid_request( + format!( + "Local LoRA adapter paths require {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} to be configured." + ), + Some("lora_path"), + )); + }; + + if !path.is_absolute() { + return Err(ApiError::invalid_request( + format!( + "Local LoRA adapter paths must be absolute and under one of the prefixes configured by {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV}." + ), + Some("lora_path"), + )); + } + + let canonical_path = path.canonicalize().map_err(|_| { + ApiError::invalid_request( + "Local LoRA adapter path must exist and be accessible.".to_string(), + Some("lora_path"), + ) + })?; + let canonical_prefixes = allowed_prefixes + .iter() + .map(|prefix| { + prefix.canonicalize().map_err(|_| { + ApiError::server_error(format!( + "configured {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} path prefix must exist and be accessible" + )) + }) + }) + .collect::, _>>()?; + + if !canonical_prefixes.iter().any(|prefix| canonical_path.starts_with(prefix)) { + return Err(ApiError::invalid_request( + "Local LoRA adapter path is outside the configured allowed prefixes.".to_string(), + Some("lora_path"), + )); + } + + Ok(Some(canonical_path.to_string_lossy().into_owned())) +} + +/// Dynamically load one LoRA adapter and expose it as an OpenAI model id. +pub async fn load_lora_adapter( + State(state): State>, + ValidatedJson(request): ValidatedJson, +) -> Result { + if request.lora_name.is_empty() || request.lora_path.is_empty() { + return Err(ApiError::invalid_request( + "Both 'lora_name' and 'lora_path' must be provided.".to_string(), + None, + )); + } + let allowed_prefixes = runtime_lora_allowed_path_prefixes(); + let lora_path = validate_lora_path_access(&request.lora_path, allowed_prefixes.as_deref())? + .unwrap_or(request.lora_path); + + let lora_name = request.lora_name; + state + .load_lora( + lora_name.clone(), + lora_path, + request.load_inplace, + request.is_3d_lora_weight, + ) + .await + .map_err(|error| match error { + LoadLoraError::AlreadyLoaded { lora_name } => ApiError::invalid_request( + format!( + "The lora adapter '{lora_name}' has already been loaded. If you want to load the adapter in place, set 'load_inplace' to true." + ), + Some("lora_name"), + ), + LoadLoraError::BaseModelName { lora_name } => ApiError::invalid_request( + format!("The lora adapter name '{lora_name}' conflicts with a served base model."), + Some("lora_name"), + ), + LoadLoraError::Engine(error) => ApiError::server_error(format!( + "failed to load LoRA adapter '{lora_name}': {}", + error.to_report_string() + )), + LoadLoraError::NotLoaded { lora_name } => ApiError::server_error(format!( + "failed to load LoRA adapter '{lora_name}': engine rejected the adapter" + )), + })?; + + Ok(format!( + "Success: LoRA adapter '{lora_name}' added successfully." + )) +} + +/// Remove one LoRA adapter from the engine and frontend registry. +pub async fn unload_lora_adapter( + State(state): State>, + ValidatedJson(request): ValidatedJson, +) -> Result { + if request.lora_name.is_empty() { + return Err(ApiError::invalid_request( + "'lora_name' needs to be provided to unload a LoRA adapter.".to_string(), + Some("lora_name"), + )); + } + + let lora_request = state + .unload_lora(&request.lora_name, request.lora_int_id) + .await + .map_err(|error| match error { + UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name), + UnloadLoraError::IntIdMismatch { + lora_name, + expected, + actual, + } => ApiError::invalid_request( + format!( + "The requested lora_int_id {actual} does not match loaded adapter '{lora_name}' with id {expected}." + ), + Some("lora_int_id"), + ), + UnloadLoraError::Engine(error) => ApiError::server_error(format!( + "failed to unload LoRA adapter '{}': {}", + request.lora_name, + error.to_report_string() + )), + UnloadLoraError::NotRemoved { + lora_name, + lora_int_id, + } => ApiError::server_error(format!( + "failed to unload LoRA adapter '{lora_name}' with id {lora_int_id}" + )), + })?; + + Ok(format!( + "Success: LoRA adapter '{}' removed successfully.", + lora_request.lora_name + )) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::validate_lora_path_access; + + fn temp_lora_dir(test_name: &str) -> PathBuf { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "vllm-lora-{test_name}-{}-{suffix}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create temp lora dir"); + path + } + + #[test] + fn lora_path_allows_hf_repo_ids_without_prefixes() { + assert_eq!( + validate_lora_path_access("org/adapter-a", None).expect("hf repo id should be allowed"), + None + ); + } + + #[test] + fn lora_path_rejects_local_paths_without_prefixes() { + assert!(validate_lora_path_access("/tmp/adapter-a", None).is_err()); + assert!(validate_lora_path_access("./adapter-a", None).is_err()); + assert!(validate_lora_path_access("~/adapter-a", None).is_err()); + assert!(validate_lora_path_access("subdir/../../../etc/sensitive", None).is_err()); + } + + #[test] + fn lora_path_rejects_existing_bare_relative_paths_without_prefixes() { + let root = + PathBuf::from("target").join(format!("vllm-lora-relative-{}", std::process::id())); + let adapter = root.join("adapter-a"); + fs::create_dir_all(&adapter).expect("create relative adapter dir"); + + assert!( + validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), None).is_err() + ); + + fs::remove_dir_all(root).ok(); + } + + #[test] + fn lora_path_allows_absolute_paths_under_configured_prefixes() { + let root = temp_lora_dir("allowed-prefix"); + let allowed = root.join("allowed"); + let adapter = allowed.join("adapter-a"); + fs::create_dir_all(&adapter).expect("create adapter dir"); + + let prefixes = [allowed]; + let resolved = + validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), Some(&prefixes)) + .expect("path under configured prefix should be allowed"); + assert_eq!( + resolved.as_deref(), + Some( + adapter + .canonicalize() + .expect("canonical adapter") + .to_str() + .expect("utf-8 temp path") + ) + ); + + fs::remove_dir_all(root).ok(); + } + + #[test] + fn lora_path_rejects_parent_escape_from_configured_prefixes() { + let root = temp_lora_dir("parent-escape"); + let allowed = root.join("allowed"); + let private_adapter = root.join("private").join("adapter-a"); + fs::create_dir_all(&allowed).expect("create allowed dir"); + fs::create_dir_all(&private_adapter).expect("create private adapter dir"); + + let escaped = allowed.join("../private/adapter-a"); + let prefixes = [allowed]; + assert!( + validate_lora_path_access(escaped.to_str().expect("utf-8 temp path"), Some(&prefixes)) + .is_err() + ); + + fs::remove_dir_all(root).ok(); + } +} diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index c0894bb70c..543a7e806c 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -49,8 +49,9 @@ pub async fn chat_completions( ) -> Response { let stream = body.stream; let request_context = resolve_request_context(&headers, body.request_id.as_deref()); + let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await; - let prepared = match prepare_chat_request(body, state.served_model_names(), request_context) { + let prepared = match prepare_chat_request(body, &lora_resolution, request_context) { Ok(prepared) => prepared, Err(error) => return error.into_response(), }; diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index a7884b11e5..2701bef809 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -8,6 +8,7 @@ use vllm_chat::{ use super::types::ChatCompletionRequest; use super::validate; use crate::error::{ApiError, bail_invalid_request}; +use crate::lora::LoraModelResolution; use crate::routes::openai::utils::structured_outputs::convert_from_response_format; use crate::routes::openai::utils::types::{ ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, @@ -41,16 +42,21 @@ pub struct PreparedRequest { /// Validate and lower one OpenAI chat completion request into the internal chat /// format. /// -/// `served_model_names` must be non-empty; the first entry is used as the -/// `model` field in responses. +/// `lora_resolution.model_names` must be non-empty; the first entry is used as +/// the base `model` field in responses when no LoRA adapter is selected. pub(crate) fn prepare_chat_request( request: ChatCompletionRequest, - served_model_names: &[String], + lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, ) -> Result { - validate::validate_request_compat(&request, served_model_names)?; + validate::validate_request_compat(&request, &lora_resolution.model_names)?; let request_id = format!("chatcmpl-{}", ctx.request_id); + let response_model = lora_resolution + .lora_request + .as_ref() + .map(|request| request.lora_name.clone()) + .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default()); let echo = request .echo .then(|| extract_last_assistant_content(&request.messages)) @@ -131,11 +137,12 @@ pub(crate) fn prepare_chat_request( cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: ctx.data_parallel_rank, + lora_request: lora_resolution.lora_request.clone(), }; Ok(PreparedRequest { request_id, - response_model: served_model_names.first().cloned().unwrap_or_default(), + response_model, include_usage, requested_logprobs, include_prompt_logprobs, @@ -352,6 +359,7 @@ mod tests { use vllm_text::output::TextDecodeOptions; use super::prepare_chat_request; + use crate::lora::LoraModelResolution; use crate::routes::openai::chat_completions::types::{ AssistantRole, ChatCompletionMessage, ChatCompletionRequest, }; @@ -365,8 +373,11 @@ mod tests { resolve_request_context(headers, request_id) } - fn served(names: &[&str]) -> Vec { - names.iter().map(|s| s.to_string()).collect() + fn served(names: &[&str]) -> LoraModelResolution { + LoraModelResolution { + model_names: names.iter().map(|s| s.to_string()).collect(), + lora_request: None, + } } fn base_request() -> ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 33813e6768..9eda8b9d2a 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -44,12 +44,12 @@ pub async fn completions( let stream = body.stream; let logprobs = body.logprobs; let request_context = resolve_request_context(&headers, body.request_id.as_deref()); + let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await; - let prepared = - match prepare_completion_request(body, state.served_model_names(), request_context) { - Ok(prepared) => prepared, - Err(error) => return error.into_response(), - }; + let prepared = match prepare_completion_request(body, &lora_resolution, request_context) { + Ok(prepared) => prepared, + Err(error) => return error.into_response(), + }; let request_span = tracing::info_span!( "completions", request_id = %prepared.request_id, diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 066c4c046f..2d4ff08939 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -2,6 +2,7 @@ use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest}; use super::types::CompletionRequest; use crate::error::ApiError; +use crate::lora::LoraModelResolution; use crate::routes::openai::completions::validate; use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value; use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params}; @@ -30,16 +31,21 @@ pub struct PreparedRequest { /// Validate and lower one OpenAI completions request into the internal /// text-generation format. /// -/// `served_model_names` must be non-empty; the first entry is used as the -/// `model` field in responses. +/// `lora_resolution.model_names` must be non-empty; the first entry is used as +/// the base `model` field in responses when no LoRA adapter is selected. pub(crate) fn prepare_completion_request( request: CompletionRequest, - served_model_names: &[String], + lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, ) -> Result { - validate::validate_request_compat(&request, served_model_names)?; + validate::validate_request_compat(&request, &lora_resolution.model_names)?; let request_id = format!("cmpl-{}", ctx.request_id); + let response_model = lora_resolution + .lora_request + .as_ref() + .map(|request| request.lora_name.clone()) + .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default()); let logprobs = match request.logprobs { Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| { @@ -104,11 +110,12 @@ pub(crate) fn prepare_completion_request( cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: ctx.data_parallel_rank, + lora_request: lora_resolution.lora_request.clone(), }; Ok(PreparedRequest { request_id, - response_model: served_model_names.first().cloned().unwrap_or_default(), + response_model, include_usage, text_request, echo, @@ -124,6 +131,7 @@ mod tests { use vllm_text::Prompt; use super::prepare_completion_request; + use crate::lora::LoraModelResolution; use crate::routes::openai::completions::types::CompletionRequest; use crate::utils::{ResolvedRequestContext, resolve_request_context}; @@ -131,8 +139,11 @@ mod tests { resolve_request_context(headers, request_id) } - fn served(names: &[&str]) -> Vec { - names.iter().map(|s| s.to_string()).collect() + fn served(names: &[&str]) -> LoraModelResolution { + LoraModelResolution { + model_names: names.iter().map(|s| s.to_string()).collect(), + lora_request: None, + } } fn base_request_json() -> serde_json::Value { diff --git a/rust/src/server/src/routes/openai/models.rs b/rust/src/server/src/routes/openai/models.rs index 42e3098fc9..42efd259e1 100644 --- a/rust/src/server/src/routes/openai/models.rs +++ b/rust/src/server/src/routes/openai/models.rs @@ -8,13 +8,13 @@ use crate::state::AppState; /// Return all configured served model names in OpenAI `list models` format. pub async fn list_models(State(state): State>) -> Json { + let model_names = state.served_model_names_with_loras().await; Json(ListModelsResponse { object: "list".to_string(), - data: state - .served_model_names() - .iter() + data: model_names + .into_iter() .map(|name| ModelObject { - id: name.clone(), + id: name, object: "model".to_string(), created: 0, owned_by: "vllm-frontend-rs".to_string(), diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index bbd5b1bc74..6a33bc7755 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -43,7 +43,8 @@ use vllm_text::{Prompt, TextBackend}; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; -use super::{build_router, build_router_with_dev_mode}; +use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora}; +use crate::lora::LoraModelResolution; use crate::routes::openai::chat_completions::convert::prepare_chat_request; use crate::state::AppState; @@ -141,6 +142,13 @@ fn default_stream_output_specs() -> Vec<(Vec, Option Vec<&str> { text.lines().filter_map(|line| line.strip_prefix("data: ")).collect() } @@ -822,12 +830,13 @@ where let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); ( - build_router_with_dev_mode( + build_router_with_dev_mode_and_lora( Arc::new(AppState::new( vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat, )), true, + true, ), engine_task, ) @@ -1078,6 +1087,367 @@ async fn version_returns_engine_vllm_version() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn load_lora_adapter_registers_model_and_forwards_lora_request() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + assert_eq!(array[2], Value::from("add_lora")); + + let args = array[3].as_array().expect("utility args"); + let lora = args[0].as_array().expect("lora request tuple"); + assert_eq!(lora[0], Value::from("adapter-a")); + assert_eq!(lora[1], Value::from(1)); + assert_eq!(lora[2], Value::from("org/adapter-a")); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + + let add = recv_engine_message(dealer).await; + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = + rmp_serde::from_slice(&add[1]).expect("decode engine request"); + assert_adapter_a_lora_request(&request); + + send_outputs( + push, + engine_outputs_for_request(&request.request_id, default_stream_output_specs()), + ) + .await; + + let add = recv_engine_message(dealer).await; + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = + rmp_serde::from_slice(&add[1]).expect("decode engine request"); + assert_adapter_a_lora_request(&request); + + send_outputs( + push, + engine_outputs_for_request(&request.request_id, default_stream_output_specs()), + ) + .await; + + let add = recv_engine_message(dealer).await; + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = + rmp_serde::from_slice(&add[1]).expect("decode engine request"); + assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22][..])); + assert_adapter_a_lora_request(&request); + + send_outputs( + push, + engine_outputs_for_request(&request.request_id, default_stream_output_specs()), + ) + .await; + + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + assert_eq!(array[2], Value::from("remove_lora")); + + let args = array[3].as_array().expect("utility args"); + assert_eq!(args[0], Value::from(1)); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + }) + }) + .await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/load_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "adapter-a", + "lora_path": "org/adapter-a" + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["data"][1]["id"], "adapter-a"); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "adapter-a", + "prompt": "hello", + "max_tokens": 2 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "adapter-a", + "stream": false, + "messages": [{"role": "user", "content": "hello"}] + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "adapter-a", + "token_ids": [11, 22], + "stream": false, + "sampling_params": { + "max_tokens": 2 + } + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/unload_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "adapter-a", + "lora_int_id": 1 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["data"].as_array().expect("model data").len(), 1); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "adapter-a", + "prompt": "hello", + "max_tokens": 2 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + drop(app); + engine_task.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unload_lora_adapter_rejects_mismatched_lora_int_id() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + assert_eq!(array[2], Value::from("add_lora")); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + }) + }) + .await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/load_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "adapter-a", + "lora_path": "org/adapter-a" + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/unload_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "adapter-a", + "lora_int_id": 99 + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["data"][1]["id"], "adapter-a"); + + drop(app); + engine_task.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn load_lora_adapter_rejects_engine_false_result() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + assert_eq!(array[2], Value::from("add_lora")); + + send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await; + }) + }) + .await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/load_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "adapter-a", + "lora_path": "org/adapter-a" + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["data"].as_array().expect("model data").len(), 1); + + drop(app); + engine_task.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn load_lora_adapter_rejects_base_model_name_collision() { + let (mut app, engine_task) = + test_admin_app_with_engine_script(|_, _| boxed_test_future(async move {})).await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/v1/load_lora_adapter") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "lora_name": "Qwen/Qwen1.5-0.5B-Chat", + "lora_path": "org/adapter-a" + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + drop(app); + engine_task.finish().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn http_metrics_record_list_models_requests() { @@ -3000,7 +3370,10 @@ async fn prepared_openai_request_streams_text_events() { "messages": [{"role": "user", "content": "hello"}] })) .expect("decode request"), - &["Qwen/Qwen1.5-0.5B-Chat".to_string()], + &LoraModelResolution { + model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + lora_request: None, + }, crate::utils::ResolvedRequestContext::default(), ) .expect("prepare request"); diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 08a64d85e0..6863000943 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -5,6 +5,9 @@ use tokio::time::{Duration, Instant, sleep_until}; use tracing::warn; use vllm_chat::ChatLlm; use vllm_engine_core_client::EngineCoreClient; +use vllm_engine_core_client::protocol::lora::LoraRequest; + +use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError}; const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); @@ -21,6 +24,8 @@ pub struct AppState { pub enable_request_id_headers: bool, /// Number of in-flight inference requests currently owned by this frontend. server_load: AtomicU64, + /// Dynamic LoRA adapter registry. + lora_manager: LoraManager, } impl AppState { @@ -43,6 +48,7 @@ impl AppState { enable_log_requests: false, enable_request_id_headers: false, server_load: AtomicU64::new(0), + lora_manager: LoraManager::new(), } } @@ -69,6 +75,49 @@ impl AppState { &self.served_model_names } + /// Return base served model names plus dynamically loaded LoRA adapter + /// names. + pub async fn served_model_names_with_loras(&self) -> Vec { + self.lora_manager.served_model_names(&self.served_model_names).await + } + + /// Resolve the requested model against one dynamic LoRA registry snapshot. + pub async fn resolve_model_with_loras(&self, model_name: Option<&str>) -> LoraModelResolution { + self.lora_manager.resolve_model(&self.served_model_names, model_name).await + } + + /// Load one dynamic LoRA adapter and register it as a public model name. + pub async fn load_lora( + &self, + lora_name: String, + lora_path: String, + load_inplace: bool, + is_3d_lora_weight: bool, + ) -> Result { + self.lora_manager + .load_lora( + self.engine_core_client(), + &self.served_model_names, + lora_name, + lora_path, + load_inplace, + is_3d_lora_weight, + ) + .await + } + + /// Remove one dynamic LoRA adapter from the engine and public model + /// registry. + pub async fn unload_lora( + &self, + lora_name: &str, + lora_int_id: Option, + ) -> Result { + self.lora_manager + .unload_lora(self.engine_core_client(), lora_name, lora_int_id) + .await + } + /// Return a reference to the underlying engine core client for utility /// calls. pub(crate) fn engine_core_client(&self) -> &EngineCoreClient { diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 54ffe1ff4b..d661c99606 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -40,11 +40,11 @@ pub fn lower_text_request( cache_salt: request.cache_salt.clone(), priority: request.priority, data_parallel_rank: request.data_parallel_rank, + lora_request: request.lora_request.clone(), // Fields below are currently placeholders. arrival_time: None, trace_headers: None, reasoning_ended: None, - lora_request: None, }; Ok(PreparedTextRequest { diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 9e2464f14a..1ca8f8a924 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -4,6 +4,7 @@ use enum_as_inner::EnumAsInner; use serde::{Deserialize, Serialize}; use serde_json::Value; use vllm_engine_core_client::protocol::StructuredOutputsParams; +use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; use crate::error::{Error, Result}; @@ -166,6 +167,9 @@ pub struct TextRequest { /// Override data parallel rank. #[serde(default)] pub data_parallel_rank: Option, + /// LoRA adapter selected for this request. + #[serde(default)] + pub lora_request: Option, } impl TextRequest { @@ -182,6 +186,7 @@ impl TextRequest { cache_salt: None, add_special_tokens: false, data_parallel_rank: None, + lora_request: None, } }