diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index a061c8eff0..a0437679be 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -29,8 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig; use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args}; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, - DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection, - TlsConfig, + DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig, + RendererSelection, TlsConfig, }; use crate::cli::unsupported::UnsupportedArgs; @@ -269,6 +269,13 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub limit_mm_per_prompt: MmLimitPerPrompt, + /// LoRA adapters to load before serving, each as `name=path` or a JSON + /// object: `{"name": "name", "path": "lora_path", "base_model_name": "id"}`. + /// Requires `--enable-lora`; startup fails if any adapter cannot be loaded. + #[arg(long, num_args = 1.., value_name = "MODULE")] + #[serde(default)] + pub lora_modules: Vec, + /// The format to render message content within a chat template. /// /// * "auto" detects the format from the template @@ -498,6 +505,7 @@ impl SharedRuntimeArgs { chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, limit_mm_per_prompt: self.limit_mm_per_prompt, + lora_modules: self.lora_modules, chat_template_content_format: self.chat_template_content_format, max_logprobs: self.max_logprobs, api_server_options, @@ -552,6 +560,7 @@ impl SharedRuntimeArgs { chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, limit_mm_per_prompt: self.limit_mm_per_prompt, + lora_modules: self.lora_modules, chat_template_content_format: self.chat_template_content_format, max_logprobs: self.max_logprobs, api_server_options, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 89979ec6d0..b1a178b670 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -4,7 +4,8 @@ use expect_test::expect; use vllm_engine_core_client::TransportMode; use vllm_server::{ - Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection, + Config, GenerationConfigMode, HttpListenerMode, LoraModulePath, ParserSelection, + RendererSelection, }; use super::{BenchCommand, Cli, Command}; @@ -103,6 +104,7 @@ fn serve_args_forward_python_flags_with_separator() { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -623,6 +625,103 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() { assert!(config.api_server_options.enable_request_id_headers); } +#[test] +fn serve_args_parse_lora_modules_in_both_forms() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--enable-lora", + "--lora-modules", + "alice=charent/self_cognition_Alice", + r#"{"name": "bob", "path": "/adapters/bob", "base_model_name": "base"}"#, + "--lora-modules", + "carol=org/carol", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + // Frontend-only flag: the managed Python engine must not see it. + assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]); + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string()); + expect![[r#" + [ + LoraModulePath { + name: "alice", + path: "charent/self_cognition_Alice", + base_model_name: None, + is_3d_lora_weight: false, + }, + LoraModulePath { + name: "bob", + path: "/adapters/bob", + base_model_name: Some( + "base", + ), + is_3d_lora_weight: false, + }, + LoraModulePath { + name: "carol", + path: "org/carol", + base_model_name: None, + is_3d_lora_weight: false, + }, + ] + "#]] + .assert_debug_eq(&config.lora_modules); +} + +#[test] +fn serve_args_reject_malformed_lora_module() { + let error = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--lora-modules", + "alice", + ]) + .unwrap_err(); + + expect![[r#" + error: invalid value 'alice' for '--lora-modules ...': expected `name=path`, got `alice` + + For more information, try '--help'. + "#]].assert_eq(&error.to_string()); +} + +#[test] +fn frontend_args_json_passes_lora_modules_into_config() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "frontend", + "--listen-fd", + "3", + "--input-address", + "ipc:///tmp/input.sock", + "--output-address", + "ipc:///tmp/output.sock", + "--args-json", + r#"{"model_tag":"Qwen/Qwen3-0.6B","lora_modules":[{"name":"alice","path":"org/alice","base_model_name":null,"is_3d_lora_weight":true}]}"#, + ]) + .unwrap(); + + let Command::Frontend(args) = cli.command else { + panic!("expected frontend args"); + }; + let config = args.into_config(); + assert_eq!( + config.lora_modules, + vec![LoraModulePath { + name: "alice".to_string(), + path: "org/alice".to_string(), + base_model_name: None, + is_3d_lora_weight: true, + }] + ); +} + #[test] fn serve_passes_api_keys_into_config() { let cli = Cli::try_parse_from([ @@ -822,6 +921,7 @@ fn frontend_args_accept_json() { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -1416,6 +1516,7 @@ fn serve_args_accept_handshake_aliases() { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -1561,6 +1662,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { @@ -1647,6 +1749,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { @@ -1754,6 +1857,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: {}, + lora_modules: [], chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index d2372ed566..29ea5d06b0 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -353,13 +353,6 @@ pub struct EngineUnsupportedArgs { #[serde_with::skip_serializing_none] #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)] pub struct ServerUnsupportedArgs { - /// LoRA modules configurations in either 'name=path' format or JSON format - /// or JSON list format. Example (old format): `'name=path'` Example (new - /// format): `{"name": "name", "path": "lora_path", - /// "base_model_name": "id"}` - #[arg(long)] - pub lora_modules: Option, - /// Whether to trust the chat template provided in the request. If False, /// the server will always use the chat template specified by /// `--chat-template` or the ones from tokenizer. diff --git a/rust/src/engine-core-client/src/protocol/lora.rs b/rust/src/engine-core-client/src/protocol/lora.rs index 4621a91da3..6afeea29f2 100644 --- a/rust/src/engine-core-client/src/protocol/lora.rs +++ b/rust/src/engine-core-client/src/protocol/lora.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use thiserror::Error; use crate::protocol::OpaqueValue; @@ -24,43 +23,3 @@ pub struct LoraRequest { #[serde(default)] pub is_3d_lora_weight: bool, } - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum LoraRequestError { - #[error("lora_name must not be empty")] - EmptyName, - #[error("lora_int_id must be nonzero")] - ZeroId, - #[error("lora_path must not be empty")] - EmptyPath, -} - -impl LoraRequest { - pub fn new( - lora_name: String, - lora_int_id: u64, - lora_path: String, - load_inplace: bool, - is_3d_lora_weight: bool, - ) -> Result { - if lora_name.trim().is_empty() { - return Err(LoraRequestError::EmptyName); - } - if lora_int_id == 0 { - return Err(LoraRequestError::ZeroId); - } - if lora_path.trim().is_empty() { - return Err(LoraRequestError::EmptyPath); - } - - Ok(Self { - lora_name, - lora_int_id, - lora_path, - base_model_name: None, - tensorizer_config_dict: None, - load_inplace, - is_3d_lora_weight, - }) - } -} diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 5c99fc92eb..d07ba242ae 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -73,6 +73,7 @@ async fn main() -> Result<()> { chat_template: None, default_chat_template_kwargs: None, limit_mm_per_prompt: HashMap::new(), + lora_modules: Vec::new(), chat_template_content_format: ChatTemplateContentFormatOption::Auto, max_logprobs: None, api_server_options: ApiServerOptions::default(), diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index eb690d4ea0..fde53183f5 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -3,12 +3,13 @@ use std::collections::HashMap; use std::fmt; +use std::str::FromStr; use std::time::Duration; use anyhow::{Result, bail}; use axum::http::{HeaderName, HeaderValue, Method}; use educe::Educe; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_chat::{ @@ -157,6 +158,48 @@ impl TlsConfig { } } +/// One LoRA adapter to load before the server accepts traffic. +/// +/// Mirrors `LoRAModulePath` in vllm/entrypoints/openai/models/protocol.py, +/// which is also the JSON shape the Python supervisor forwards in +/// `--args-json`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LoraModulePath { + /// Public model id the adapter is served under. + pub name: String, + /// Local path or Hugging Face repo id of the adapter. + pub path: String, + /// Base model reported as `parent` in `/v1/models`; defaults to the + /// primary served model name. + #[serde(default)] + pub base_model_name: Option, + #[serde(default)] + pub is_3d_lora_weight: bool, +} + +impl FromStr for LoraModulePath { + type Err = String; + + /// Accept the two CLI forms Python's `LoRAParserAction` accepts: + /// `name=path`, or a JSON object with the struct's fields. + fn from_str(value: &str) -> Result { + if !value.trim_start().starts_with('{') { + let (name, path) = value + .split_once('=') + .filter(|(name, path)| !name.is_empty() && !path.is_empty()) + .ok_or_else(|| format!("expected `name=path`, got `{value}`"))?; + return Ok(Self { + name: name.to_string(), + path: path.to_string(), + base_model_name: None, + is_3d_lora_weight: false, + }); + } + serde_json::from_str(value) + .map_err(|e| format!("expected `name=path` or a JSON object: {e}")) + } +} + /// Normalized runtime configuration for the minimal OpenAI-compatible server. #[derive(Educe, Clone, PartialEq, Eq, Serialize)] #[educe(Debug)] @@ -192,6 +235,9 @@ pub struct Config { /// Maximum number of input items allowed per prompt for each modality. /// Unspecified modalities are unlimited. pub limit_mm_per_prompt: MmLimitPerPrompt, + /// LoRA adapters loaded at startup, in order. Startup fails if any of + /// them cannot be loaded. + pub lora_modules: Vec, /// How to serialize `message.content` for chat-template rendering. pub chat_template_content_format: ChatTemplateContentFormatOption, /// Optional maximum number of top log probabilities accepted by the @@ -290,3 +336,66 @@ impl fmt::Debug for RedactedApiKeys<'_> { fn fmt_redacted_api_keys(api_keys: &[String], f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&RedactedApiKeys(api_keys), f) } + +#[cfg(test)] +mod lora_module_path_tests { + use super::LoraModulePath; + + #[test] + fn parses_name_equals_path() { + let module: LoraModulePath = "alice=charent/self_cognition_Alice".parse().unwrap(); + assert_eq!( + module, + LoraModulePath { + name: "alice".to_string(), + path: "charent/self_cognition_Alice".to_string(), + base_model_name: None, + is_3d_lora_weight: false, + } + ); + } + + #[test] + fn parses_json_object() { + let module: LoraModulePath = + r#"{"name": "alice", "path": "/adapters/alice", "base_model_name": "base", "is_3d_lora_weight": true}"# + .parse() + .unwrap(); + assert_eq!( + module, + LoraModulePath { + name: "alice".to_string(), + path: "/adapters/alice".to_string(), + base_model_name: Some("base".to_string()), + is_3d_lora_weight: true, + } + ); + } + + #[test] + fn parses_name_equals_path_with_comma() { + let module: LoraModulePath = "alice=/adapters/a,b".parse().unwrap(); + assert_eq!(module.name, "alice"); + assert_eq!(module.path, "/adapters/a,b"); + } + + #[test] + fn json_object_optional_fields_default() { + let module: LoraModulePath = r#"{"name": "a", "path": "org/a"}"#.parse().unwrap(); + assert_eq!(module.base_model_name, None); + assert!(!module.is_3d_lora_weight); + } + + #[test] + fn rejects_empty_name_or_path() { + assert!("=org/a".parse::().is_err()); + assert!("a=".parse::().is_err()); + } + + #[test] + fn rejects_bare_path_and_bad_json() { + assert!("org/a".parse::().is_err()); + assert!(r#"{"name": "a"}"#.parse::().is_err()); + assert!(r#"[{"name": "a", "path": "org/a"}]"#.parse::().is_err()); + } +} diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index 987414b963..ddb74bf40f 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -13,6 +13,7 @@ use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode; use super::{ControlServer, pb}; +use crate::config::LoraModulePath; use crate::lora::{LoadLoraError, LoraDisabledError, LoraPathAccessError, UnloadLoraError}; use crate::state::AppState; @@ -147,7 +148,7 @@ fn lora_to_proto(adapter: &LoraRequest) -> pb::LoraAdapter { fn load_lora_status(error: LoadLoraError) -> Status { let code = match &error { LoadLoraError::Disabled(_) => Code::FailedPrecondition, - LoadLoraError::InvalidRequest(_) => Code::InvalidArgument, + LoadLoraError::InvalidAdapter { .. } => Code::InvalidArgument, LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => Code::InvalidArgument, LoadLoraError::PathAccess(LoraPathAccessError::InvalidConfiguration { .. }) => { Code::Internal @@ -247,11 +248,13 @@ impl pb::control_server::Control for ControlServiceImpl { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let adapter = self - .state - .load_lora(request.lora_name, request.source_path, false, false) - .await - .map_err(load_lora_status)?; + let module = LoraModulePath { + name: request.lora_name, + path: request.source_path, + base_model_name: None, + is_3d_lora_weight: false, + }; + let adapter = self.state.load_lora(module, false).await.map_err(load_lora_status)?; Ok(Response::new(pb::LoadLoraResponse { adapter: Some(lora_to_proto(&adapter)), })) diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index e8138b2494..098c803805 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -29,7 +29,7 @@ use axum::body::Body; use axum::http::Request; pub use config::{ ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT, - HttpListenerMode, TlsConfig, + HttpListenerMode, LoraModulePath, TlsConfig, }; use hyper::body::Incoming; use hyper::server::conn::http1; @@ -86,7 +86,7 @@ fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str { } /// Build the shared application state for one configured model and one engine -/// client. +/// client, including any static LoRA adapters from `--lora-modules`. async fn build_state(config: &Config) -> Result> { // If no served names are specified, fall back to the backend model path so // that the API always has at least one valid model ID. Use the same primary @@ -139,7 +139,7 @@ async fn build_state(config: &Config) -> Result> { .with_tool_call_parser(config.tool_call_parser.clone()) .with_reasoning_parser(config.reasoning_parser.clone()); - Ok(Arc::new( + let state = Arc::new( AppState::new(served_model_names, chat) .with_model_path(config.model.clone()) .with_api_server_options(config.api_server_options) @@ -147,7 +147,31 @@ async fn build_state(config: &Config) -> Result> { .with_api_keys(config.api_keys.clone()) .with_cors(config.cors.clone()) .with_profiler(config.profiler.clone()), - )) + ); + + // Load operator-configured static LoRA adapters before serving, failing + // startup on any error like Python's `init_static_loras` does. + load_static_loras(&state, &config.lora_modules).await?; + + Ok(state) +} + +async fn load_static_loras(state: &AppState, modules: &[LoraModulePath]) -> Result<()> { + for module in modules { + let request = state.load_static_lora(module).await.with_context(|| { + format!( + "failed to load LoRA adapter `{}` from --lora-modules", + module.name + ) + })?; + info!( + lora_name = %request.lora_name, + lora_int_id = request.lora_int_id, + lora_path = %request.lora_path, + "loaded static LoRA adapter" + ); + } + Ok(()) } /// Run the OpenAI-compatible HTTP server until the supplied shutdown token is @@ -182,7 +206,8 @@ where .transpose() .context("invalid TLS configuration")?; - // Also check shutdown during the (potentially long) startup handshake. + // Also check shutdown during the (potentially long) startup handshake and + // static LoRA loading. let state = tokio::select! { result = build_state(&config) => result?, _ = shutdown.cancelled() => return Ok(()), diff --git a/rust/src/server/src/lora.rs b/rust/src/server/src/lora.rs index b339688578..c3bf426e7b 100644 --- a/rust/src/server/src/lora.rs +++ b/rust/src/server/src/lora.rs @@ -9,7 +9,9 @@ use thiserror::Error; use thiserror_ext::Macro; use tokio::sync::{Mutex, RwLock}; use vllm_engine_core_client::EngineCoreClient; -use vllm_engine_core_client::protocol::lora::{LoraRequest, LoraRequestError}; +use vllm_engine_core_client::protocol::lora::LoraRequest; + +use crate::config::LoraModulePath; const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES"; @@ -103,12 +105,12 @@ pub(crate) struct LoraManager { #[error("engine was not started with LoRA enabled")] pub(crate) struct LoraDisabledError; -#[derive(Debug, Error)] +#[derive(Debug, Error, Macro)] pub(crate) enum LoadLoraError { #[error(transparent)] Disabled(#[from] LoraDisabledError), - #[error(transparent)] - InvalidRequest(#[from] LoraRequestError), + #[error("{message}")] + InvalidAdapter { message: String }, #[error(transparent)] PathAccess(#[from] LoraPathAccessError), #[error("LoRA adapter `{lora_name}` is already loaded")] @@ -186,14 +188,49 @@ impl LoraManager { &self, engine_core_client: &EngineCoreClient, base_model_names: &[String], - lora_name: String, - lora_path: String, + mut module: LoraModulePath, load_inplace: bool, - is_3d_lora_weight: bool, ) -> Result { let allowed_prefixes = runtime_lora_allowed_path_prefixes(); - let lora_path = validate_lora_path_access(&lora_path, allowed_prefixes.as_deref())? - .unwrap_or(lora_path); + if let Some(canonical_path) = + validate_lora_path_access(&module.path, allowed_prefixes.as_deref())? + { + module.path = canonical_path; + } + self.register(engine_core_client, base_model_names, module, load_inplace).await + } + + /// Load an operator-configured adapter (`--lora-modules`). Unlike the + /// runtime endpoint, the path is trusted as given: no allowed-prefix + /// check applies. + pub async fn load_static_lora( + &self, + engine_core_client: &EngineCoreClient, + base_model_names: &[String], + module: &LoraModulePath, + ) -> Result { + self.register(engine_core_client, base_model_names, module.clone(), false).await + } + + async fn register( + &self, + engine_core_client: &EngineCoreClient, + base_model_names: &[String], + module: LoraModulePath, + load_inplace: bool, + ) -> Result { + let LoraModulePath { + name: lora_name, + path: lora_path, + base_model_name, + is_3d_lora_weight, + } = module; + if lora_name.trim().is_empty() { + bail_invalid_adapter!("lora_name must not be empty"); + } + if lora_path.trim().is_empty() { + bail_invalid_adapter!("lora_path must not be empty"); + } let _guard = self.update_lock.lock().await; if base_model_names.iter().any(|name| name == &lora_name) { return Err(LoadLoraError::BaseModelName { lora_name }); @@ -206,14 +243,15 @@ impl LoraManager { let lora_int_id = existing_lora_int_id .unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1); - let lora_request = LoraRequest::new( - lora_name.clone(), + let lora_request = LoraRequest { + lora_name: lora_name.clone(), lora_int_id, lora_path, + base_model_name, + tensorizer_config_dict: None, load_inplace, is_3d_lora_weight, - ) - .map_err(LoadLoraError::InvalidRequest)?; + }; drop(requests); let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| { diff --git a/rust/src/server/src/routes/lora.rs b/rust/src/server/src/routes/lora.rs index a9bd7d3f30..ac0cb11e1a 100644 --- a/rust/src/server/src/routes/lora.rs +++ b/rust/src/server/src/routes/lora.rs @@ -8,6 +8,7 @@ use serde::Deserialize; use thiserror_ext::AsReport; use validator::Validate; +use crate::config::LoraModulePath; use crate::error::ApiError; use crate::lora::{LoadLoraError, LoraPathAccessError, UnloadLoraError}; use crate::routes::openai::utils::types::Normalizable; @@ -39,7 +40,7 @@ fn load_lora_api_error(error: LoadLoraError) -> ApiError { let message = error.to_report_string(); match error { LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None), - LoadLoraError::InvalidRequest(_) => ApiError::invalid_request(message, None), + LoadLoraError::InvalidAdapter { .. } => ApiError::invalid_request(message, None), LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => { ApiError::invalid_request(message, Some("lora_path")) } @@ -81,13 +82,14 @@ pub async fn load_lora_adapter( ValidatedJson(request): ValidatedJson, ) -> Result { let lora_name = request.lora_name; + let module = LoraModulePath { + name: lora_name.clone(), + path: request.lora_path, + base_model_name: None, + is_3d_lora_weight: request.is_3d_lora_weight, + }; state - .load_lora( - lora_name.clone(), - request.lora_path, - request.load_inplace, - request.is_3d_lora_weight, - ) + .load_lora(module, request.load_inplace) .await .map_err(load_lora_api_error)?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 484eed35d6..c96cb7fc90 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -57,7 +57,8 @@ use super::{ build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag, render::build_router as build_render_router, }; -use crate::config::{ApiServerOptions, CorsConfig}; +use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath}; +use crate::lora::LoadLoraError; use crate::render::RenderState; use crate::state::AppState; @@ -941,6 +942,20 @@ async fn test_admin_app_with_ready_and_engine_script( ready: EngineCoreReadyResponse, script: F, ) -> (axum::Router, MockEngineTask) +where + F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static, +{ + let (state, engine_task) = test_admin_state_with_ready_and_engine_script(ready, script).await; + ( + build_router_with_dev_mode_and_lora(state, true, true), + engine_task, + ) +} + +async fn test_admin_state_with_ready_and_engine_script( + ready: EngineCoreReadyResponse, + script: F, +) -> (Arc, MockEngineTask) where F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static, { @@ -968,18 +983,148 @@ where let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); ( - build_router_with_dev_mode_and_lora( - Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - )), - true, - true, - ), + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), engine_task, ) } +/// Engine script that answers one `add_lora` utility call with `loaded` and +/// hands the decoded LoRA request tuple to `check`. +fn add_lora_script( + loaded: bool, + check: impl FnOnce(&[Value]) + Send + 'static, +) -> impl for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static +{ + move |dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + assert_eq!(array[2], Value::from("add_lora")); + let args = array[3].as_array().expect("utility args"); + check(args[0].as_array().expect("lora request tuple")); + send_outputs(push, utility_outputs(call_id, utility_result_value(loaded))).await; + }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn static_lora_module_loads_and_lists_with_parent() { + // An absolute local path: the runtime endpoint would reject it without + // VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES, static config trusts it. + let mut ready = default_ready_response(); + ready.supports_lora = true; + ready.max_loras = 4; + let (state, _engine_task) = test_admin_state_with_ready_and_engine_script( + ready, + add_lora_script(true, |lora| { + assert_eq!(lora[0], Value::from("alice")); + assert_eq!(lora[1], Value::from(1)); + assert_eq!(lora[2], Value::from("/adapters/alice")); + assert_eq!(lora[3], Value::from("base-model")); + }), + ) + .await; + + let module = LoraModulePath { + name: "alice".to_string(), + path: "/adapters/alice".to_string(), + base_model_name: Some("base-model".to_string()), + is_3d_lora_weight: false, + }; + let request = state.load_static_lora(&module).await.expect("load static lora"); + assert_eq!(request.lora_name, "alice"); + assert_eq!(request.lora_int_id, 1); + + let mut app = build_router_with_dev_mode_and_lora(state, true, false); + let models = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["data"][1]["id"], "alice"); + assert_eq!(json["data"][1]["root"], "/adapters/alice"); + assert_eq!(json["data"][1]["parent"], "base-model"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn static_lora_module_requires_lora_enabled_engine() { + let (state, _engine_task) = test_admin_state_with_ready_and_engine_script( + default_ready_response(), + |_dealer, _push| boxed_test_future(async {}), + ) + .await; + + let module = LoraModulePath { + name: "alice".to_string(), + path: "org/alice".to_string(), + base_model_name: None, + is_3d_lora_weight: false, + }; + let error = state.load_static_lora(&module).await.expect_err("engine has no lora"); + assert!(matches!(error, LoadLoraError::Disabled(_)), "{error:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn static_lora_module_fails_when_engine_rejects_it() { + let mut ready = default_ready_response(); + ready.supports_lora = true; + ready.max_loras = 4; + let (state, _engine_task) = + test_admin_state_with_ready_and_engine_script(ready, add_lora_script(false, |_| {})).await; + + let module = LoraModulePath { + name: "alice".to_string(), + path: "org/alice".to_string(), + base_model_name: None, + is_3d_lora_weight: false, + }; + let error = state.load_static_lora(&module).await.expect_err("engine rejected"); + assert!( + matches!(error, LoadLoraError::NotLoaded { .. }), + "{error:?}" + ); + assert!(state.served_lora_requests().await.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn static_lora_module_rejects_empty_name_or_path() { + // The JSON form of `--lora-modules` is not validated at parse time; the + // manager rejects empty fields before any engine RPC. + let mut ready = default_ready_response(); + ready.supports_lora = true; + ready.max_loras = 4; + let (state, _engine_task) = + test_admin_state_with_ready_and_engine_script(ready, |_dealer, _push| { + boxed_test_future(async {}) + }) + .await; + + for (name, path) in [("", "org/alice"), ("alice", "")] { + let module = LoraModulePath { + name: name.to_string(), + path: path.to_string(), + base_model_name: None, + is_3d_lora_weight: false, + }; + let error = state.load_static_lora(&module).await.expect_err("empty field"); + assert!( + matches!(error, LoadLoraError::InvalidAdapter { .. }), + "{error:?}" + ); + } +} + async fn test_app_with_engine_handle() -> (axum::Router, MockEngineTask) { test_app_with_stream_output_specs(default_stream_output_specs()).await } diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index b13270a40c..b234b0249b 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -14,7 +14,7 @@ use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::runtime::BackgroundShutdownRuntime; -use crate::config::{ApiServerOptions, CorsConfig}; +use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath}; use crate::lora::{ LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError, }; @@ -181,24 +181,34 @@ impl AppState { /// Returns error if the engine was started without LoRA support. pub async fn load_lora( &self, - lora_name: String, - lora_path: String, + module: LoraModulePath, load_inplace: bool, - is_3d_lora_weight: bool, ) -> Result { self.ensure_lora_enabled()?; self.lora_manager .load_lora( self.engine_core_client(), &self.served_model_names, - lora_name, - lora_path, + module, load_inplace, - is_3d_lora_weight, ) .await } + /// Load one adapter from `--lora-modules` and register it as a public + /// model name. + /// + /// Returns error if the engine was started without LoRA support. + pub async fn load_static_lora( + &self, + module: &LoraModulePath, + ) -> Result { + self.ensure_lora_enabled()?; + self.lora_manager + .load_static_lora(self.engine_core_client(), &self.served_model_names, module) + .await + } + /// Remove one dynamic LoRA adapter from the engine and public model /// registry. ///