iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Rust Frontend] Support --lora-modules for static adapter loading (#54837)

Browse Source 
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Will Eaton <weaton@redhat.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Will Eaton
2026-09-03 19:12:45 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 21a2211040
commit 2db1c4dc31
12 changed files with 497 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/cmd/src/cli.rs
+11 -2
View File
@@ -29,8 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
use vllm_server::{
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
TlsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
RendererSelection, TlsConfig,
};
use crate::cli::unsupported::UnsupportedArgs;
@@ -269,6 +269,13 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub limit_mm_per_prompt: MmLimitPerPrompt,
/// LoRA adapters to load before serving, each as `name=path` or a JSON
/// object: `{"name": "name", "path": "lora_path", "base_model_name": "id"}`.
/// Requires `--enable-lora`; startup fails if any adapter cannot be loaded.
#[arg(long, num_args = 1.., value_name = "MODULE")]
#[serde(default)]
pub lora_modules: Vec<LoraModulePath>,
/// The format to render message content within a chat template.
///
/// * "auto" detects the format from the template
@@ -498,6 +505,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
@@ -552,6 +560,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
rust/src/cmd/src/cli/tests.rs
+105 -1
View File
@@ -4,7 +4,8 @@
use expect_test::expect;
use vllm_engine_core_client::TransportMode;
use vllm_server::{
Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
Config, GenerationConfigMode, HttpListenerMode, LoraModulePath, ParserSelection,
RendererSelection,
};
use super::{BenchCommand, Cli, Command};
@@ -103,6 +104,7 @@ fn serve_args_forward_python_flags_with_separator() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -623,6 +625,103 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
assert!(config.api_server_options.enable_request_id_headers);
}
#[test]
fn serve_args_parse_lora_modules_in_both_forms() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--enable-lora",
"--lora-modules",
"alice=charent/self_cognition_Alice",
r#"{"name": "bob", "path": "/adapters/bob", "base_model_name": "base"}"#,
"--lora-modules",
"carol=org/carol",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
// Frontend-only flag: the managed Python engine must not see it.
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
expect![[r#"
[
LoraModulePath {
name: "alice",
path: "charent/self_cognition_Alice",
base_model_name: None,
is_3d_lora_weight: false,
},
LoraModulePath {
name: "bob",
path: "/adapters/bob",
base_model_name: Some(
"base",
),
is_3d_lora_weight: false,
},
LoraModulePath {
name: "carol",
path: "org/carol",
base_model_name: None,
is_3d_lora_weight: false,
},
]
"#]]
.assert_debug_eq(&config.lora_modules);
}
#[test]
fn serve_args_reject_malformed_lora_module() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--lora-modules",
"alice",
])
.unwrap_err();
expect![[r#"
error: invalid value 'alice' for '--lora-modules <MODULE>...': expected `name=path`, got `alice`
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn frontend_args_json_passes_lora_modules_into_config() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","lora_modules":[{"name":"alice","path":"org/alice","base_model_name":null,"is_3d_lora_weight":true}]}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
let config = args.into_config();
assert_eq!(
config.lora_modules,
vec![LoraModulePath {
name: "alice".to_string(),
path: "org/alice".to_string(),
base_model_name: None,
is_3d_lora_weight: true,
}]
);
}
#[test]
fn serve_passes_api_keys_into_config() {
let cli = Cli::try_parse_from([
@@ -822,6 +921,7 @@ fn frontend_args_accept_json() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1416,6 +1516,7 @@ fn serve_args_accept_handshake_aliases() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1561,6 +1662,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1647,6 +1749,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1754,6 +1857,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
rust/src/cmd/src/cli/unsupported.rs
-7
View File
@@ -353,13 +353,6 @@ pub struct EngineUnsupportedArgs {
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
pub struct ServerUnsupportedArgs {
/// LoRA modules configurations in either 'name=path' format or JSON format
/// or JSON list format. Example (old format): `'name=path'` Example (new
/// format): `{"name": "name", "path": "lora_path",
/// "base_model_name": "id"}`
#[arg(long)]
pub lora_modules: Option<Unsupported>,
/// Whether to trust the chat template provided in the request. If False,
/// the server will always use the chat template specified by
/// `--chat-template` or the ones from tokenizer.
rust/src/engine-core-client/src/protocol/lora.rs
-41
View File
@@ -2,7 +2,6 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror::Error;
use crate::protocol::OpaqueValue;
@@ -24,43 +23,3 @@ pub struct LoraRequest {
#[serde(default)]
pub is_3d_lora_weight: bool,
}
#[derive(Debug, Error, PartialEq, Eq)]
pub enum LoraRequestError {
#[error("lora_name must not be empty")]
EmptyName,
#[error("lora_int_id must be nonzero")]
ZeroId,
#[error("lora_path must not be empty")]
EmptyPath,
}
impl LoraRequest {
pub fn new(
lora_name: String,
lora_int_id: u64,
lora_path: String,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Result<Self, LoraRequestError> {
if lora_name.trim().is_empty() {
return Err(LoraRequestError::EmptyName);
}
if lora_int_id == 0 {
return Err(LoraRequestError::ZeroId);
}
if lora_path.trim().is_empty() {
return Err(LoraRequestError::EmptyPath);
}
Ok(Self {
lora_name,
lora_int_id,
lora_path,
base_model_name: None,
tensorizer_config_dict: None,
load_inplace,
is_3d_lora_weight,
})
}
}
rust/src/server/examples/external_engine_openai_qwen.rs
+1
View File
@@ -73,6 +73,7 @@ async fn main() -> Result<()> {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: HashMap::new(),
lora_modules: Vec::new(),
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_logprobs: None,
api_server_options: ApiServerOptions::default(),
rust/src/server/src/config.rs
+110 -1
View File
@@ -3,12 +3,13 @@
use std::collections::HashMap;
use std::fmt;
use std::str::FromStr;
use std::time::Duration;
use anyhow::{Result, bail};
use axum::http::{HeaderName, HeaderValue, Method};
use educe::Educe;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{
@@ -157,6 +158,48 @@ impl TlsConfig {
}
}
/// One LoRA adapter to load before the server accepts traffic.
///
/// Mirrors `LoRAModulePath` in vllm/entrypoints/openai/models/protocol.py,
/// which is also the JSON shape the Python supervisor forwards in
/// `--args-json`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LoraModulePath {
/// Public model id the adapter is served under.
pub name: String,
/// Local path or Hugging Face repo id of the adapter.
pub path: String,
/// Base model reported as `parent` in `/v1/models`; defaults to the
/// primary served model name.
#[serde(default)]
pub base_model_name: Option<String>,
#[serde(default)]
pub is_3d_lora_weight: bool,
}
impl FromStr for LoraModulePath {
type Err = String;
/// Accept the two CLI forms Python's `LoRAParserAction` accepts:
/// `name=path`, or a JSON object with the struct's fields.
fn from_str(value: &str) -> Result<Self, Self::Err> {
if !value.trim_start().starts_with('{') {
let (name, path) = value
.split_once('=')
.filter(|(name, path)| !name.is_empty() && !path.is_empty())
.ok_or_else(|| format!("expected `name=path`, got `{value}`"))?;
return Ok(Self {
name: name.to_string(),
path: path.to_string(),
base_model_name: None,
is_3d_lora_weight: false,
});
}
serde_json::from_str(value)
.map_err(|e| format!("expected `name=path` or a JSON object: {e}"))
}
}
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
#[derive(Educe, Clone, PartialEq, Eq, Serialize)]
#[educe(Debug)]
@@ -192,6 +235,9 @@ pub struct Config {
/// Maximum number of input items allowed per prompt for each modality.
/// Unspecified modalities are unlimited.
pub limit_mm_per_prompt: MmLimitPerPrompt,
/// LoRA adapters loaded at startup, in order. Startup fails if any of
/// them cannot be loaded.
pub lora_modules: Vec<LoraModulePath>,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Optional maximum number of top log probabilities accepted by the
@@ -290,3 +336,66 @@ impl fmt::Debug for RedactedApiKeys<'_> {
fn fmt_redacted_api_keys(api_keys: &[String], f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&RedactedApiKeys(api_keys), f)
}
#[cfg(test)]
mod lora_module_path_tests {
use super::LoraModulePath;
#[test]
fn parses_name_equals_path() {
let module: LoraModulePath = "alice=charent/self_cognition_Alice".parse().unwrap();
assert_eq!(
module,
LoraModulePath {
name: "alice".to_string(),
path: "charent/self_cognition_Alice".to_string(),
base_model_name: None,
is_3d_lora_weight: false,
}
);
}
#[test]
fn parses_json_object() {
let module: LoraModulePath =
r#"{"name": "alice", "path": "/adapters/alice", "base_model_name": "base", "is_3d_lora_weight": true}"#
.parse()
.unwrap();
assert_eq!(
module,
LoraModulePath {
name: "alice".to_string(),
path: "/adapters/alice".to_string(),
base_model_name: Some("base".to_string()),
is_3d_lora_weight: true,
}
);
}
#[test]
fn parses_name_equals_path_with_comma() {
let module: LoraModulePath = "alice=/adapters/a,b".parse().unwrap();
assert_eq!(module.name, "alice");
assert_eq!(module.path, "/adapters/a,b");
}
#[test]
fn json_object_optional_fields_default() {
let module: LoraModulePath = r#"{"name": "a", "path": "org/a"}"#.parse().unwrap();
assert_eq!(module.base_model_name, None);
assert!(!module.is_3d_lora_weight);
}
#[test]
fn rejects_empty_name_or_path() {
assert!("=org/a".parse::<LoraModulePath>().is_err());
assert!("a=".parse::<LoraModulePath>().is_err());
}
#[test]
fn rejects_bare_path_and_bad_json() {
assert!("org/a".parse::<LoraModulePath>().is_err());
assert!(r#"{"name": "a"}"#.parse::<LoraModulePath>().is_err());
assert!(r#"[{"name": "a", "path": "org/a"}]"#.parse::<LoraModulePath>().is_err());
}
}
rust/src/server/src/grpc/control.rs
+9 -6
View File
@@ -13,6 +13,7 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
use super::{ControlServer, pb};
use crate::config::LoraModulePath;
use crate::lora::{LoadLoraError, LoraDisabledError, LoraPathAccessError, UnloadLoraError};
use crate::state::AppState;
@@ -147,7 +148,7 @@ fn lora_to_proto(adapter: &LoraRequest) -> pb::LoraAdapter {
fn load_lora_status(error: LoadLoraError) -> Status {
let code = match &error {
LoadLoraError::Disabled(_) => Code::FailedPrecondition,
LoadLoraError::InvalidRequest(_) => Code::InvalidArgument,
LoadLoraError::InvalidAdapter { .. } => Code::InvalidArgument,
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => Code::InvalidArgument,
LoadLoraError::PathAccess(LoraPathAccessError::InvalidConfiguration { .. }) => {
Code::Internal
@@ -247,11 +248,13 @@ impl pb::control_server::Control for ControlServiceImpl {
request: Request<pb::LoadLoraRequest>,
) -> Result<Response<pb::LoadLoraResponse>, Status> {
let request = request.into_inner();
let adapter = self
.state
.load_lora(request.lora_name, request.source_path, false, false)
.await
.map_err(load_lora_status)?;
let module = LoraModulePath {
name: request.lora_name,
path: request.source_path,
base_model_name: None,
is_3d_lora_weight: false,
};
let adapter = self.state.load_lora(module, false).await.map_err(load_lora_status)?;
Ok(Response::new(pb::LoadLoraResponse {
adapter: Some(lora_to_proto(&adapter)),
}))
rust/src/server/src/lib.rs
+30 -5
View File
@@ -29,7 +29,7 @@ use axum::body::Body;
use axum::http::Request;
pub use config::{
ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
HttpListenerMode, TlsConfig,
HttpListenerMode, LoraModulePath, TlsConfig,
};
use hyper::body::Incoming;
use hyper::server::conn::http1;
@@ -86,7 +86,7 @@ fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
}
/// Build the shared application state for one configured model and one engine
/// client.
/// client, including any static LoRA adapters from `--lora-modules`.
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
// If no served names are specified, fall back to the backend model path so
// that the API always has at least one valid model ID. Use the same primary
@@ -139,7 +139,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.with_tool_call_parser(config.tool_call_parser.clone())
.with_reasoning_parser(config.reasoning_parser.clone());
Ok(Arc::new(
let state = Arc::new(
AppState::new(served_model_names, chat)
.with_model_path(config.model.clone())
.with_api_server_options(config.api_server_options)
@@ -147,7 +147,31 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.with_api_keys(config.api_keys.clone())
.with_cors(config.cors.clone())
.with_profiler(config.profiler.clone()),
))
);
// Load operator-configured static LoRA adapters before serving, failing
// startup on any error like Python's `init_static_loras` does.
load_static_loras(&state, &config.lora_modules).await?;
Ok(state)
}
async fn load_static_loras(state: &AppState, modules: &[LoraModulePath]) -> Result<()> {
for module in modules {
let request = state.load_static_lora(module).await.with_context(|| {
format!(
"failed to load LoRA adapter `{}` from --lora-modules",
module.name
)
})?;
info!(
lora_name = %request.lora_name,
lora_int_id = request.lora_int_id,
lora_path = %request.lora_path,
"loaded static LoRA adapter"
);
}
Ok(())
}
/// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
@@ -182,7 +206,8 @@ where
.transpose()
.context("invalid TLS configuration")?;
// Also check shutdown during the (potentially long) startup handshake.
// Also check shutdown during the (potentially long) startup handshake and
// static LoRA loading.
let state = tokio::select! {
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
rust/src/server/src/lora.rs
+51 -13
View File
@@ -9,7 +9,9 @@ use thiserror::Error;
use thiserror_ext::Macro;
use tokio::sync::{Mutex, RwLock};
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::{LoraRequest, LoraRequestError};
use vllm_engine_core_client::protocol::lora::LoraRequest;
use crate::config::LoraModulePath;
const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
@@ -103,12 +105,12 @@ pub(crate) struct LoraManager {
#[error("engine was not started with LoRA enabled")]
pub(crate) struct LoraDisabledError;
#[derive(Debug, Error)]
#[derive(Debug, Error, Macro)]
pub(crate) enum LoadLoraError {
#[error(transparent)]
Disabled(#[from] LoraDisabledError),
#[error(transparent)]
InvalidRequest(#[from] LoraRequestError),
#[error("{message}")]
InvalidAdapter { message: String },
#[error(transparent)]
PathAccess(#[from] LoraPathAccessError),
#[error("LoRA adapter `{lora_name}` is already loaded")]
@@ -186,14 +188,49 @@ impl LoraManager {
&self,
engine_core_client: &EngineCoreClient,
base_model_names: &[String],
lora_name: String,
lora_path: String,
mut module: LoraModulePath,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Result<LoraRequest, LoadLoraError> {
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
let lora_path = validate_lora_path_access(&lora_path, allowed_prefixes.as_deref())?
.unwrap_or(lora_path);
if let Some(canonical_path) =
validate_lora_path_access(&module.path, allowed_prefixes.as_deref())?
{
module.path = canonical_path;
}
self.register(engine_core_client, base_model_names, module, load_inplace).await
}
/// Load an operator-configured adapter (`--lora-modules`). Unlike the
/// runtime endpoint, the path is trusted as given: no allowed-prefix
/// check applies.
pub async fn load_static_lora(
&self,
engine_core_client: &EngineCoreClient,
base_model_names: &[String],
module: &LoraModulePath,
) -> Result<LoraRequest, LoadLoraError> {
self.register(engine_core_client, base_model_names, module.clone(), false).await
}
async fn register(
&self,
engine_core_client: &EngineCoreClient,
base_model_names: &[String],
module: LoraModulePath,
load_inplace: bool,
) -> Result<LoraRequest, LoadLoraError> {
let LoraModulePath {
name: lora_name,
path: lora_path,
base_model_name,
is_3d_lora_weight,
} = module;
if lora_name.trim().is_empty() {
bail_invalid_adapter!("lora_name must not be empty");
}
if lora_path.trim().is_empty() {
bail_invalid_adapter!("lora_path must not be empty");
}
let _guard = self.update_lock.lock().await;
if base_model_names.iter().any(|name| name == &lora_name) {
return Err(LoadLoraError::BaseModelName { lora_name });
@@ -206,14 +243,15 @@ impl LoraManager {
let lora_int_id = existing_lora_int_id
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
let lora_request = LoraRequest::new(
lora_name.clone(),
let lora_request = LoraRequest {
lora_name: lora_name.clone(),
lora_int_id,
lora_path,
base_model_name,
tensorizer_config_dict: None,
load_inplace,
is_3d_lora_weight,
)
.map_err(LoadLoraError::InvalidRequest)?;
};
drop(requests);
let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| {
rust/src/server/src/routes/lora.rs
+9 -7
View File
@@ -8,6 +8,7 @@ use serde::Deserialize;
use thiserror_ext::AsReport;
use validator::Validate;
use crate::config::LoraModulePath;
use crate::error::ApiError;
use crate::lora::{LoadLoraError, LoraPathAccessError, UnloadLoraError};
use crate::routes::openai::utils::types::Normalizable;
@@ -39,7 +40,7 @@ fn load_lora_api_error(error: LoadLoraError) -> ApiError {
let message = error.to_report_string();
match error {
LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
LoadLoraError::InvalidRequest(_) => ApiError::invalid_request(message, None),
LoadLoraError::InvalidAdapter { .. } => ApiError::invalid_request(message, None),
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => {
ApiError::invalid_request(message, Some("lora_path"))
}
@@ -81,13 +82,14 @@ pub async fn load_lora_adapter(
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
) -> Result<String, ApiError> {
let lora_name = request.lora_name;
let module = LoraModulePath {
name: lora_name.clone(),
path: request.lora_path,
base_model_name: None,
is_3d_lora_weight: request.is_3d_lora_weight,
};
state
.load_lora(
lora_name.clone(),
request.lora_path,
request.load_inplace,
request.is_3d_lora_weight,
)
.load_lora(module, request.load_inplace)
.await
.map_err(load_lora_api_error)?;
rust/src/server/src/routes/tests.rs
+154 -9
View File
@@ -57,7 +57,8 @@ use super::{
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
render::build_router as build_render_router,
};
use crate::config::{ApiServerOptions, CorsConfig};
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
use crate::lora::LoadLoraError;
use crate::render::RenderState;
use crate::state::AppState;
@@ -941,6 +942,20 @@ async fn test_admin_app_with_ready_and_engine_script<F>(
ready: EngineCoreReadyResponse,
script: F,
) -> (axum::Router, MockEngineTask)
where
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
{
let (state, engine_task) = test_admin_state_with_ready_and_engine_script(ready, script).await;
(
build_router_with_dev_mode_and_lora(state, true, true),
engine_task,
)
}
async fn test_admin_state_with_ready_and_engine_script<F>(
ready: EngineCoreReadyResponse,
script: F,
) -> (Arc<AppState>, MockEngineTask)
where
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
{
@@ -968,18 +983,148 @@ where
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
(
build_router_with_dev_mode_and_lora(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
true,
),
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
engine_task,
)
}
/// Engine script that answers one `add_lora` utility call with `loaded` and
/// hands the decoded LoRA request tuple to `check`.
fn add_lora_script(
loaded: bool,
check: impl FnOnce(&[Value]) + Send + 'static,
) -> impl for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static
{
move |dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("add_lora"));
let args = array[3].as_array().expect("utility args");
check(args[0].as_array().expect("lora request tuple"));
send_outputs(push, utility_outputs(call_id, utility_result_value(loaded))).await;
})
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn static_lora_module_loads_and_lists_with_parent() {
// An absolute local path: the runtime endpoint would reject it without
// VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES, static config trusts it.
let mut ready = default_ready_response();
ready.supports_lora = true;
ready.max_loras = 4;
let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
ready,
add_lora_script(true, |lora| {
assert_eq!(lora[0], Value::from("alice"));
assert_eq!(lora[1], Value::from(1));
assert_eq!(lora[2], Value::from("/adapters/alice"));
assert_eq!(lora[3], Value::from("base-model"));
}),
)
.await;
let module = LoraModulePath {
name: "alice".to_string(),
path: "/adapters/alice".to_string(),
base_model_name: Some("base-model".to_string()),
is_3d_lora_weight: false,
};
let request = state.load_static_lora(&module).await.expect("load static lora");
assert_eq!(request.lora_name, "alice");
assert_eq!(request.lora_int_id, 1);
let mut app = build_router_with_dev_mode_and_lora(state, true, false);
let models = app
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
.await
.expect("call app");
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["data"][1]["id"], "alice");
assert_eq!(json["data"][1]["root"], "/adapters/alice");
assert_eq!(json["data"][1]["parent"], "base-model");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn static_lora_module_requires_lora_enabled_engine() {
let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
default_ready_response(),
|_dealer, _push| boxed_test_future(async {}),
)
.await;
let module = LoraModulePath {
name: "alice".to_string(),
path: "org/alice".to_string(),
base_model_name: None,
is_3d_lora_weight: false,
};
let error = state.load_static_lora(&module).await.expect_err("engine has no lora");
assert!(matches!(error, LoadLoraError::Disabled(_)), "{error:?}");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn static_lora_module_fails_when_engine_rejects_it() {
let mut ready = default_ready_response();
ready.supports_lora = true;
ready.max_loras = 4;
let (state, _engine_task) =
test_admin_state_with_ready_and_engine_script(ready, add_lora_script(false, |_| {})).await;
let module = LoraModulePath {
name: "alice".to_string(),
path: "org/alice".to_string(),
base_model_name: None,
is_3d_lora_weight: false,
};
let error = state.load_static_lora(&module).await.expect_err("engine rejected");
assert!(
matches!(error, LoadLoraError::NotLoaded { .. }),
"{error:?}"
);
assert!(state.served_lora_requests().await.is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn static_lora_module_rejects_empty_name_or_path() {
// The JSON form of `--lora-modules` is not validated at parse time; the
// manager rejects empty fields before any engine RPC.
let mut ready = default_ready_response();
ready.supports_lora = true;
ready.max_loras = 4;
let (state, _engine_task) =
test_admin_state_with_ready_and_engine_script(ready, |_dealer, _push| {
boxed_test_future(async {})
})
.await;
for (name, path) in [("", "org/alice"), ("alice", "")] {
let module = LoraModulePath {
name: name.to_string(),
path: path.to_string(),
base_model_name: None,
is_3d_lora_weight: false,
};
let error = state.load_static_lora(&module).await.expect_err("empty field");
assert!(
matches!(error, LoadLoraError::InvalidAdapter { .. }),
"{error:?}"
);
}
}
async fn test_app_with_engine_handle() -> (axum::Router, MockEngineTask) {
test_app_with_stream_output_specs(default_stream_output_specs()).await
}
rust/src/server/src/state.rs
+17 -7
View File
@@ -14,7 +14,7 @@ use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
use crate::config::{ApiServerOptions, CorsConfig};
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
use crate::lora::{
LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError,
};
@@ -181,24 +181,34 @@ impl AppState {
/// Returns error if the engine was started without LoRA support.
pub async fn load_lora(
&self,
lora_name: String,
lora_path: String,
module: LoraModulePath,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Result<LoraRequest, LoadLoraError> {
self.ensure_lora_enabled()?;
self.lora_manager
.load_lora(
self.engine_core_client(),
&self.served_model_names,
lora_name,
lora_path,
module,
load_inplace,
is_3d_lora_weight,
)
.await
}
/// Load one adapter from `--lora-modules` and register it as a public
/// model name.
///
/// Returns error if the engine was started without LoRA support.
pub async fn load_static_lora(
&self,
module: &LoraModulePath,
) -> Result<LoraRequest, LoadLoraError> {
self.ensure_lora_enabled()?;
self.lora_manager
.load_static_lora(self.engine_core_client(), &self.served_model_names, module)
.await
}
/// Remove one dynamic LoRA adapter from the engine and public model
/// registry.
///