[Rust Frontend] Support --lora-modules for static adapter loading (#54837)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Will Eaton <weaton@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+11
-2
@@ -29,8 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
|
|||||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||||
use vllm_server::{
|
use vllm_server::{
|
||||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
|
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
|
||||||
TlsConfig,
|
RendererSelection, TlsConfig,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::cli::unsupported::UnsupportedArgs;
|
use crate::cli::unsupported::UnsupportedArgs;
|
||||||
@@ -269,6 +269,13 @@ pub struct SharedRuntimeArgs {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||||
|
|
||||||
|
/// LoRA adapters to load before serving, each as `name=path` or a JSON
|
||||||
|
/// object: `{"name": "name", "path": "lora_path", "base_model_name": "id"}`.
|
||||||
|
/// Requires `--enable-lora`; startup fails if any adapter cannot be loaded.
|
||||||
|
#[arg(long, num_args = 1.., value_name = "MODULE")]
|
||||||
|
#[serde(default)]
|
||||||
|
pub lora_modules: Vec<LoraModulePath>,
|
||||||
|
|
||||||
/// The format to render message content within a chat template.
|
/// The format to render message content within a chat template.
|
||||||
///
|
///
|
||||||
/// * "auto" detects the format from the template
|
/// * "auto" detects the format from the template
|
||||||
@@ -498,6 +505,7 @@ impl SharedRuntimeArgs {
|
|||||||
chat_template: self.chat_template,
|
chat_template: self.chat_template,
|
||||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||||
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
||||||
|
lora_modules: self.lora_modules,
|
||||||
chat_template_content_format: self.chat_template_content_format,
|
chat_template_content_format: self.chat_template_content_format,
|
||||||
max_logprobs: self.max_logprobs,
|
max_logprobs: self.max_logprobs,
|
||||||
api_server_options,
|
api_server_options,
|
||||||
@@ -552,6 +560,7 @@ impl SharedRuntimeArgs {
|
|||||||
chat_template: self.chat_template,
|
chat_template: self.chat_template,
|
||||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||||
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
||||||
|
lora_modules: self.lora_modules,
|
||||||
chat_template_content_format: self.chat_template_content_format,
|
chat_template_content_format: self.chat_template_content_format,
|
||||||
max_logprobs: self.max_logprobs,
|
max_logprobs: self.max_logprobs,
|
||||||
api_server_options,
|
api_server_options,
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
use expect_test::expect;
|
use expect_test::expect;
|
||||||
use vllm_engine_core_client::TransportMode;
|
use vllm_engine_core_client::TransportMode;
|
||||||
use vllm_server::{
|
use vllm_server::{
|
||||||
Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
|
Config, GenerationConfigMode, HttpListenerMode, LoraModulePath, ParserSelection,
|
||||||
|
RendererSelection,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{BenchCommand, Cli, Command};
|
use super::{BenchCommand, Cli, Command};
|
||||||
@@ -103,6 +104,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
enable_log_requests: false,
|
enable_log_requests: false,
|
||||||
enable_prompt_tokens_details: false,
|
enable_prompt_tokens_details: false,
|
||||||
@@ -623,6 +625,103 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
|
|||||||
assert!(config.api_server_options.enable_request_id_headers);
|
assert!(config.api_server_options.enable_request_id_headers);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_parse_lora_modules_in_both_forms() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"serve",
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
"--enable-lora",
|
||||||
|
"--lora-modules",
|
||||||
|
"alice=charent/self_cognition_Alice",
|
||||||
|
r#"{"name": "bob", "path": "/adapters/bob", "base_model_name": "base"}"#,
|
||||||
|
"--lora-modules",
|
||||||
|
"carol=org/carol",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
// Frontend-only flag: the managed Python engine must not see it.
|
||||||
|
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
|
||||||
|
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||||
|
expect![[r#"
|
||||||
|
[
|
||||||
|
LoraModulePath {
|
||||||
|
name: "alice",
|
||||||
|
path: "charent/self_cognition_Alice",
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
},
|
||||||
|
LoraModulePath {
|
||||||
|
name: "bob",
|
||||||
|
path: "/adapters/bob",
|
||||||
|
base_model_name: Some(
|
||||||
|
"base",
|
||||||
|
),
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
},
|
||||||
|
LoraModulePath {
|
||||||
|
name: "carol",
|
||||||
|
path: "org/carol",
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.lora_modules);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_reject_malformed_lora_module() {
|
||||||
|
let error = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"serve",
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
"--lora-modules",
|
||||||
|
"alice",
|
||||||
|
])
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
expect![[r#"
|
||||||
|
error: invalid value 'alice' for '--lora-modules <MODULE>...': expected `name=path`, got `alice`
|
||||||
|
|
||||||
|
For more information, try '--help'.
|
||||||
|
"#]].assert_eq(&error.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn frontend_args_json_passes_lora_modules_into_config() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"frontend",
|
||||||
|
"--listen-fd",
|
||||||
|
"3",
|
||||||
|
"--input-address",
|
||||||
|
"ipc:///tmp/input.sock",
|
||||||
|
"--output-address",
|
||||||
|
"ipc:///tmp/output.sock",
|
||||||
|
"--args-json",
|
||||||
|
r#"{"model_tag":"Qwen/Qwen3-0.6B","lora_modules":[{"name":"alice","path":"org/alice","base_model_name":null,"is_3d_lora_weight":true}]}"#,
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Frontend(args) = cli.command else {
|
||||||
|
panic!("expected frontend args");
|
||||||
|
};
|
||||||
|
let config = args.into_config();
|
||||||
|
assert_eq!(
|
||||||
|
config.lora_modules,
|
||||||
|
vec![LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "org/alice".to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: true,
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serve_passes_api_keys_into_config() {
|
fn serve_passes_api_keys_into_config() {
|
||||||
let cli = Cli::try_parse_from([
|
let cli = Cli::try_parse_from([
|
||||||
@@ -822,6 +921,7 @@ fn frontend_args_accept_json() {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
enable_log_requests: false,
|
enable_log_requests: false,
|
||||||
enable_prompt_tokens_details: false,
|
enable_prompt_tokens_details: false,
|
||||||
@@ -1416,6 +1516,7 @@ fn serve_args_accept_handshake_aliases() {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
enable_log_requests: false,
|
enable_log_requests: false,
|
||||||
enable_prompt_tokens_details: false,
|
enable_prompt_tokens_details: false,
|
||||||
@@ -1561,6 +1662,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
api_server_options: ApiServerOptions {
|
api_server_options: ApiServerOptions {
|
||||||
@@ -1647,6 +1749,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
api_server_options: ApiServerOptions {
|
api_server_options: ApiServerOptions {
|
||||||
@@ -1754,6 +1857,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: {},
|
limit_mm_per_prompt: {},
|
||||||
|
lora_modules: [],
|
||||||
chat_template_content_format: Auto,
|
chat_template_content_format: Auto,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
api_server_options: ApiServerOptions {
|
api_server_options: ApiServerOptions {
|
||||||
|
|||||||
@@ -353,13 +353,6 @@ pub struct EngineUnsupportedArgs {
|
|||||||
#[serde_with::skip_serializing_none]
|
#[serde_with::skip_serializing_none]
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
|
||||||
pub struct ServerUnsupportedArgs {
|
pub struct ServerUnsupportedArgs {
|
||||||
/// LoRA modules configurations in either 'name=path' format or JSON format
|
|
||||||
/// or JSON list format. Example (old format): `'name=path'` Example (new
|
|
||||||
/// format): `{"name": "name", "path": "lora_path",
|
|
||||||
/// "base_model_name": "id"}`
|
|
||||||
#[arg(long)]
|
|
||||||
pub lora_modules: Option<Unsupported>,
|
|
||||||
|
|
||||||
/// Whether to trust the chat template provided in the request. If False,
|
/// Whether to trust the chat template provided in the request. If False,
|
||||||
/// the server will always use the chat template specified by
|
/// the server will always use the chat template specified by
|
||||||
/// `--chat-template` or the ones from tokenizer.
|
/// `--chat-template` or the ones from tokenizer.
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||||
use thiserror::Error;
|
|
||||||
|
|
||||||
use crate::protocol::OpaqueValue;
|
use crate::protocol::OpaqueValue;
|
||||||
|
|
||||||
@@ -24,43 +23,3 @@ pub struct LoraRequest {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub is_3d_lora_weight: bool,
|
pub is_3d_lora_weight: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Error, PartialEq, Eq)]
|
|
||||||
pub enum LoraRequestError {
|
|
||||||
#[error("lora_name must not be empty")]
|
|
||||||
EmptyName,
|
|
||||||
#[error("lora_int_id must be nonzero")]
|
|
||||||
ZeroId,
|
|
||||||
#[error("lora_path must not be empty")]
|
|
||||||
EmptyPath,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LoraRequest {
|
|
||||||
pub fn new(
|
|
||||||
lora_name: String,
|
|
||||||
lora_int_id: u64,
|
|
||||||
lora_path: String,
|
|
||||||
load_inplace: bool,
|
|
||||||
is_3d_lora_weight: bool,
|
|
||||||
) -> Result<Self, LoraRequestError> {
|
|
||||||
if lora_name.trim().is_empty() {
|
|
||||||
return Err(LoraRequestError::EmptyName);
|
|
||||||
}
|
|
||||||
if lora_int_id == 0 {
|
|
||||||
return Err(LoraRequestError::ZeroId);
|
|
||||||
}
|
|
||||||
if lora_path.trim().is_empty() {
|
|
||||||
return Err(LoraRequestError::EmptyPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Self {
|
|
||||||
lora_name,
|
|
||||||
lora_int_id,
|
|
||||||
lora_path,
|
|
||||||
base_model_name: None,
|
|
||||||
tensorizer_config_dict: None,
|
|
||||||
load_inplace,
|
|
||||||
is_3d_lora_weight,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ async fn main() -> Result<()> {
|
|||||||
chat_template: None,
|
chat_template: None,
|
||||||
default_chat_template_kwargs: None,
|
default_chat_template_kwargs: None,
|
||||||
limit_mm_per_prompt: HashMap::new(),
|
limit_mm_per_prompt: HashMap::new(),
|
||||||
|
lora_modules: Vec::new(),
|
||||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
api_server_options: ApiServerOptions::default(),
|
api_server_options: ApiServerOptions::default(),
|
||||||
|
|||||||
@@ -3,12 +3,13 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
use std::str::FromStr;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{Result, bail};
|
use anyhow::{Result, bail};
|
||||||
use axum::http::{HeaderName, HeaderValue, Method};
|
use axum::http::{HeaderName, HeaderValue, Method};
|
||||||
use educe::Educe;
|
use educe::Educe;
|
||||||
use serde::Serialize;
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||||
use vllm_chat::{
|
use vllm_chat::{
|
||||||
@@ -157,6 +158,48 @@ impl TlsConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One LoRA adapter to load before the server accepts traffic.
|
||||||
|
///
|
||||||
|
/// Mirrors `LoRAModulePath` in vllm/entrypoints/openai/models/protocol.py,
|
||||||
|
/// which is also the JSON shape the Python supervisor forwards in
|
||||||
|
/// `--args-json`.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct LoraModulePath {
|
||||||
|
/// Public model id the adapter is served under.
|
||||||
|
pub name: String,
|
||||||
|
/// Local path or Hugging Face repo id of the adapter.
|
||||||
|
pub path: String,
|
||||||
|
/// Base model reported as `parent` in `/v1/models`; defaults to the
|
||||||
|
/// primary served model name.
|
||||||
|
#[serde(default)]
|
||||||
|
pub base_model_name: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub is_3d_lora_weight: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FromStr for LoraModulePath {
|
||||||
|
type Err = String;
|
||||||
|
|
||||||
|
/// Accept the two CLI forms Python's `LoRAParserAction` accepts:
|
||||||
|
/// `name=path`, or a JSON object with the struct's fields.
|
||||||
|
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||||
|
if !value.trim_start().starts_with('{') {
|
||||||
|
let (name, path) = value
|
||||||
|
.split_once('=')
|
||||||
|
.filter(|(name, path)| !name.is_empty() && !path.is_empty())
|
||||||
|
.ok_or_else(|| format!("expected `name=path`, got `{value}`"))?;
|
||||||
|
return Ok(Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
path: path.to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
serde_json::from_str(value)
|
||||||
|
.map_err(|e| format!("expected `name=path` or a JSON object: {e}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
|
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
|
||||||
#[derive(Educe, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Educe, Clone, PartialEq, Eq, Serialize)]
|
||||||
#[educe(Debug)]
|
#[educe(Debug)]
|
||||||
@@ -192,6 +235,9 @@ pub struct Config {
|
|||||||
/// Maximum number of input items allowed per prompt for each modality.
|
/// Maximum number of input items allowed per prompt for each modality.
|
||||||
/// Unspecified modalities are unlimited.
|
/// Unspecified modalities are unlimited.
|
||||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||||
|
/// LoRA adapters loaded at startup, in order. Startup fails if any of
|
||||||
|
/// them cannot be loaded.
|
||||||
|
pub lora_modules: Vec<LoraModulePath>,
|
||||||
/// How to serialize `message.content` for chat-template rendering.
|
/// How to serialize `message.content` for chat-template rendering.
|
||||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||||
/// Optional maximum number of top log probabilities accepted by the
|
/// Optional maximum number of top log probabilities accepted by the
|
||||||
@@ -290,3 +336,66 @@ impl fmt::Debug for RedactedApiKeys<'_> {
|
|||||||
fn fmt_redacted_api_keys(api_keys: &[String], f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt_redacted_api_keys(api_keys: &[String], f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
fmt::Debug::fmt(&RedactedApiKeys(api_keys), f)
|
fmt::Debug::fmt(&RedactedApiKeys(api_keys), f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod lora_module_path_tests {
|
||||||
|
use super::LoraModulePath;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_name_equals_path() {
|
||||||
|
let module: LoraModulePath = "alice=charent/self_cognition_Alice".parse().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
module,
|
||||||
|
LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "charent/self_cognition_Alice".to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_json_object() {
|
||||||
|
let module: LoraModulePath =
|
||||||
|
r#"{"name": "alice", "path": "/adapters/alice", "base_model_name": "base", "is_3d_lora_weight": true}"#
|
||||||
|
.parse()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
module,
|
||||||
|
LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "/adapters/alice".to_string(),
|
||||||
|
base_model_name: Some("base".to_string()),
|
||||||
|
is_3d_lora_weight: true,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_name_equals_path_with_comma() {
|
||||||
|
let module: LoraModulePath = "alice=/adapters/a,b".parse().unwrap();
|
||||||
|
assert_eq!(module.name, "alice");
|
||||||
|
assert_eq!(module.path, "/adapters/a,b");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn json_object_optional_fields_default() {
|
||||||
|
let module: LoraModulePath = r#"{"name": "a", "path": "org/a"}"#.parse().unwrap();
|
||||||
|
assert_eq!(module.base_model_name, None);
|
||||||
|
assert!(!module.is_3d_lora_weight);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_empty_name_or_path() {
|
||||||
|
assert!("=org/a".parse::<LoraModulePath>().is_err());
|
||||||
|
assert!("a=".parse::<LoraModulePath>().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_bare_path_and_bad_json() {
|
||||||
|
assert!("org/a".parse::<LoraModulePath>().is_err());
|
||||||
|
assert!(r#"{"name": "a"}"#.parse::<LoraModulePath>().is_err());
|
||||||
|
assert!(r#"[{"name": "a", "path": "org/a"}]"#.parse::<LoraModulePath>().is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
|
|||||||
use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
|
use vllm_engine_core_client::protocol::utility::PauseMode as EnginePauseMode;
|
||||||
|
|
||||||
use super::{ControlServer, pb};
|
use super::{ControlServer, pb};
|
||||||
|
use crate::config::LoraModulePath;
|
||||||
use crate::lora::{LoadLoraError, LoraDisabledError, LoraPathAccessError, UnloadLoraError};
|
use crate::lora::{LoadLoraError, LoraDisabledError, LoraPathAccessError, UnloadLoraError};
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
@@ -147,7 +148,7 @@ fn lora_to_proto(adapter: &LoraRequest) -> pb::LoraAdapter {
|
|||||||
fn load_lora_status(error: LoadLoraError) -> Status {
|
fn load_lora_status(error: LoadLoraError) -> Status {
|
||||||
let code = match &error {
|
let code = match &error {
|
||||||
LoadLoraError::Disabled(_) => Code::FailedPrecondition,
|
LoadLoraError::Disabled(_) => Code::FailedPrecondition,
|
||||||
LoadLoraError::InvalidRequest(_) => Code::InvalidArgument,
|
LoadLoraError::InvalidAdapter { .. } => Code::InvalidArgument,
|
||||||
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => Code::InvalidArgument,
|
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => Code::InvalidArgument,
|
||||||
LoadLoraError::PathAccess(LoraPathAccessError::InvalidConfiguration { .. }) => {
|
LoadLoraError::PathAccess(LoraPathAccessError::InvalidConfiguration { .. }) => {
|
||||||
Code::Internal
|
Code::Internal
|
||||||
@@ -247,11 +248,13 @@ impl pb::control_server::Control for ControlServiceImpl {
|
|||||||
request: Request<pb::LoadLoraRequest>,
|
request: Request<pb::LoadLoraRequest>,
|
||||||
) -> Result<Response<pb::LoadLoraResponse>, Status> {
|
) -> Result<Response<pb::LoadLoraResponse>, Status> {
|
||||||
let request = request.into_inner();
|
let request = request.into_inner();
|
||||||
let adapter = self
|
let module = LoraModulePath {
|
||||||
.state
|
name: request.lora_name,
|
||||||
.load_lora(request.lora_name, request.source_path, false, false)
|
path: request.source_path,
|
||||||
.await
|
base_model_name: None,
|
||||||
.map_err(load_lora_status)?;
|
is_3d_lora_weight: false,
|
||||||
|
};
|
||||||
|
let adapter = self.state.load_lora(module, false).await.map_err(load_lora_status)?;
|
||||||
Ok(Response::new(pb::LoadLoraResponse {
|
Ok(Response::new(pb::LoadLoraResponse {
|
||||||
adapter: Some(lora_to_proto(&adapter)),
|
adapter: Some(lora_to_proto(&adapter)),
|
||||||
}))
|
}))
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ use axum::body::Body;
|
|||||||
use axum::http::Request;
|
use axum::http::Request;
|
||||||
pub use config::{
|
pub use config::{
|
||||||
ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
|
ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
|
||||||
HttpListenerMode, TlsConfig,
|
HttpListenerMode, LoraModulePath, TlsConfig,
|
||||||
};
|
};
|
||||||
use hyper::body::Incoming;
|
use hyper::body::Incoming;
|
||||||
use hyper::server::conn::http1;
|
use hyper::server::conn::http1;
|
||||||
@@ -86,7 +86,7 @@ fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Build the shared application state for one configured model and one engine
|
/// Build the shared application state for one configured model and one engine
|
||||||
/// client.
|
/// client, including any static LoRA adapters from `--lora-modules`.
|
||||||
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||||
// If no served names are specified, fall back to the backend model path so
|
// If no served names are specified, fall back to the backend model path so
|
||||||
// that the API always has at least one valid model ID. Use the same primary
|
// that the API always has at least one valid model ID. Use the same primary
|
||||||
@@ -139,7 +139,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
|||||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||||
.with_reasoning_parser(config.reasoning_parser.clone());
|
.with_reasoning_parser(config.reasoning_parser.clone());
|
||||||
|
|
||||||
Ok(Arc::new(
|
let state = Arc::new(
|
||||||
AppState::new(served_model_names, chat)
|
AppState::new(served_model_names, chat)
|
||||||
.with_model_path(config.model.clone())
|
.with_model_path(config.model.clone())
|
||||||
.with_api_server_options(config.api_server_options)
|
.with_api_server_options(config.api_server_options)
|
||||||
@@ -147,7 +147,31 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
|||||||
.with_api_keys(config.api_keys.clone())
|
.with_api_keys(config.api_keys.clone())
|
||||||
.with_cors(config.cors.clone())
|
.with_cors(config.cors.clone())
|
||||||
.with_profiler(config.profiler.clone()),
|
.with_profiler(config.profiler.clone()),
|
||||||
))
|
);
|
||||||
|
|
||||||
|
// Load operator-configured static LoRA adapters before serving, failing
|
||||||
|
// startup on any error like Python's `init_static_loras` does.
|
||||||
|
load_static_loras(&state, &config.lora_modules).await?;
|
||||||
|
|
||||||
|
Ok(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_static_loras(state: &AppState, modules: &[LoraModulePath]) -> Result<()> {
|
||||||
|
for module in modules {
|
||||||
|
let request = state.load_static_lora(module).await.with_context(|| {
|
||||||
|
format!(
|
||||||
|
"failed to load LoRA adapter `{}` from --lora-modules",
|
||||||
|
module.name
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
info!(
|
||||||
|
lora_name = %request.lora_name,
|
||||||
|
lora_int_id = request.lora_int_id,
|
||||||
|
lora_path = %request.lora_path,
|
||||||
|
"loaded static LoRA adapter"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
|
/// Run the OpenAI-compatible HTTP server until the supplied shutdown token is
|
||||||
@@ -182,7 +206,8 @@ where
|
|||||||
.transpose()
|
.transpose()
|
||||||
.context("invalid TLS configuration")?;
|
.context("invalid TLS configuration")?;
|
||||||
|
|
||||||
// Also check shutdown during the (potentially long) startup handshake.
|
// Also check shutdown during the (potentially long) startup handshake and
|
||||||
|
// static LoRA loading.
|
||||||
let state = tokio::select! {
|
let state = tokio::select! {
|
||||||
result = build_state(&config) => result?,
|
result = build_state(&config) => result?,
|
||||||
_ = shutdown.cancelled() => return Ok(()),
|
_ = shutdown.cancelled() => return Ok(()),
|
||||||
|
|||||||
+51
-13
@@ -9,7 +9,9 @@ use thiserror::Error;
|
|||||||
use thiserror_ext::Macro;
|
use thiserror_ext::Macro;
|
||||||
use tokio::sync::{Mutex, RwLock};
|
use tokio::sync::{Mutex, RwLock};
|
||||||
use vllm_engine_core_client::EngineCoreClient;
|
use vllm_engine_core_client::EngineCoreClient;
|
||||||
use vllm_engine_core_client::protocol::lora::{LoraRequest, LoraRequestError};
|
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||||
|
|
||||||
|
use crate::config::LoraModulePath;
|
||||||
|
|
||||||
const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
|
const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
|
||||||
|
|
||||||
@@ -103,12 +105,12 @@ pub(crate) struct LoraManager {
|
|||||||
#[error("engine was not started with LoRA enabled")]
|
#[error("engine was not started with LoRA enabled")]
|
||||||
pub(crate) struct LoraDisabledError;
|
pub(crate) struct LoraDisabledError;
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error, Macro)]
|
||||||
pub(crate) enum LoadLoraError {
|
pub(crate) enum LoadLoraError {
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Disabled(#[from] LoraDisabledError),
|
Disabled(#[from] LoraDisabledError),
|
||||||
#[error(transparent)]
|
#[error("{message}")]
|
||||||
InvalidRequest(#[from] LoraRequestError),
|
InvalidAdapter { message: String },
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
PathAccess(#[from] LoraPathAccessError),
|
PathAccess(#[from] LoraPathAccessError),
|
||||||
#[error("LoRA adapter `{lora_name}` is already loaded")]
|
#[error("LoRA adapter `{lora_name}` is already loaded")]
|
||||||
@@ -186,14 +188,49 @@ impl LoraManager {
|
|||||||
&self,
|
&self,
|
||||||
engine_core_client: &EngineCoreClient,
|
engine_core_client: &EngineCoreClient,
|
||||||
base_model_names: &[String],
|
base_model_names: &[String],
|
||||||
lora_name: String,
|
mut module: LoraModulePath,
|
||||||
lora_path: String,
|
|
||||||
load_inplace: bool,
|
load_inplace: bool,
|
||||||
is_3d_lora_weight: bool,
|
|
||||||
) -> Result<LoraRequest, LoadLoraError> {
|
) -> Result<LoraRequest, LoadLoraError> {
|
||||||
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
|
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
|
||||||
let lora_path = validate_lora_path_access(&lora_path, allowed_prefixes.as_deref())?
|
if let Some(canonical_path) =
|
||||||
.unwrap_or(lora_path);
|
validate_lora_path_access(&module.path, allowed_prefixes.as_deref())?
|
||||||
|
{
|
||||||
|
module.path = canonical_path;
|
||||||
|
}
|
||||||
|
self.register(engine_core_client, base_model_names, module, load_inplace).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load an operator-configured adapter (`--lora-modules`). Unlike the
|
||||||
|
/// runtime endpoint, the path is trusted as given: no allowed-prefix
|
||||||
|
/// check applies.
|
||||||
|
pub async fn load_static_lora(
|
||||||
|
&self,
|
||||||
|
engine_core_client: &EngineCoreClient,
|
||||||
|
base_model_names: &[String],
|
||||||
|
module: &LoraModulePath,
|
||||||
|
) -> Result<LoraRequest, LoadLoraError> {
|
||||||
|
self.register(engine_core_client, base_model_names, module.clone(), false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn register(
|
||||||
|
&self,
|
||||||
|
engine_core_client: &EngineCoreClient,
|
||||||
|
base_model_names: &[String],
|
||||||
|
module: LoraModulePath,
|
||||||
|
load_inplace: bool,
|
||||||
|
) -> Result<LoraRequest, LoadLoraError> {
|
||||||
|
let LoraModulePath {
|
||||||
|
name: lora_name,
|
||||||
|
path: lora_path,
|
||||||
|
base_model_name,
|
||||||
|
is_3d_lora_weight,
|
||||||
|
} = module;
|
||||||
|
if lora_name.trim().is_empty() {
|
||||||
|
bail_invalid_adapter!("lora_name must not be empty");
|
||||||
|
}
|
||||||
|
if lora_path.trim().is_empty() {
|
||||||
|
bail_invalid_adapter!("lora_path must not be empty");
|
||||||
|
}
|
||||||
let _guard = self.update_lock.lock().await;
|
let _guard = self.update_lock.lock().await;
|
||||||
if base_model_names.iter().any(|name| name == &lora_name) {
|
if base_model_names.iter().any(|name| name == &lora_name) {
|
||||||
return Err(LoadLoraError::BaseModelName { lora_name });
|
return Err(LoadLoraError::BaseModelName { lora_name });
|
||||||
@@ -206,14 +243,15 @@ impl LoraManager {
|
|||||||
|
|
||||||
let lora_int_id = existing_lora_int_id
|
let lora_int_id = existing_lora_int_id
|
||||||
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
|
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
|
||||||
let lora_request = LoraRequest::new(
|
let lora_request = LoraRequest {
|
||||||
lora_name.clone(),
|
lora_name: lora_name.clone(),
|
||||||
lora_int_id,
|
lora_int_id,
|
||||||
lora_path,
|
lora_path,
|
||||||
|
base_model_name,
|
||||||
|
tensorizer_config_dict: None,
|
||||||
load_inplace,
|
load_inplace,
|
||||||
is_3d_lora_weight,
|
is_3d_lora_weight,
|
||||||
)
|
};
|
||||||
.map_err(LoadLoraError::InvalidRequest)?;
|
|
||||||
drop(requests);
|
drop(requests);
|
||||||
|
|
||||||
let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| {
|
let loaded = engine_core_client.add_lora(&lora_request).await.map_err(|source| {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use serde::Deserialize;
|
|||||||
use thiserror_ext::AsReport;
|
use thiserror_ext::AsReport;
|
||||||
use validator::Validate;
|
use validator::Validate;
|
||||||
|
|
||||||
|
use crate::config::LoraModulePath;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::lora::{LoadLoraError, LoraPathAccessError, UnloadLoraError};
|
use crate::lora::{LoadLoraError, LoraPathAccessError, UnloadLoraError};
|
||||||
use crate::routes::openai::utils::types::Normalizable;
|
use crate::routes::openai::utils::types::Normalizable;
|
||||||
@@ -39,7 +40,7 @@ fn load_lora_api_error(error: LoadLoraError) -> ApiError {
|
|||||||
let message = error.to_report_string();
|
let message = error.to_report_string();
|
||||||
match error {
|
match error {
|
||||||
LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
|
LoadLoraError::Disabled(_) => ApiError::invalid_request(message, None),
|
||||||
LoadLoraError::InvalidRequest(_) => ApiError::invalid_request(message, None),
|
LoadLoraError::InvalidAdapter { .. } => ApiError::invalid_request(message, None),
|
||||||
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => {
|
LoadLoraError::PathAccess(LoraPathAccessError::InvalidPath { .. }) => {
|
||||||
ApiError::invalid_request(message, Some("lora_path"))
|
ApiError::invalid_request(message, Some("lora_path"))
|
||||||
}
|
}
|
||||||
@@ -81,13 +82,14 @@ pub async fn load_lora_adapter(
|
|||||||
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
|
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
|
||||||
) -> Result<String, ApiError> {
|
) -> Result<String, ApiError> {
|
||||||
let lora_name = request.lora_name;
|
let lora_name = request.lora_name;
|
||||||
|
let module = LoraModulePath {
|
||||||
|
name: lora_name.clone(),
|
||||||
|
path: request.lora_path,
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: request.is_3d_lora_weight,
|
||||||
|
};
|
||||||
state
|
state
|
||||||
.load_lora(
|
.load_lora(module, request.load_inplace)
|
||||||
lora_name.clone(),
|
|
||||||
request.lora_path,
|
|
||||||
request.load_inplace,
|
|
||||||
request.is_3d_lora_weight,
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.map_err(load_lora_api_error)?;
|
.map_err(load_lora_api_error)?;
|
||||||
|
|
||||||
|
|||||||
@@ -57,7 +57,8 @@ use super::{
|
|||||||
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
||||||
render::build_router as build_render_router,
|
render::build_router as build_render_router,
|
||||||
};
|
};
|
||||||
use crate::config::{ApiServerOptions, CorsConfig};
|
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
|
||||||
|
use crate::lora::LoadLoraError;
|
||||||
use crate::render::RenderState;
|
use crate::render::RenderState;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
@@ -941,6 +942,20 @@ async fn test_admin_app_with_ready_and_engine_script<F>(
|
|||||||
ready: EngineCoreReadyResponse,
|
ready: EngineCoreReadyResponse,
|
||||||
script: F,
|
script: F,
|
||||||
) -> (axum::Router, MockEngineTask)
|
) -> (axum::Router, MockEngineTask)
|
||||||
|
where
|
||||||
|
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
|
||||||
|
{
|
||||||
|
let (state, engine_task) = test_admin_state_with_ready_and_engine_script(ready, script).await;
|
||||||
|
(
|
||||||
|
build_router_with_dev_mode_and_lora(state, true, true),
|
||||||
|
engine_task,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_admin_state_with_ready_and_engine_script<F>(
|
||||||
|
ready: EngineCoreReadyResponse,
|
||||||
|
script: F,
|
||||||
|
) -> (Arc<AppState>, MockEngineTask)
|
||||||
where
|
where
|
||||||
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
|
F: for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static,
|
||||||
{
|
{
|
||||||
@@ -968,18 +983,148 @@ where
|
|||||||
|
|
||||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||||
(
|
(
|
||||||
build_router_with_dev_mode_and_lora(
|
Arc::new(AppState::new(
|
||||||
Arc::new(AppState::new(
|
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
chat,
|
||||||
chat,
|
)),
|
||||||
)),
|
|
||||||
true,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
engine_task,
|
engine_task,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Engine script that answers one `add_lora` utility call with `loaded` and
|
||||||
|
/// hands the decoded LoRA request tuple to `check`.
|
||||||
|
fn add_lora_script(
|
||||||
|
loaded: bool,
|
||||||
|
check: impl FnOnce(&[Value]) + Send + 'static,
|
||||||
|
) -> impl for<'a> FnOnce(&'a mut DealerSocket, &'a mut PushSocket) -> TestFuture<'a> + Send + 'static
|
||||||
|
{
|
||||||
|
move |dealer, push| {
|
||||||
|
boxed_test_future(async move {
|
||||||
|
let utility = recv_engine_message(dealer).await;
|
||||||
|
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||||
|
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||||
|
let array = payload.as_array().expect("utility payload array");
|
||||||
|
let call_id = array[1].as_u64().expect("call id");
|
||||||
|
assert_eq!(array[2], Value::from("add_lora"));
|
||||||
|
let args = array[3].as_array().expect("utility args");
|
||||||
|
check(args[0].as_array().expect("lora request tuple"));
|
||||||
|
send_outputs(push, utility_outputs(call_id, utility_result_value(loaded))).await;
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn static_lora_module_loads_and_lists_with_parent() {
|
||||||
|
// An absolute local path: the runtime endpoint would reject it without
|
||||||
|
// VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES, static config trusts it.
|
||||||
|
let mut ready = default_ready_response();
|
||||||
|
ready.supports_lora = true;
|
||||||
|
ready.max_loras = 4;
|
||||||
|
let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
|
||||||
|
ready,
|
||||||
|
add_lora_script(true, |lora| {
|
||||||
|
assert_eq!(lora[0], Value::from("alice"));
|
||||||
|
assert_eq!(lora[1], Value::from(1));
|
||||||
|
assert_eq!(lora[2], Value::from("/adapters/alice"));
|
||||||
|
assert_eq!(lora[3], Value::from("base-model"));
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let module = LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "/adapters/alice".to_string(),
|
||||||
|
base_model_name: Some("base-model".to_string()),
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
};
|
||||||
|
let request = state.load_static_lora(&module).await.expect("load static lora");
|
||||||
|
assert_eq!(request.lora_name, "alice");
|
||||||
|
assert_eq!(request.lora_int_id, 1);
|
||||||
|
|
||||||
|
let mut app = build_router_with_dev_mode_and_lora(state, true, false);
|
||||||
|
let models = app
|
||||||
|
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
|
||||||
|
.await
|
||||||
|
.expect("call app");
|
||||||
|
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
|
||||||
|
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||||
|
assert_eq!(json["data"][1]["id"], "alice");
|
||||||
|
assert_eq!(json["data"][1]["root"], "/adapters/alice");
|
||||||
|
assert_eq!(json["data"][1]["parent"], "base-model");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn static_lora_module_requires_lora_enabled_engine() {
|
||||||
|
let (state, _engine_task) = test_admin_state_with_ready_and_engine_script(
|
||||||
|
default_ready_response(),
|
||||||
|
|_dealer, _push| boxed_test_future(async {}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let module = LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "org/alice".to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
};
|
||||||
|
let error = state.load_static_lora(&module).await.expect_err("engine has no lora");
|
||||||
|
assert!(matches!(error, LoadLoraError::Disabled(_)), "{error:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn static_lora_module_fails_when_engine_rejects_it() {
|
||||||
|
let mut ready = default_ready_response();
|
||||||
|
ready.supports_lora = true;
|
||||||
|
ready.max_loras = 4;
|
||||||
|
let (state, _engine_task) =
|
||||||
|
test_admin_state_with_ready_and_engine_script(ready, add_lora_script(false, |_| {})).await;
|
||||||
|
|
||||||
|
let module = LoraModulePath {
|
||||||
|
name: "alice".to_string(),
|
||||||
|
path: "org/alice".to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
};
|
||||||
|
let error = state.load_static_lora(&module).await.expect_err("engine rejected");
|
||||||
|
assert!(
|
||||||
|
matches!(error, LoadLoraError::NotLoaded { .. }),
|
||||||
|
"{error:?}"
|
||||||
|
);
|
||||||
|
assert!(state.served_lora_requests().await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn static_lora_module_rejects_empty_name_or_path() {
|
||||||
|
// The JSON form of `--lora-modules` is not validated at parse time; the
|
||||||
|
// manager rejects empty fields before any engine RPC.
|
||||||
|
let mut ready = default_ready_response();
|
||||||
|
ready.supports_lora = true;
|
||||||
|
ready.max_loras = 4;
|
||||||
|
let (state, _engine_task) =
|
||||||
|
test_admin_state_with_ready_and_engine_script(ready, |_dealer, _push| {
|
||||||
|
boxed_test_future(async {})
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
for (name, path) in [("", "org/alice"), ("alice", "")] {
|
||||||
|
let module = LoraModulePath {
|
||||||
|
name: name.to_string(),
|
||||||
|
path: path.to_string(),
|
||||||
|
base_model_name: None,
|
||||||
|
is_3d_lora_weight: false,
|
||||||
|
};
|
||||||
|
let error = state.load_static_lora(&module).await.expect_err("empty field");
|
||||||
|
assert!(
|
||||||
|
matches!(error, LoadLoraError::InvalidAdapter { .. }),
|
||||||
|
"{error:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn test_app_with_engine_handle() -> (axum::Router, MockEngineTask) {
|
async fn test_app_with_engine_handle() -> (axum::Router, MockEngineTask) {
|
||||||
test_app_with_stream_output_specs(default_stream_output_specs()).await
|
test_app_with_stream_output_specs(default_stream_output_specs()).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use vllm_engine_core_client::EngineCoreClient;
|
|||||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||||
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
|
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
|
||||||
|
|
||||||
use crate::config::{ApiServerOptions, CorsConfig};
|
use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath};
|
||||||
use crate::lora::{
|
use crate::lora::{
|
||||||
LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError,
|
LoadLoraError, LoraDisabledError, LoraManager, LoraModelResolution, UnloadLoraError,
|
||||||
};
|
};
|
||||||
@@ -181,24 +181,34 @@ impl AppState {
|
|||||||
/// Returns error if the engine was started without LoRA support.
|
/// Returns error if the engine was started without LoRA support.
|
||||||
pub async fn load_lora(
|
pub async fn load_lora(
|
||||||
&self,
|
&self,
|
||||||
lora_name: String,
|
module: LoraModulePath,
|
||||||
lora_path: String,
|
|
||||||
load_inplace: bool,
|
load_inplace: bool,
|
||||||
is_3d_lora_weight: bool,
|
|
||||||
) -> Result<LoraRequest, LoadLoraError> {
|
) -> Result<LoraRequest, LoadLoraError> {
|
||||||
self.ensure_lora_enabled()?;
|
self.ensure_lora_enabled()?;
|
||||||
self.lora_manager
|
self.lora_manager
|
||||||
.load_lora(
|
.load_lora(
|
||||||
self.engine_core_client(),
|
self.engine_core_client(),
|
||||||
&self.served_model_names,
|
&self.served_model_names,
|
||||||
lora_name,
|
module,
|
||||||
lora_path,
|
|
||||||
load_inplace,
|
load_inplace,
|
||||||
is_3d_lora_weight,
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Load one adapter from `--lora-modules` and register it as a public
|
||||||
|
/// model name.
|
||||||
|
///
|
||||||
|
/// Returns error if the engine was started without LoRA support.
|
||||||
|
pub async fn load_static_lora(
|
||||||
|
&self,
|
||||||
|
module: &LoraModulePath,
|
||||||
|
) -> Result<LoraRequest, LoadLoraError> {
|
||||||
|
self.ensure_lora_enabled()?;
|
||||||
|
self.lora_manager
|
||||||
|
.load_static_lora(self.engine_core_client(), &self.served_model_names, module)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
/// Remove one dynamic LoRA adapter from the engine and public model
|
/// Remove one dynamic LoRA adapter from the engine and public model
|
||||||
/// registry.
|
/// registry.
|
||||||
///
|
///
|
||||||
|
|||||||
Reference in New Issue
Block a user