iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Support --lora-modules for static adapter loading (#54837)

Browse Source 
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Will Eaton <weaton@redhat.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Will Eaton
2026-09-03 19:12:45 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 21a2211040
commit 2db1c4dc31
12 changed files with 497 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/cmd/src/cli.rs
+11 -2
View File
@@ -29,8 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
use vllm_server::{
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
TlsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
RendererSelection, TlsConfig,
};
use crate::cli::unsupported::UnsupportedArgs;
@@ -269,6 +269,13 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub limit_mm_per_prompt: MmLimitPerPrompt,
/// LoRA adapters to load before serving, each as `name=path` or a JSON
/// object: `{"name": "name", "path": "lora_path", "base_model_name": "id"}`.
/// Requires `--enable-lora`; startup fails if any adapter cannot be loaded.
#[arg(long, num_args = 1.., value_name = "MODULE")]
#[serde(default)]
pub lora_modules: Vec<LoraModulePath>,
/// The format to render message content within a chat template.
///
/// * "auto" detects the format from the template
@@ -498,6 +505,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
@@ -552,6 +560,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
rust/src/cmd/src/cli/tests.rs
+105 -1
View File
@@ -4,7 +4,8 @@
use expect_test::expect;
use vllm_engine_core_client::TransportMode;
use vllm_server::{
Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
Config, GenerationConfigMode, HttpListenerMode, LoraModulePath, ParserSelection,
RendererSelection,
};
use super::{BenchCommand, Cli, Command};
@@ -103,6 +104,7 @@ fn serve_args_forward_python_flags_with_separator() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -623,6 +625,103 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
assert!(config.api_server_options.enable_request_id_headers);
}
#[test]
fn serve_args_parse_lora_modules_in_both_forms() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--enable-lora",
"--lora-modules",
"alice=charent/self_cognition_Alice",
r#"{"name": "bob", "path": "/adapters/bob", "base_model_name": "base"}"#,
"--lora-modules",
"carol=org/carol",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
// Frontend-only flag: the managed Python engine must not see it.
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
expect![[r#"
[
LoraModulePath {
name: "alice",
path: "charent/self_cognition_Alice",
base_model_name: None,
is_3d_lora_weight: false,
},
LoraModulePath {
name: "bob",
path: "/adapters/bob",
base_model_name: Some(
"base",
),
is_3d_lora_weight: false,
},
LoraModulePath {
name: "carol",
path: "org/carol",
base_model_name: None,
is_3d_lora_weight: false,
},
]
"#]]
.assert_debug_eq(&config.lora_modules);
}
#[test]
fn serve_args_reject_malformed_lora_module() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--lora-modules",
"alice",
])
.unwrap_err();
expect![[r#"
error: invalid value 'alice' for '--lora-modules <MODULE>...': expected `name=path`, got `alice`
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn frontend_args_json_passes_lora_modules_into_config() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","lora_modules":[{"name":"alice","path":"org/alice","base_model_name":null,"is_3d_lora_weight":true}]}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
let config = args.into_config();
assert_eq!(
config.lora_modules,
vec![LoraModulePath {
name: "alice".to_string(),
path: "org/alice".to_string(),
base_model_name: None,
is_3d_lora_weight: true,
}]
);
}
#[test]
fn serve_passes_api_keys_into_config() {
let cli = Cli::try_parse_from([
@@ -822,6 +921,7 @@ fn frontend_args_accept_json() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1416,6 +1516,7 @@ fn serve_args_accept_handshake_aliases() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1561,6 +1662,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1647,6 +1749,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1754,6 +1857,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
rust/src/cmd/src/cli/unsupported.rs
-7
View File
@@ -353,13 +353,6 @@ pub struct EngineUnsupportedArgs {
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
pub struct ServerUnsupportedArgs {
/// LoRA modules configurations in either 'name=path' format or JSON format
/// or JSON list format. Example (old format): `'name=path'` Example (new
/// format): `{"name": "name", "path": "lora_path",
/// "base_model_name": "id"}`
#[arg(long)]
pub lora_modules: Option<Unsupported>,
/// Whether to trust the chat template provided in the request. If False,
/// the server will always use the chat template specified by
/// `--chat-template` or the ones from tokenizer.