[Rust Frontend] Add dynamic LoRA endpoints (#43778)

Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Xunzhuo
2026-06-03 07:55:29 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 4aaed4ca22
commit 6550ff12f2
24 changed files with 1081 additions and 49 deletions
+19
View File
@@ -10,6 +10,7 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop
use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::lora::LoraRequest;
use crate::protocol::utility::EngineCoreUtilityRequest;
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::transport::{self, ConnectedEngine};
@@ -659,6 +660,24 @@ impl EngineCoreClient {
Ok(results.into_iter().all(|ok| ok))
}
/// Load or refresh one LoRA adapter on every connected engine.
pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("add_lora", (lora_request,))
.await?
.into_iter()
.all(|loaded| loaded))
}
/// Remove one LoRA adapter from every connected engine.
pub async fn remove_lora(&self, lora_id: u64) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("remove_lora", (lora_id,))
.await?
.into_iter()
.all(|removed| removed))
}
/// Put the engine to sleep.
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
self.call_utility::<(), _>("sleep", (level, mode)).await?;
@@ -0,0 +1,42 @@
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::protocol::OpaqueValue;
/// Request for a LoRA adapter.
///
/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
/// `array_like=True` struct. Keep the field order aligned with Python.
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct LoraRequest {
pub lora_name: String,
pub lora_int_id: u64,
pub lora_path: String,
#[serde(default)]
pub base_model_name: Option<String>,
#[serde(default)]
pub tensorizer_config_dict: Option<OpaqueValue>,
#[serde(default)]
pub load_inplace: bool,
#[serde(default)]
pub is_3d_lora_weight: bool,
}
impl LoraRequest {
pub fn new(
lora_name: String,
lora_int_id: u64,
lora_path: String,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Self {
Self {
lora_name,
lora_int_id,
lora_path,
base_model_name: None,
tensorizer_config_dict: None,
load_inplace,
is_3d_lora_weight,
}
}
}
@@ -48,6 +48,7 @@ mod classified_outputs;
pub mod dtype;
pub mod handshake;
pub mod logprobs;
pub mod lora;
pub mod multimodal;
pub mod stats;
pub mod tensor;
@@ -349,7 +350,7 @@ pub struct EngineCoreRequest {
pub pooling_params: Option<OpaqueValue>,
pub arrival_time: f64,
#[serde(default)]
pub lora_request: Option<OpaqueValue>,
pub lora_request: Option<lora::LoraRequest>,
#[serde(default)]
pub cache_salt: Option<String>,
#[serde(default)]