[Rust Frontend] Add dynamic LoRA endpoints (#43778)
Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -10,6 +10,7 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
use crate::protocol::utility::EngineCoreUtilityRequest;
|
||||
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
@@ -659,6 +660,24 @@ impl EngineCoreClient {
|
||||
Ok(results.into_iter().all(|ok| ok))
|
||||
}
|
||||
|
||||
/// Load or refresh one LoRA adapter on every connected engine.
|
||||
pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result<bool> {
|
||||
Ok(self
|
||||
.call_utility::<bool, _>("add_lora", (lora_request,))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|loaded| loaded))
|
||||
}
|
||||
|
||||
/// Remove one LoRA adapter from every connected engine.
|
||||
pub async fn remove_lora(&self, lora_id: u64) -> Result<bool> {
|
||||
Ok(self
|
||||
.call_utility::<bool, _>("remove_lora", (lora_id,))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|removed| removed))
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
|
||||
self.call_utility::<(), _>("sleep", (level, mode)).await?;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::protocol::OpaqueValue;
|
||||
|
||||
/// Request for a LoRA adapter.
|
||||
///
|
||||
/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
|
||||
/// `array_like=True` struct. Keep the field order aligned with Python.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct LoraRequest {
|
||||
pub lora_name: String,
|
||||
pub lora_int_id: u64,
|
||||
pub lora_path: String,
|
||||
#[serde(default)]
|
||||
pub base_model_name: Option<String>,
|
||||
#[serde(default)]
|
||||
pub tensorizer_config_dict: Option<OpaqueValue>,
|
||||
#[serde(default)]
|
||||
pub load_inplace: bool,
|
||||
#[serde(default)]
|
||||
pub is_3d_lora_weight: bool,
|
||||
}
|
||||
|
||||
impl LoraRequest {
|
||||
pub fn new(
|
||||
lora_name: String,
|
||||
lora_int_id: u64,
|
||||
lora_path: String,
|
||||
load_inplace: bool,
|
||||
is_3d_lora_weight: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
lora_name,
|
||||
lora_int_id,
|
||||
lora_path,
|
||||
base_model_name: None,
|
||||
tensorizer_config_dict: None,
|
||||
load_inplace,
|
||||
is_3d_lora_weight,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,7 @@ mod classified_outputs;
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
pub mod logprobs;
|
||||
pub mod lora;
|
||||
pub mod multimodal;
|
||||
pub mod stats;
|
||||
pub mod tensor;
|
||||
@@ -349,7 +350,7 @@ pub struct EngineCoreRequest {
|
||||
pub pooling_params: Option<OpaqueValue>,
|
||||
pub arrival_time: f64,
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<OpaqueValue>,
|
||||
pub lora_request: Option<lora::LoraRequest>,
|
||||
#[serde(default)]
|
||||
pub cache_salt: Option<String>,
|
||||
#[serde(default)]
|
||||
|
||||
Reference in New Issue
Block a user