[Rust Frontend] Add dynamic LoRA endpoints (#43778)

Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Xunzhuo
2026-06-03 07:55:29 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 4aaed4ca22
commit 6550ff12f2
24 changed files with 1081 additions and 49 deletions
+1
View File
@@ -189,6 +189,7 @@ impl ChatLlm {
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
lora_request: request.lora_request,
};
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
+5
View File
@@ -3,6 +3,7 @@ use std::collections::HashMap;
use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
pub use vllm_text::SamplingParams;
use vllm_text::TextDecodeOptions;
pub use vllm_tool_parser::Tool as ChatTool;
@@ -426,6 +427,9 @@ pub struct ChatRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
}
impl ChatRequest {
@@ -445,6 +449,7 @@ impl ChatRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
lora_request: None,
}
}