[Rust Frontend] Add dynamic LoRA endpoints (#43778)
Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -189,6 +189,7 @@ impl ChatLlm {
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
lora_request: request.lora_request,
|
||||
};
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::collections::HashMap;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
pub use vllm_text::SamplingParams;
|
||||
use vllm_text::TextDecodeOptions;
|
||||
pub use vllm_tool_parser::Tool as ChatTool;
|
||||
@@ -426,6 +427,9 @@ pub struct ChatRequest {
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// LoRA adapter selected for this request.
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
}
|
||||
|
||||
impl ChatRequest {
|
||||
@@ -445,6 +449,7 @@ impl ChatRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user