[Rust Frontend] Add dynamic LoRA endpoints (#43778)
Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -91,6 +91,7 @@ pub fn to_text_request(
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
lora_request: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ mod config;
|
||||
mod error;
|
||||
mod grpc;
|
||||
mod listener;
|
||||
mod lora;
|
||||
mod middleware;
|
||||
mod routes;
|
||||
mod state;
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
|
||||
/// Snapshot of the currently served model names plus the requested LoRA, if
|
||||
/// the model name resolves to a dynamic adapter.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct LoraModelResolution {
|
||||
pub model_names: Vec<String>,
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
}
|
||||
|
||||
/// Runtime registry for dynamically loaded LoRA adapters.
|
||||
pub(crate) struct LoraManager {
|
||||
/// Dynamically loaded LoRA adapters keyed by public model name.
|
||||
requests: RwLock<BTreeMap<String, LoraRequest>>,
|
||||
/// Monotonic adapter id allocator. LoRA ids are one-indexed.
|
||||
id_counter: AtomicU64,
|
||||
/// Serialize dynamic LoRA registry updates around engine utility calls.
|
||||
update_lock: Mutex<()>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum LoadLoraError {
|
||||
AlreadyLoaded { lora_name: String },
|
||||
BaseModelName { lora_name: String },
|
||||
Engine(vllm_engine_core_client::Error),
|
||||
NotLoaded { lora_name: String },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum UnloadLoraError {
|
||||
NotFound {
|
||||
lora_name: String,
|
||||
},
|
||||
IntIdMismatch {
|
||||
lora_name: String,
|
||||
expected: u64,
|
||||
actual: u64,
|
||||
},
|
||||
Engine(vllm_engine_core_client::Error),
|
||||
NotRemoved {
|
||||
lora_name: String,
|
||||
lora_int_id: u64,
|
||||
},
|
||||
}
|
||||
|
||||
impl LoraManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
requests: RwLock::new(BTreeMap::new()),
|
||||
id_counter: AtomicU64::new(0),
|
||||
update_lock: Mutex::new(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return base served model names plus dynamically loaded LoRA adapter
|
||||
/// names.
|
||||
pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec<String> {
|
||||
let mut names = base_model_names.to_vec();
|
||||
names.extend(self.requests.read().await.keys().cloned());
|
||||
names
|
||||
}
|
||||
|
||||
/// Resolve the requested model against one consistent LoRA registry
|
||||
/// snapshot.
|
||||
pub async fn resolve_model(
|
||||
&self,
|
||||
base_model_names: &[String],
|
||||
model_name: Option<&str>,
|
||||
) -> LoraModelResolution {
|
||||
let requests = self.requests.read().await;
|
||||
let mut model_names = base_model_names.to_vec();
|
||||
model_names.extend(requests.keys().cloned());
|
||||
let lora_request = model_name.and_then(|name| requests.get(name).cloned());
|
||||
|
||||
LoraModelResolution {
|
||||
model_names,
|
||||
lora_request,
|
||||
}
|
||||
}
|
||||
|
||||
/// Load one dynamic LoRA adapter and register it as a public model name.
|
||||
pub async fn load_lora(
|
||||
&self,
|
||||
engine_core_client: &EngineCoreClient,
|
||||
base_model_names: &[String],
|
||||
lora_name: String,
|
||||
lora_path: String,
|
||||
load_inplace: bool,
|
||||
is_3d_lora_weight: bool,
|
||||
) -> Result<LoraRequest, LoadLoraError> {
|
||||
let _guard = self.update_lock.lock().await;
|
||||
if base_model_names.iter().any(|name| name == &lora_name) {
|
||||
return Err(LoadLoraError::BaseModelName { lora_name });
|
||||
}
|
||||
if !load_inplace && self.requests.read().await.contains_key(&lora_name) {
|
||||
return Err(LoadLoraError::AlreadyLoaded { lora_name });
|
||||
}
|
||||
|
||||
let lora_int_id = self
|
||||
.requests
|
||||
.read()
|
||||
.await
|
||||
.get(&lora_name)
|
||||
.map(|request| request.lora_int_id)
|
||||
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
|
||||
let lora_request = LoraRequest::new(
|
||||
lora_name.clone(),
|
||||
lora_int_id,
|
||||
lora_path,
|
||||
load_inplace,
|
||||
is_3d_lora_weight,
|
||||
);
|
||||
|
||||
let loaded = engine_core_client
|
||||
.add_lora(&lora_request)
|
||||
.await
|
||||
.map_err(LoadLoraError::Engine)?;
|
||||
if !loaded {
|
||||
return Err(LoadLoraError::NotLoaded { lora_name });
|
||||
}
|
||||
self.requests.write().await.insert(lora_name, lora_request.clone());
|
||||
Ok(lora_request)
|
||||
}
|
||||
|
||||
/// Remove one dynamic LoRA adapter from the engine and public model
|
||||
/// registry.
|
||||
pub async fn unload_lora(
|
||||
&self,
|
||||
engine_core_client: &EngineCoreClient,
|
||||
lora_name: &str,
|
||||
requested_lora_int_id: Option<u64>,
|
||||
) -> Result<LoraRequest, UnloadLoraError> {
|
||||
let _guard = self.update_lock.lock().await;
|
||||
let lora_request = self.requests.read().await.get(lora_name).cloned().ok_or_else(|| {
|
||||
UnloadLoraError::NotFound {
|
||||
lora_name: lora_name.to_string(),
|
||||
}
|
||||
})?;
|
||||
|
||||
if let Some(actual) = requested_lora_int_id
|
||||
&& actual != lora_request.lora_int_id
|
||||
{
|
||||
return Err(UnloadLoraError::IntIdMismatch {
|
||||
lora_name: lora_name.to_string(),
|
||||
expected: lora_request.lora_int_id,
|
||||
actual,
|
||||
});
|
||||
}
|
||||
|
||||
let removed = engine_core_client
|
||||
.remove_lora(lora_request.lora_int_id)
|
||||
.await
|
||||
.map_err(UnloadLoraError::Engine)?;
|
||||
if !removed {
|
||||
return Err(UnloadLoraError::NotRemoved {
|
||||
lora_name: lora_request.lora_name,
|
||||
lora_int_id: lora_request.lora_int_id,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request))
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ mod collective_rpc;
|
||||
mod health;
|
||||
mod inference;
|
||||
mod load;
|
||||
mod lora;
|
||||
mod metrics;
|
||||
pub(crate) mod openai;
|
||||
mod sleep;
|
||||
@@ -25,12 +26,40 @@ fn server_dev_mode_enabled() -> bool {
|
||||
.is_some_and(|value| value != 0)
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
build_router_with_dev_mode(state, server_dev_mode_enabled())
|
||||
fn runtime_lora_updating_enabled() -> bool {
|
||||
std::env::var("VLLM_ALLOW_RUNTIME_LORA_UPDATING")
|
||||
.ok()
|
||||
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
build_router_with_options(
|
||||
state,
|
||||
server_dev_mode_enabled(),
|
||||
runtime_lora_updating_enabled(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> Router {
|
||||
build_router_with_dev_mode_and_lora(state, dev_mode_enabled, false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn build_router_with_dev_mode_and_lora(
|
||||
state: Arc<AppState>,
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
) -> Router {
|
||||
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
|
||||
}
|
||||
|
||||
fn build_router_with_options(
|
||||
state: Arc<AppState>,
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
) -> Router {
|
||||
let mut router = Router::new()
|
||||
// Health & monitoring
|
||||
.route("/health", get(health::health))
|
||||
@@ -44,6 +73,12 @@ fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> R
|
||||
// vLLM specific inference endpoints
|
||||
.route("/inference/v1/generate", post(inference::generate));
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
router = router
|
||||
.route("/v1/load_lora_adapter", post(lora::load_lora_adapter))
|
||||
.route("/v1/unload_lora_adapter", post(lora::unload_lora_adapter));
|
||||
}
|
||||
|
||||
if dev_mode_enabled {
|
||||
// Development-only
|
||||
router = router
|
||||
|
||||
@@ -42,8 +42,8 @@ pub async fn generate(
|
||||
ValidatedJson(body): ValidatedJson<GenerateRequest>,
|
||||
) -> Response {
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let prepared = match prepare_generate_request(body, state.served_model_names(), request_context)
|
||||
{
|
||||
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
|
||||
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
|
||||
use super::types::GenerateRequest;
|
||||
use super::validate;
|
||||
use crate::error::ApiError;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
|
||||
|
||||
/// Lowered generate request plus the response request ID.
|
||||
@@ -21,10 +22,10 @@ pub struct PreparedRequest {
|
||||
/// text-generation format.
|
||||
pub fn prepare_generate_request(
|
||||
request: GenerateRequest,
|
||||
served_model_names: &[String],
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
let stream = request.stream;
|
||||
let include_usage = request
|
||||
@@ -57,6 +58,7 @@ pub fn prepare_generate_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
@@ -76,9 +78,17 @@ mod tests {
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::prepare_generate_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::inference::generate::types::GenerateRequest;
|
||||
use crate::utils::ResolvedRequestContext;
|
||||
|
||||
fn served(names: &[&str]) -> LoraModelResolution {
|
||||
LoraModelResolution {
|
||||
model_names: names.iter().map(|s| s.to_string()).collect(),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_maps_token_prompt_and_sampling_params() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
@@ -100,7 +110,7 @@ mod tests {
|
||||
|
||||
let prepared = prepare_generate_request(
|
||||
request,
|
||||
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
@@ -143,7 +153,7 @@ mod tests {
|
||||
|
||||
let prepared = prepare_generate_request(
|
||||
request,
|
||||
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::State;
|
||||
use serde::Deserialize;
|
||||
use thiserror_ext::AsReport;
|
||||
use validator::Validate;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::lora::{LoadLoraError, UnloadLoraError};
|
||||
use crate::routes::openai::utils::types::Normalizable;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
|
||||
const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
|
||||
|
||||
#[derive(Debug, Deserialize, Validate)]
|
||||
pub(crate) struct LoadLoraAdapterRequest {
|
||||
lora_name: String,
|
||||
lora_path: String,
|
||||
#[serde(default)]
|
||||
load_inplace: bool,
|
||||
#[serde(default)]
|
||||
is_3d_lora_weight: bool,
|
||||
}
|
||||
|
||||
impl Normalizable for LoadLoraAdapterRequest {}
|
||||
|
||||
#[derive(Debug, Deserialize, Validate)]
|
||||
pub(crate) struct UnloadLoraAdapterRequest {
|
||||
lora_name: String,
|
||||
#[serde(default)]
|
||||
lora_int_id: Option<u64>,
|
||||
}
|
||||
|
||||
impl Normalizable for UnloadLoraAdapterRequest {}
|
||||
|
||||
fn runtime_lora_allowed_path_prefixes() -> Option<Vec<PathBuf>> {
|
||||
let prefixes = std::env::var_os(RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV)?;
|
||||
let prefixes: Vec<_> = std::env::split_paths(&prefixes)
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.collect();
|
||||
(!prefixes.is_empty()).then_some(prefixes)
|
||||
}
|
||||
|
||||
fn looks_like_local_lora_path(lora_path: &str) -> bool {
|
||||
let path = Path::new(lora_path);
|
||||
path.is_absolute()
|
||||
|| lora_path.starts_with('~')
|
||||
|| lora_path.starts_with('.')
|
||||
|| path.components().any(|component| matches!(component, Component::ParentDir))
|
||||
}
|
||||
|
||||
fn validate_lora_path_access(
|
||||
lora_path: &str,
|
||||
allowed_prefixes: Option<&[PathBuf]>,
|
||||
) -> Result<Option<String>, ApiError> {
|
||||
let path = Path::new(lora_path);
|
||||
if !looks_like_local_lora_path(lora_path) && !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(allowed_prefixes) = allowed_prefixes else {
|
||||
return Err(ApiError::invalid_request(
|
||||
format!(
|
||||
"Local LoRA adapter paths require {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} to be configured."
|
||||
),
|
||||
Some("lora_path"),
|
||||
));
|
||||
};
|
||||
|
||||
if !path.is_absolute() {
|
||||
return Err(ApiError::invalid_request(
|
||||
format!(
|
||||
"Local LoRA adapter paths must be absolute and under one of the prefixes configured by {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV}."
|
||||
),
|
||||
Some("lora_path"),
|
||||
));
|
||||
}
|
||||
|
||||
let canonical_path = path.canonicalize().map_err(|_| {
|
||||
ApiError::invalid_request(
|
||||
"Local LoRA adapter path must exist and be accessible.".to_string(),
|
||||
Some("lora_path"),
|
||||
)
|
||||
})?;
|
||||
let canonical_prefixes = allowed_prefixes
|
||||
.iter()
|
||||
.map(|prefix| {
|
||||
prefix.canonicalize().map_err(|_| {
|
||||
ApiError::server_error(format!(
|
||||
"configured {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} path prefix must exist and be accessible"
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
if !canonical_prefixes.iter().any(|prefix| canonical_path.starts_with(prefix)) {
|
||||
return Err(ApiError::invalid_request(
|
||||
"Local LoRA adapter path is outside the configured allowed prefixes.".to_string(),
|
||||
Some("lora_path"),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Some(canonical_path.to_string_lossy().into_owned()))
|
||||
}
|
||||
|
||||
/// Dynamically load one LoRA adapter and expose it as an OpenAI model id.
|
||||
pub async fn load_lora_adapter(
|
||||
State(state): State<Arc<AppState>>,
|
||||
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
|
||||
) -> Result<String, ApiError> {
|
||||
if request.lora_name.is_empty() || request.lora_path.is_empty() {
|
||||
return Err(ApiError::invalid_request(
|
||||
"Both 'lora_name' and 'lora_path' must be provided.".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
|
||||
let lora_path = validate_lora_path_access(&request.lora_path, allowed_prefixes.as_deref())?
|
||||
.unwrap_or(request.lora_path);
|
||||
|
||||
let lora_name = request.lora_name;
|
||||
state
|
||||
.load_lora(
|
||||
lora_name.clone(),
|
||||
lora_path,
|
||||
request.load_inplace,
|
||||
request.is_3d_lora_weight,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
LoadLoraError::AlreadyLoaded { lora_name } => ApiError::invalid_request(
|
||||
format!(
|
||||
"The lora adapter '{lora_name}' has already been loaded. If you want to load the adapter in place, set 'load_inplace' to true."
|
||||
),
|
||||
Some("lora_name"),
|
||||
),
|
||||
LoadLoraError::BaseModelName { lora_name } => ApiError::invalid_request(
|
||||
format!("The lora adapter name '{lora_name}' conflicts with a served base model."),
|
||||
Some("lora_name"),
|
||||
),
|
||||
LoadLoraError::Engine(error) => ApiError::server_error(format!(
|
||||
"failed to load LoRA adapter '{lora_name}': {}",
|
||||
error.to_report_string()
|
||||
)),
|
||||
LoadLoraError::NotLoaded { lora_name } => ApiError::server_error(format!(
|
||||
"failed to load LoRA adapter '{lora_name}': engine rejected the adapter"
|
||||
)),
|
||||
})?;
|
||||
|
||||
Ok(format!(
|
||||
"Success: LoRA adapter '{lora_name}' added successfully."
|
||||
))
|
||||
}
|
||||
|
||||
/// Remove one LoRA adapter from the engine and frontend registry.
|
||||
pub async fn unload_lora_adapter(
|
||||
State(state): State<Arc<AppState>>,
|
||||
ValidatedJson(request): ValidatedJson<UnloadLoraAdapterRequest>,
|
||||
) -> Result<String, ApiError> {
|
||||
if request.lora_name.is_empty() {
|
||||
return Err(ApiError::invalid_request(
|
||||
"'lora_name' needs to be provided to unload a LoRA adapter.".to_string(),
|
||||
Some("lora_name"),
|
||||
));
|
||||
}
|
||||
|
||||
let lora_request = state
|
||||
.unload_lora(&request.lora_name, request.lora_int_id)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name),
|
||||
UnloadLoraError::IntIdMismatch {
|
||||
lora_name,
|
||||
expected,
|
||||
actual,
|
||||
} => ApiError::invalid_request(
|
||||
format!(
|
||||
"The requested lora_int_id {actual} does not match loaded adapter '{lora_name}' with id {expected}."
|
||||
),
|
||||
Some("lora_int_id"),
|
||||
),
|
||||
UnloadLoraError::Engine(error) => ApiError::server_error(format!(
|
||||
"failed to unload LoRA adapter '{}': {}",
|
||||
request.lora_name,
|
||||
error.to_report_string()
|
||||
)),
|
||||
UnloadLoraError::NotRemoved {
|
||||
lora_name,
|
||||
lora_int_id,
|
||||
} => ApiError::server_error(format!(
|
||||
"failed to unload LoRA adapter '{lora_name}' with id {lora_int_id}"
|
||||
)),
|
||||
})?;
|
||||
|
||||
Ok(format!(
|
||||
"Success: LoRA adapter '{}' removed successfully.",
|
||||
lora_request.lora_name
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::validate_lora_path_access;
|
||||
|
||||
fn temp_lora_dir(test_name: &str) -> PathBuf {
|
||||
let suffix = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("clock should be after unix epoch")
|
||||
.as_nanos();
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"vllm-lora-{test_name}-{}-{suffix}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&path).expect("create temp lora dir");
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_path_allows_hf_repo_ids_without_prefixes() {
|
||||
assert_eq!(
|
||||
validate_lora_path_access("org/adapter-a", None).expect("hf repo id should be allowed"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_path_rejects_local_paths_without_prefixes() {
|
||||
assert!(validate_lora_path_access("/tmp/adapter-a", None).is_err());
|
||||
assert!(validate_lora_path_access("./adapter-a", None).is_err());
|
||||
assert!(validate_lora_path_access("~/adapter-a", None).is_err());
|
||||
assert!(validate_lora_path_access("subdir/../../../etc/sensitive", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_path_rejects_existing_bare_relative_paths_without_prefixes() {
|
||||
let root =
|
||||
PathBuf::from("target").join(format!("vllm-lora-relative-{}", std::process::id()));
|
||||
let adapter = root.join("adapter-a");
|
||||
fs::create_dir_all(&adapter).expect("create relative adapter dir");
|
||||
|
||||
assert!(
|
||||
validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), None).is_err()
|
||||
);
|
||||
|
||||
fs::remove_dir_all(root).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_path_allows_absolute_paths_under_configured_prefixes() {
|
||||
let root = temp_lora_dir("allowed-prefix");
|
||||
let allowed = root.join("allowed");
|
||||
let adapter = allowed.join("adapter-a");
|
||||
fs::create_dir_all(&adapter).expect("create adapter dir");
|
||||
|
||||
let prefixes = [allowed];
|
||||
let resolved =
|
||||
validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), Some(&prefixes))
|
||||
.expect("path under configured prefix should be allowed");
|
||||
assert_eq!(
|
||||
resolved.as_deref(),
|
||||
Some(
|
||||
adapter
|
||||
.canonicalize()
|
||||
.expect("canonical adapter")
|
||||
.to_str()
|
||||
.expect("utf-8 temp path")
|
||||
)
|
||||
);
|
||||
|
||||
fs::remove_dir_all(root).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lora_path_rejects_parent_escape_from_configured_prefixes() {
|
||||
let root = temp_lora_dir("parent-escape");
|
||||
let allowed = root.join("allowed");
|
||||
let private_adapter = root.join("private").join("adapter-a");
|
||||
fs::create_dir_all(&allowed).expect("create allowed dir");
|
||||
fs::create_dir_all(&private_adapter).expect("create private adapter dir");
|
||||
|
||||
let escaped = allowed.join("../private/adapter-a");
|
||||
let prefixes = [allowed];
|
||||
assert!(
|
||||
validate_lora_path_access(escaped.to_str().expect("utf-8 temp path"), Some(&prefixes))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
fs::remove_dir_all(root).ok();
|
||||
}
|
||||
}
|
||||
@@ -49,8 +49,9 @@ pub async fn chat_completions(
|
||||
) -> Response {
|
||||
let stream = body.stream;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
|
||||
|
||||
let prepared = match prepare_chat_request(body, state.served_model_names(), request_context) {
|
||||
let prepared = match prepare_chat_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
@@ -8,6 +8,7 @@ use vllm_chat::{
|
||||
use super::types::ChatCompletionRequest;
|
||||
use super::validate;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
|
||||
@@ -41,16 +42,21 @@ pub struct PreparedRequest {
|
||||
/// Validate and lower one OpenAI chat completion request into the internal chat
|
||||
/// format.
|
||||
///
|
||||
/// `served_model_names` must be non-empty; the first entry is used as the
|
||||
/// `model` field in responses.
|
||||
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
|
||||
/// the base `model` field in responses when no LoRA adapter is selected.
|
||||
pub(crate) fn prepare_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
served_model_names: &[String],
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
let request_id = format!("chatcmpl-{}", ctx.request_id);
|
||||
let response_model = lora_resolution
|
||||
.lora_request
|
||||
.as_ref()
|
||||
.map(|request| request.lora_name.clone())
|
||||
.unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
|
||||
let echo = request
|
||||
.echo
|
||||
.then(|| extract_last_assistant_content(&request.messages))
|
||||
@@ -131,11 +137,12 @@ pub(crate) fn prepare_chat_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model: served_model_names.first().cloned().unwrap_or_default(),
|
||||
response_model,
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
@@ -352,6 +359,7 @@ mod tests {
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
|
||||
use super::prepare_chat_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||
};
|
||||
@@ -365,8 +373,11 @@ mod tests {
|
||||
resolve_request_context(headers, request_id)
|
||||
}
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
fn served(names: &[&str]) -> LoraModelResolution {
|
||||
LoraModelResolution {
|
||||
model_names: names.iter().map(|s| s.to_string()).collect(),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn base_request() -> ChatCompletionRequest {
|
||||
|
||||
@@ -44,12 +44,12 @@ pub async fn completions(
|
||||
let stream = body.stream;
|
||||
let logprobs = body.logprobs;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
|
||||
|
||||
let prepared =
|
||||
match prepare_completion_request(body, state.served_model_names(), request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let prepared = match prepare_completion_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let request_span = tracing::info_span!(
|
||||
"completions",
|
||||
request_id = %prepared.request_id,
|
||||
|
||||
@@ -2,6 +2,7 @@ use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::CompletionRequest;
|
||||
use crate::error::ApiError;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::validate;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
|
||||
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
|
||||
@@ -30,16 +31,21 @@ pub struct PreparedRequest {
|
||||
/// Validate and lower one OpenAI completions request into the internal
|
||||
/// text-generation format.
|
||||
///
|
||||
/// `served_model_names` must be non-empty; the first entry is used as the
|
||||
/// `model` field in responses.
|
||||
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
|
||||
/// the base `model` field in responses when no LoRA adapter is selected.
|
||||
pub(crate) fn prepare_completion_request(
|
||||
request: CompletionRequest,
|
||||
served_model_names: &[String],
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, served_model_names)?;
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
let request_id = format!("cmpl-{}", ctx.request_id);
|
||||
let response_model = lora_resolution
|
||||
.lora_request
|
||||
.as_ref()
|
||||
.map(|request| request.lora_name.clone())
|
||||
.unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
|
||||
|
||||
let logprobs = match request.logprobs {
|
||||
Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
|
||||
@@ -104,11 +110,12 @@ pub(crate) fn prepare_completion_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model: served_model_names.first().cloned().unwrap_or_default(),
|
||||
response_model,
|
||||
include_usage,
|
||||
text_request,
|
||||
echo,
|
||||
@@ -124,6 +131,7 @@ mod tests {
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::prepare_completion_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::types::CompletionRequest;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||
|
||||
@@ -131,8 +139,11 @@ mod tests {
|
||||
resolve_request_context(headers, request_id)
|
||||
}
|
||||
|
||||
fn served(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|s| s.to_string()).collect()
|
||||
fn served(names: &[&str]) -> LoraModelResolution {
|
||||
LoraModelResolution {
|
||||
model_names: names.iter().map(|s| s.to_string()).collect(),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn base_request_json() -> serde_json::Value {
|
||||
|
||||
@@ -8,13 +8,13 @@ use crate::state::AppState;
|
||||
|
||||
/// Return all configured served model names in OpenAI `list models` format.
|
||||
pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
|
||||
let model_names = state.served_model_names_with_loras().await;
|
||||
Json(ListModelsResponse {
|
||||
object: "list".to_string(),
|
||||
data: state
|
||||
.served_model_names()
|
||||
.iter()
|
||||
data: model_names
|
||||
.into_iter()
|
||||
.map(|name| ModelObject {
|
||||
id: name.clone(),
|
||||
id: name,
|
||||
object: "model".to_string(),
|
||||
created: 0,
|
||||
owned_by: "vllm-frontend-rs".to_string(),
|
||||
|
||||
@@ -43,7 +43,8 @@ use vllm_text::{Prompt, TextBackend};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{build_router, build_router_with_dev_mode};
|
||||
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
|
||||
use crate::state::AppState;
|
||||
|
||||
@@ -141,6 +142,13 @@ fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason
|
||||
]
|
||||
}
|
||||
|
||||
fn assert_adapter_a_lora_request(request: &EngineCoreRequest) {
|
||||
let lora = request.lora_request.as_ref().expect("lora request");
|
||||
assert_eq!(lora.lora_name, "adapter-a");
|
||||
assert_eq!(lora.lora_int_id, 1);
|
||||
assert_eq!(lora.lora_path, "org/adapter-a");
|
||||
}
|
||||
|
||||
fn sse_data_payloads(text: &str) -> Vec<&str> {
|
||||
text.lines().filter_map(|line| line.strip_prefix("data: ")).collect()
|
||||
}
|
||||
@@ -822,12 +830,13 @@ where
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
(
|
||||
build_router_with_dev_mode(
|
||||
build_router_with_dev_mode_and_lora(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
true,
|
||||
),
|
||||
engine_task,
|
||||
)
|
||||
@@ -1078,6 +1087,367 @@ async fn version_returns_engine_vllm_version() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
assert_eq!(array[2], Value::from("add_lora"));
|
||||
|
||||
let args = array[3].as_array().expect("utility args");
|
||||
let lora = args[0].as_array().expect("lora request tuple");
|
||||
assert_eq!(lora[0], Value::from("adapter-a"));
|
||||
assert_eq!(lora[1], Value::from(1));
|
||||
assert_eq!(lora[2], Value::from("org/adapter-a"));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode engine request");
|
||||
assert_adapter_a_lora_request(&request);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode engine request");
|
||||
assert_adapter_a_lora_request(&request);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode engine request");
|
||||
assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22][..]));
|
||||
assert_adapter_a_lora_request(&request);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
assert_eq!(array[2], Value::from("remove_lora"));
|
||||
|
||||
let args = array[3].as_array().expect("utility args");
|
||||
assert_eq!(args[0], Value::from(1));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/load_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "adapter-a",
|
||||
"lora_path": "org/adapter-a"
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let models = app
|
||||
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
|
||||
.await
|
||||
.expect("call app");
|
||||
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["data"][1]["id"], "adapter-a");
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "adapter-a",
|
||||
"prompt": "hello",
|
||||
"max_tokens": 2
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "adapter-a",
|
||||
"stream": false,
|
||||
"messages": [{"role": "user", "content": "hello"}]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "adapter-a",
|
||||
"token_ids": [11, 22],
|
||||
"stream": false,
|
||||
"sampling_params": {
|
||||
"max_tokens": 2
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/unload_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "adapter-a",
|
||||
"lora_int_id": 1
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let models = app
|
||||
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
|
||||
.await
|
||||
.expect("call app");
|
||||
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["data"].as_array().expect("model data").len(), 1);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "adapter-a",
|
||||
"prompt": "hello",
|
||||
"max_tokens": 2
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
drop(app);
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
assert_eq!(array[2], Value::from("add_lora"));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/load_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "adapter-a",
|
||||
"lora_path": "org/adapter-a"
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/unload_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "adapter-a",
|
||||
"lora_int_id": 99
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
|
||||
let models = app
|
||||
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
|
||||
.await
|
||||
.expect("call app");
|
||||
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["data"][1]["id"], "adapter-a");
|
||||
|
||||
drop(app);
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn load_lora_adapter_rejects_engine_false_result() {
|
||||
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let utility = recv_engine_message(dealer).await;
|
||||
assert_eq!(utility[0].as_ref(), &[0x03]);
|
||||
|
||||
let payload = decode_value(&utility[1]).expect("decode utility payload");
|
||||
let array = payload.as_array().expect("utility payload array");
|
||||
let call_id = array[1].as_u64().expect("call id");
|
||||
assert_eq!(array[2], Value::from("add_lora"));
|
||||
|
||||
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/load_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "adapter-a",
|
||||
"lora_path": "org/adapter-a"
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
|
||||
let models = app
|
||||
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
|
||||
.await
|
||||
.expect("call app");
|
||||
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["data"].as_array().expect("model data").len(), 1);
|
||||
|
||||
drop(app);
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn load_lora_adapter_rejects_base_model_name_collision() {
|
||||
let (mut app, engine_task) =
|
||||
test_admin_app_with_engine_script(|_, _| boxed_test_future(async move {})).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/load_lora_adapter")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"lora_name": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"lora_path": "org/adapter-a"
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
|
||||
drop(app);
|
||||
engine_task.finish().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn http_metrics_record_list_models_requests() {
|
||||
@@ -3000,7 +3370,10 @@ async fn prepared_openai_request_streams_text_events() {
|
||||
"messages": [{"role": "user", "content": "hello"}]
|
||||
}))
|
||||
.expect("decode request"),
|
||||
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
&LoraModelResolution {
|
||||
model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
lora_request: None,
|
||||
},
|
||||
crate::utils::ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare request");
|
||||
|
||||
@@ -5,6 +5,9 @@ use tokio::time::{Duration, Instant, sleep_until};
|
||||
use tracing::warn;
|
||||
use vllm_chat::ChatLlm;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
|
||||
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
|
||||
|
||||
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
@@ -21,6 +24,8 @@ pub struct AppState {
|
||||
pub enable_request_id_headers: bool,
|
||||
/// Number of in-flight inference requests currently owned by this frontend.
|
||||
server_load: AtomicU64,
|
||||
/// Dynamic LoRA adapter registry.
|
||||
lora_manager: LoraManager,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
@@ -43,6 +48,7 @@ impl AppState {
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
server_load: AtomicU64::new(0),
|
||||
lora_manager: LoraManager::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +75,49 @@ impl AppState {
|
||||
&self.served_model_names
|
||||
}
|
||||
|
||||
/// Return base served model names plus dynamically loaded LoRA adapter
|
||||
/// names.
|
||||
pub async fn served_model_names_with_loras(&self) -> Vec<String> {
|
||||
self.lora_manager.served_model_names(&self.served_model_names).await
|
||||
}
|
||||
|
||||
/// Resolve the requested model against one dynamic LoRA registry snapshot.
|
||||
pub async fn resolve_model_with_loras(&self, model_name: Option<&str>) -> LoraModelResolution {
|
||||
self.lora_manager.resolve_model(&self.served_model_names, model_name).await
|
||||
}
|
||||
|
||||
/// Load one dynamic LoRA adapter and register it as a public model name.
|
||||
pub async fn load_lora(
|
||||
&self,
|
||||
lora_name: String,
|
||||
lora_path: String,
|
||||
load_inplace: bool,
|
||||
is_3d_lora_weight: bool,
|
||||
) -> Result<LoraRequest, LoadLoraError> {
|
||||
self.lora_manager
|
||||
.load_lora(
|
||||
self.engine_core_client(),
|
||||
&self.served_model_names,
|
||||
lora_name,
|
||||
lora_path,
|
||||
load_inplace,
|
||||
is_3d_lora_weight,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Remove one dynamic LoRA adapter from the engine and public model
|
||||
/// registry.
|
||||
pub async fn unload_lora(
|
||||
&self,
|
||||
lora_name: &str,
|
||||
lora_int_id: Option<u64>,
|
||||
) -> Result<LoraRequest, UnloadLoraError> {
|
||||
self.lora_manager
|
||||
.unload_lora(self.engine_core_client(), lora_name, lora_int_id)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Return a reference to the underlying engine core client for utility
|
||||
/// calls.
|
||||
pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
|
||||
Reference in New Issue
Block a user