[Rust Frontend] Add dynamic LoRA endpoints (#43778)

Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Xunzhuo
2026-06-03 07:55:29 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 4aaed4ca22
commit 6550ff12f2
24 changed files with 1081 additions and 49 deletions
+1
View File
@@ -189,6 +189,7 @@ impl ChatLlm {
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
lora_request: request.lora_request,
};
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
+5
View File
@@ -3,6 +3,7 @@ use std::collections::HashMap;
use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
pub use vllm_text::SamplingParams;
use vllm_text::TextDecodeOptions;
pub use vllm_tool_parser::Tool as ChatTool;
@@ -426,6 +427,9 @@ pub struct ChatRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
}
impl ChatRequest {
@@ -445,6 +449,7 @@ impl ChatRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
lora_request: None,
}
}
+11
View File
@@ -87,6 +87,17 @@ fn serve_args_auto_forward_python_flags_without_separator() {
);
}
#[test]
fn serve_args_auto_forward_enable_lora_to_python() {
let cli =
Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--enable-lora"]).unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
}
#[test]
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
-9
View File
@@ -326,15 +326,6 @@ pub struct EngineUnsupportedArgs {
#[arg(long)]
pub mm_processor_cache_type: Option<Unsupported>,
/// If True, enable handling of LoRA adapters.
#[arg(
long,
visible_alias = "no-enable-lora",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_lora: Option<Unsupported>,
/// Dictionary mapping specific modalities to LoRA model paths.
#[arg(long)]
pub default_mm_loras: Option<Unsupported>,
+19
View File
@@ -10,6 +10,7 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop
use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::lora::LoraRequest;
use crate::protocol::utility::EngineCoreUtilityRequest;
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::transport::{self, ConnectedEngine};
@@ -659,6 +660,24 @@ impl EngineCoreClient {
Ok(results.into_iter().all(|ok| ok))
}
/// Load or refresh one LoRA adapter on every connected engine.
pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("add_lora", (lora_request,))
.await?
.into_iter()
.all(|loaded| loaded))
}
/// Remove one LoRA adapter from every connected engine.
pub async fn remove_lora(&self, lora_id: u64) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("remove_lora", (lora_id,))
.await?
.into_iter()
.all(|removed| removed))
}
/// Put the engine to sleep.
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
self.call_utility::<(), _>("sleep", (level, mode)).await?;
@@ -0,0 +1,42 @@
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::protocol::OpaqueValue;
/// Request for a LoRA adapter.
///
/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
/// `array_like=True` struct. Keep the field order aligned with Python.
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct LoraRequest {
pub lora_name: String,
pub lora_int_id: u64,
pub lora_path: String,
#[serde(default)]
pub base_model_name: Option<String>,
#[serde(default)]
pub tensorizer_config_dict: Option<OpaqueValue>,
#[serde(default)]
pub load_inplace: bool,
#[serde(default)]
pub is_3d_lora_weight: bool,
}
impl LoraRequest {
pub fn new(
lora_name: String,
lora_int_id: u64,
lora_path: String,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Self {
Self {
lora_name,
lora_int_id,
lora_path,
base_model_name: None,
tensorizer_config_dict: None,
load_inplace,
is_3d_lora_weight,
}
}
}
@@ -48,6 +48,7 @@ mod classified_outputs;
pub mod dtype;
pub mod handshake;
pub mod logprobs;
pub mod lora;
pub mod multimodal;
pub mod stats;
pub mod tensor;
@@ -349,7 +350,7 @@ pub struct EngineCoreRequest {
pub pooling_params: Option<OpaqueValue>,
pub arrival_time: f64,
#[serde(default)]
pub lora_request: Option<OpaqueValue>,
pub lora_request: Option<lora::LoraRequest>,
#[serde(default)]
pub cache_salt: Option<String>,
#[serde(default)]
+3 -2
View File
@@ -2,8 +2,9 @@ use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH};
use uuid::Uuid;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams, OpaqueValue};
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams};
use crate::error::{Error, Result};
@@ -34,7 +35,7 @@ pub struct GenerateRequest {
pub priority: i32,
pub data_parallel_rank: Option<u32>,
pub reasoning_ended: Option<bool>,
pub lora_request: Option<OpaqueValue>,
pub lora_request: Option<LoraRequest>,
}
#[derive(Debug)]
+1
View File
@@ -91,6 +91,7 @@ pub fn to_text_request(
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
add_special_tokens: true,
data_parallel_rank: None,
lora_request: None,
})
}
+1
View File
@@ -4,6 +4,7 @@ mod config;
mod error;
mod grpc;
mod listener;
mod lora;
mod middleware;
mod routes;
mod state;
+168
View File
@@ -0,0 +1,168 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::{Mutex, RwLock};
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
/// Snapshot of the currently served model names plus the requested LoRA, if
/// the model name resolves to a dynamic adapter.
#[derive(Debug, Clone)]
pub(crate) struct LoraModelResolution {
pub model_names: Vec<String>,
pub lora_request: Option<LoraRequest>,
}
/// Runtime registry for dynamically loaded LoRA adapters.
pub(crate) struct LoraManager {
/// Dynamically loaded LoRA adapters keyed by public model name.
requests: RwLock<BTreeMap<String, LoraRequest>>,
/// Monotonic adapter id allocator. LoRA ids are one-indexed.
id_counter: AtomicU64,
/// Serialize dynamic LoRA registry updates around engine utility calls.
update_lock: Mutex<()>,
}
#[derive(Debug)]
pub(crate) enum LoadLoraError {
AlreadyLoaded { lora_name: String },
BaseModelName { lora_name: String },
Engine(vllm_engine_core_client::Error),
NotLoaded { lora_name: String },
}
#[derive(Debug)]
pub(crate) enum UnloadLoraError {
NotFound {
lora_name: String,
},
IntIdMismatch {
lora_name: String,
expected: u64,
actual: u64,
},
Engine(vllm_engine_core_client::Error),
NotRemoved {
lora_name: String,
lora_int_id: u64,
},
}
impl LoraManager {
pub fn new() -> Self {
Self {
requests: RwLock::new(BTreeMap::new()),
id_counter: AtomicU64::new(0),
update_lock: Mutex::new(()),
}
}
/// Return base served model names plus dynamically loaded LoRA adapter
/// names.
pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec<String> {
let mut names = base_model_names.to_vec();
names.extend(self.requests.read().await.keys().cloned());
names
}
/// Resolve the requested model against one consistent LoRA registry
/// snapshot.
pub async fn resolve_model(
&self,
base_model_names: &[String],
model_name: Option<&str>,
) -> LoraModelResolution {
let requests = self.requests.read().await;
let mut model_names = base_model_names.to_vec();
model_names.extend(requests.keys().cloned());
let lora_request = model_name.and_then(|name| requests.get(name).cloned());
LoraModelResolution {
model_names,
lora_request,
}
}
/// Load one dynamic LoRA adapter and register it as a public model name.
pub async fn load_lora(
&self,
engine_core_client: &EngineCoreClient,
base_model_names: &[String],
lora_name: String,
lora_path: String,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Result<LoraRequest, LoadLoraError> {
let _guard = self.update_lock.lock().await;
if base_model_names.iter().any(|name| name == &lora_name) {
return Err(LoadLoraError::BaseModelName { lora_name });
}
if !load_inplace && self.requests.read().await.contains_key(&lora_name) {
return Err(LoadLoraError::AlreadyLoaded { lora_name });
}
let lora_int_id = self
.requests
.read()
.await
.get(&lora_name)
.map(|request| request.lora_int_id)
.unwrap_or_else(|| self.id_counter.fetch_add(1, Ordering::Relaxed) + 1);
let lora_request = LoraRequest::new(
lora_name.clone(),
lora_int_id,
lora_path,
load_inplace,
is_3d_lora_weight,
);
let loaded = engine_core_client
.add_lora(&lora_request)
.await
.map_err(LoadLoraError::Engine)?;
if !loaded {
return Err(LoadLoraError::NotLoaded { lora_name });
}
self.requests.write().await.insert(lora_name, lora_request.clone());
Ok(lora_request)
}
/// Remove one dynamic LoRA adapter from the engine and public model
/// registry.
pub async fn unload_lora(
&self,
engine_core_client: &EngineCoreClient,
lora_name: &str,
requested_lora_int_id: Option<u64>,
) -> Result<LoraRequest, UnloadLoraError> {
let _guard = self.update_lock.lock().await;
let lora_request = self.requests.read().await.get(lora_name).cloned().ok_or_else(|| {
UnloadLoraError::NotFound {
lora_name: lora_name.to_string(),
}
})?;
if let Some(actual) = requested_lora_int_id
&& actual != lora_request.lora_int_id
{
return Err(UnloadLoraError::IntIdMismatch {
lora_name: lora_name.to_string(),
expected: lora_request.lora_int_id,
actual,
});
}
let removed = engine_core_client
.remove_lora(lora_request.lora_int_id)
.await
.map_err(UnloadLoraError::Engine)?;
if !removed {
return Err(UnloadLoraError::NotRemoved {
lora_name: lora_request.lora_name,
lora_int_id: lora_request.lora_int_id,
});
}
Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request))
}
}
+38 -3
View File
@@ -3,6 +3,7 @@ mod collective_rpc;
mod health;
mod inference;
mod load;
mod lora;
mod metrics;
pub(crate) mod openai;
mod sleep;
@@ -25,12 +26,40 @@ fn server_dev_mode_enabled() -> bool {
.is_some_and(|value| value != 0)
}
/// Build the minimal OpenAI-compatible router for one configured model.
pub fn build_router(state: Arc<AppState>) -> Router {
build_router_with_dev_mode(state, server_dev_mode_enabled())
fn runtime_lora_updating_enabled() -> bool {
std::env::var("VLLM_ALLOW_RUNTIME_LORA_UPDATING")
.ok()
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
}
/// Build the minimal OpenAI-compatible router for one configured model.
pub fn build_router(state: Arc<AppState>) -> Router {
build_router_with_options(
state,
server_dev_mode_enabled(),
runtime_lora_updating_enabled(),
)
}
#[cfg(test)]
fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> Router {
build_router_with_dev_mode_and_lora(state, dev_mode_enabled, false)
}
#[cfg(test)]
fn build_router_with_dev_mode_and_lora(
state: Arc<AppState>,
dev_mode_enabled: bool,
runtime_lora_updating_enabled: bool,
) -> Router {
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
}
fn build_router_with_options(
state: Arc<AppState>,
dev_mode_enabled: bool,
runtime_lora_updating_enabled: bool,
) -> Router {
let mut router = Router::new()
// Health & monitoring
.route("/health", get(health::health))
@@ -44,6 +73,12 @@ fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> R
// vLLM specific inference endpoints
.route("/inference/v1/generate", post(inference::generate));
if runtime_lora_updating_enabled {
router = router
.route("/v1/load_lora_adapter", post(lora::load_lora_adapter))
.route("/v1/unload_lora_adapter", post(lora::unload_lora_adapter));
}
if dev_mode_enabled {
// Development-only
router = router
@@ -42,8 +42,8 @@ pub async fn generate(
ValidatedJson(body): ValidatedJson<GenerateRequest>,
) -> Response {
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let prepared = match prepare_generate_request(body, state.served_model_names(), request_context)
{
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
@@ -3,6 +3,7 @@ use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
use super::types::GenerateRequest;
use super::validate;
use crate::error::ApiError;
use crate::lora::LoraModelResolution;
use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
/// Lowered generate request plus the response request ID.
@@ -21,10 +22,10 @@ pub struct PreparedRequest {
/// text-generation format.
pub fn prepare_generate_request(
request: GenerateRequest,
served_model_names: &[String],
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
let stream = request.stream;
let include_usage = request
@@ -57,6 +58,7 @@ pub fn prepare_generate_request(
cache_salt: request.cache_salt,
add_special_tokens: false,
data_parallel_rank: ctx.data_parallel_rank,
lora_request: lora_resolution.lora_request.clone(),
};
Ok(PreparedRequest {
@@ -76,9 +78,17 @@ mod tests {
use vllm_text::Prompt;
use super::prepare_generate_request;
use crate::lora::LoraModelResolution;
use crate::routes::inference::generate::types::GenerateRequest;
use crate::utils::ResolvedRequestContext;
fn served(names: &[&str]) -> LoraModelResolution {
LoraModelResolution {
model_names: names.iter().map(|s| s.to_string()).collect(),
lora_request: None,
}
}
#[test]
fn prepare_generate_request_maps_token_prompt_and_sampling_params() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -100,7 +110,7 @@ mod tests {
let prepared = prepare_generate_request(
request,
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
@@ -143,7 +153,7 @@ mod tests {
let prepared = prepare_generate_request(
request,
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
+296
View File
@@ -0,0 +1,296 @@
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use axum::extract::State;
use serde::Deserialize;
use thiserror_ext::AsReport;
use validator::Validate;
use crate::error::ApiError;
use crate::lora::{LoadLoraError, UnloadLoraError};
use crate::routes::openai::utils::types::Normalizable;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
const RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV: &str = "VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES";
#[derive(Debug, Deserialize, Validate)]
pub(crate) struct LoadLoraAdapterRequest {
lora_name: String,
lora_path: String,
#[serde(default)]
load_inplace: bool,
#[serde(default)]
is_3d_lora_weight: bool,
}
impl Normalizable for LoadLoraAdapterRequest {}
#[derive(Debug, Deserialize, Validate)]
pub(crate) struct UnloadLoraAdapterRequest {
lora_name: String,
#[serde(default)]
lora_int_id: Option<u64>,
}
impl Normalizable for UnloadLoraAdapterRequest {}
fn runtime_lora_allowed_path_prefixes() -> Option<Vec<PathBuf>> {
let prefixes = std::env::var_os(RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV)?;
let prefixes: Vec<_> = std::env::split_paths(&prefixes)
.filter(|path| !path.as_os_str().is_empty())
.collect();
(!prefixes.is_empty()).then_some(prefixes)
}
fn looks_like_local_lora_path(lora_path: &str) -> bool {
let path = Path::new(lora_path);
path.is_absolute()
|| lora_path.starts_with('~')
|| lora_path.starts_with('.')
|| path.components().any(|component| matches!(component, Component::ParentDir))
}
fn validate_lora_path_access(
lora_path: &str,
allowed_prefixes: Option<&[PathBuf]>,
) -> Result<Option<String>, ApiError> {
let path = Path::new(lora_path);
if !looks_like_local_lora_path(lora_path) && !path.exists() {
return Ok(None);
}
let Some(allowed_prefixes) = allowed_prefixes else {
return Err(ApiError::invalid_request(
format!(
"Local LoRA adapter paths require {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} to be configured."
),
Some("lora_path"),
));
};
if !path.is_absolute() {
return Err(ApiError::invalid_request(
format!(
"Local LoRA adapter paths must be absolute and under one of the prefixes configured by {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV}."
),
Some("lora_path"),
));
}
let canonical_path = path.canonicalize().map_err(|_| {
ApiError::invalid_request(
"Local LoRA adapter path must exist and be accessible.".to_string(),
Some("lora_path"),
)
})?;
let canonical_prefixes = allowed_prefixes
.iter()
.map(|prefix| {
prefix.canonicalize().map_err(|_| {
ApiError::server_error(format!(
"configured {RUNTIME_LORA_ALLOWED_PATH_PREFIXES_ENV} path prefix must exist and be accessible"
))
})
})
.collect::<Result<Vec<_>, _>>()?;
if !canonical_prefixes.iter().any(|prefix| canonical_path.starts_with(prefix)) {
return Err(ApiError::invalid_request(
"Local LoRA adapter path is outside the configured allowed prefixes.".to_string(),
Some("lora_path"),
));
}
Ok(Some(canonical_path.to_string_lossy().into_owned()))
}
/// Dynamically load one LoRA adapter and expose it as an OpenAI model id.
pub async fn load_lora_adapter(
State(state): State<Arc<AppState>>,
ValidatedJson(request): ValidatedJson<LoadLoraAdapterRequest>,
) -> Result<String, ApiError> {
if request.lora_name.is_empty() || request.lora_path.is_empty() {
return Err(ApiError::invalid_request(
"Both 'lora_name' and 'lora_path' must be provided.".to_string(),
None,
));
}
let allowed_prefixes = runtime_lora_allowed_path_prefixes();
let lora_path = validate_lora_path_access(&request.lora_path, allowed_prefixes.as_deref())?
.unwrap_or(request.lora_path);
let lora_name = request.lora_name;
state
.load_lora(
lora_name.clone(),
lora_path,
request.load_inplace,
request.is_3d_lora_weight,
)
.await
.map_err(|error| match error {
LoadLoraError::AlreadyLoaded { lora_name } => ApiError::invalid_request(
format!(
"The lora adapter '{lora_name}' has already been loaded. If you want to load the adapter in place, set 'load_inplace' to true."
),
Some("lora_name"),
),
LoadLoraError::BaseModelName { lora_name } => ApiError::invalid_request(
format!("The lora adapter name '{lora_name}' conflicts with a served base model."),
Some("lora_name"),
),
LoadLoraError::Engine(error) => ApiError::server_error(format!(
"failed to load LoRA adapter '{lora_name}': {}",
error.to_report_string()
)),
LoadLoraError::NotLoaded { lora_name } => ApiError::server_error(format!(
"failed to load LoRA adapter '{lora_name}': engine rejected the adapter"
)),
})?;
Ok(format!(
"Success: LoRA adapter '{lora_name}' added successfully."
))
}
/// Remove one LoRA adapter from the engine and frontend registry.
pub async fn unload_lora_adapter(
State(state): State<Arc<AppState>>,
ValidatedJson(request): ValidatedJson<UnloadLoraAdapterRequest>,
) -> Result<String, ApiError> {
if request.lora_name.is_empty() {
return Err(ApiError::invalid_request(
"'lora_name' needs to be provided to unload a LoRA adapter.".to_string(),
Some("lora_name"),
));
}
let lora_request = state
.unload_lora(&request.lora_name, request.lora_int_id)
.await
.map_err(|error| match error {
UnloadLoraError::NotFound { lora_name } => ApiError::model_not_found(lora_name),
UnloadLoraError::IntIdMismatch {
lora_name,
expected,
actual,
} => ApiError::invalid_request(
format!(
"The requested lora_int_id {actual} does not match loaded adapter '{lora_name}' with id {expected}."
),
Some("lora_int_id"),
),
UnloadLoraError::Engine(error) => ApiError::server_error(format!(
"failed to unload LoRA adapter '{}': {}",
request.lora_name,
error.to_report_string()
)),
UnloadLoraError::NotRemoved {
lora_name,
lora_int_id,
} => ApiError::server_error(format!(
"failed to unload LoRA adapter '{lora_name}' with id {lora_int_id}"
)),
})?;
Ok(format!(
"Success: LoRA adapter '{}' removed successfully.",
lora_request.lora_name
))
}
#[cfg(test)]
mod tests {
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use super::validate_lora_path_access;
fn temp_lora_dir(test_name: &str) -> PathBuf {
let suffix = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("clock should be after unix epoch")
.as_nanos();
let path = std::env::temp_dir().join(format!(
"vllm-lora-{test_name}-{}-{suffix}",
std::process::id()
));
fs::create_dir_all(&path).expect("create temp lora dir");
path
}
#[test]
fn lora_path_allows_hf_repo_ids_without_prefixes() {
assert_eq!(
validate_lora_path_access("org/adapter-a", None).expect("hf repo id should be allowed"),
None
);
}
#[test]
fn lora_path_rejects_local_paths_without_prefixes() {
assert!(validate_lora_path_access("/tmp/adapter-a", None).is_err());
assert!(validate_lora_path_access("./adapter-a", None).is_err());
assert!(validate_lora_path_access("~/adapter-a", None).is_err());
assert!(validate_lora_path_access("subdir/../../../etc/sensitive", None).is_err());
}
#[test]
fn lora_path_rejects_existing_bare_relative_paths_without_prefixes() {
let root =
PathBuf::from("target").join(format!("vllm-lora-relative-{}", std::process::id()));
let adapter = root.join("adapter-a");
fs::create_dir_all(&adapter).expect("create relative adapter dir");
assert!(
validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), None).is_err()
);
fs::remove_dir_all(root).ok();
}
#[test]
fn lora_path_allows_absolute_paths_under_configured_prefixes() {
let root = temp_lora_dir("allowed-prefix");
let allowed = root.join("allowed");
let adapter = allowed.join("adapter-a");
fs::create_dir_all(&adapter).expect("create adapter dir");
let prefixes = [allowed];
let resolved =
validate_lora_path_access(adapter.to_str().expect("utf-8 temp path"), Some(&prefixes))
.expect("path under configured prefix should be allowed");
assert_eq!(
resolved.as_deref(),
Some(
adapter
.canonicalize()
.expect("canonical adapter")
.to_str()
.expect("utf-8 temp path")
)
);
fs::remove_dir_all(root).ok();
}
#[test]
fn lora_path_rejects_parent_escape_from_configured_prefixes() {
let root = temp_lora_dir("parent-escape");
let allowed = root.join("allowed");
let private_adapter = root.join("private").join("adapter-a");
fs::create_dir_all(&allowed).expect("create allowed dir");
fs::create_dir_all(&private_adapter).expect("create private adapter dir");
let escaped = allowed.join("../private/adapter-a");
let prefixes = [allowed];
assert!(
validate_lora_path_access(escaped.to_str().expect("utf-8 temp path"), Some(&prefixes))
.is_err()
);
fs::remove_dir_all(root).ok();
}
}
@@ -49,8 +49,9 @@ pub async fn chat_completions(
) -> Response {
let stream = body.stream;
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
let prepared = match prepare_chat_request(body, state.served_model_names(), request_context) {
let prepared = match prepare_chat_request(body, &lora_resolution, request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
@@ -8,6 +8,7 @@ use vllm_chat::{
use super::types::ChatCompletionRequest;
use super::validate;
use crate::error::{ApiError, bail_invalid_request};
use crate::lora::LoraModelResolution;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
@@ -41,16 +42,21 @@ pub struct PreparedRequest {
/// Validate and lower one OpenAI chat completion request into the internal chat
/// format.
///
/// `served_model_names` must be non-empty; the first entry is used as the
/// `model` field in responses.
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
/// the base `model` field in responses when no LoRA adapter is selected.
pub(crate) fn prepare_chat_request(
request: ChatCompletionRequest,
served_model_names: &[String],
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
let request_id = format!("chatcmpl-{}", ctx.request_id);
let response_model = lora_resolution
.lora_request
.as_ref()
.map(|request| request.lora_name.clone())
.unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
let echo = request
.echo
.then(|| extract_last_assistant_content(&request.messages))
@@ -131,11 +137,12 @@ pub(crate) fn prepare_chat_request(
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
lora_request: lora_resolution.lora_request.clone(),
};
Ok(PreparedRequest {
request_id,
response_model: served_model_names.first().cloned().unwrap_or_default(),
response_model,
include_usage,
requested_logprobs,
include_prompt_logprobs,
@@ -352,6 +359,7 @@ mod tests {
use vllm_text::output::TextDecodeOptions;
use super::prepare_chat_request;
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
@@ -365,8 +373,11 @@ mod tests {
resolve_request_context(headers, request_id)
}
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
fn served(names: &[&str]) -> LoraModelResolution {
LoraModelResolution {
model_names: names.iter().map(|s| s.to_string()).collect(),
lora_request: None,
}
}
fn base_request() -> ChatCompletionRequest {
@@ -44,12 +44,12 @@ pub async fn completions(
let stream = body.stream;
let logprobs = body.logprobs;
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
let prepared =
match prepare_completion_request(body, state.served_model_names(), request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let prepared = match prepare_completion_request(body, &lora_resolution, request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let request_span = tracing::info_span!(
"completions",
request_id = %prepared.request_id,
@@ -2,6 +2,7 @@ use vllm_text::{SamplingParams, TextDecodeOptions, TextRequest};
use super::types::CompletionRequest;
use crate::error::ApiError;
use crate::lora::LoraModelResolution;
use crate::routes::openai::completions::validate;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer_params};
@@ -30,16 +31,21 @@ pub struct PreparedRequest {
/// Validate and lower one OpenAI completions request into the internal
/// text-generation format.
///
/// `served_model_names` must be non-empty; the first entry is used as the
/// `model` field in responses.
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
/// the base `model` field in responses when no LoRA adapter is selected.
pub(crate) fn prepare_completion_request(
request: CompletionRequest,
served_model_names: &[String],
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, served_model_names)?;
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
let request_id = format!("cmpl-{}", ctx.request_id);
let response_model = lora_resolution
.lora_request
.as_ref()
.map(|request| request.lora_name.clone())
.unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
let logprobs = match request.logprobs {
Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
@@ -104,11 +110,12 @@ pub(crate) fn prepare_completion_request(
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
lora_request: lora_resolution.lora_request.clone(),
};
Ok(PreparedRequest {
request_id,
response_model: served_model_names.first().cloned().unwrap_or_default(),
response_model,
include_usage,
text_request,
echo,
@@ -124,6 +131,7 @@ mod tests {
use vllm_text::Prompt;
use super::prepare_completion_request;
use crate::lora::LoraModelResolution;
use crate::routes::openai::completions::types::CompletionRequest;
use crate::utils::{ResolvedRequestContext, resolve_request_context};
@@ -131,8 +139,11 @@ mod tests {
resolve_request_context(headers, request_id)
}
fn served(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
fn served(names: &[&str]) -> LoraModelResolution {
LoraModelResolution {
model_names: names.iter().map(|s| s.to_string()).collect(),
lora_request: None,
}
}
fn base_request_json() -> serde_json::Value {
+4 -4
View File
@@ -8,13 +8,13 @@ use crate::state::AppState;
/// Return all configured served model names in OpenAI `list models` format.
pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
let model_names = state.served_model_names_with_loras().await;
Json(ListModelsResponse {
object: "list".to_string(),
data: state
.served_model_names()
.iter()
data: model_names
.into_iter()
.map(|name| ModelObject {
id: name.clone(),
id: name,
object: "model".to_string(),
created: 0,
owned_by: "vllm-frontend-rs".to_string(),
+376 -3
View File
@@ -43,7 +43,8 @@ use vllm_text::{Prompt, TextBackend};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{build_router, build_router_with_dev_mode};
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
use crate::state::AppState;
@@ -141,6 +142,13 @@ fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason
]
}
fn assert_adapter_a_lora_request(request: &EngineCoreRequest) {
let lora = request.lora_request.as_ref().expect("lora request");
assert_eq!(lora.lora_name, "adapter-a");
assert_eq!(lora.lora_int_id, 1);
assert_eq!(lora.lora_path, "org/adapter-a");
}
fn sse_data_payloads(text: &str) -> Vec<&str> {
text.lines().filter_map(|line| line.strip_prefix("data: ")).collect()
}
@@ -822,12 +830,13 @@ where
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
(
build_router_with_dev_mode(
build_router_with_dev_mode_and_lora(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
true,
),
engine_task,
)
@@ -1078,6 +1087,367 @@ async fn version_returns_engine_vllm_version() {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("add_lora"));
let args = array[3].as_array().expect("utility args");
let lora = args[0].as_array().expect("lora request tuple");
assert_eq!(lora[0], Value::from("adapter-a"));
assert_eq!(lora[1], Value::from(1));
assert_eq!(lora[2], Value::from("org/adapter-a"));
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode engine request");
assert_adapter_a_lora_request(&request);
send_outputs(
push,
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
)
.await;
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode engine request");
assert_adapter_a_lora_request(&request);
send_outputs(
push,
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
)
.await;
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode engine request");
assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22][..]));
assert_adapter_a_lora_request(&request);
send_outputs(
push,
engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
)
.await;
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("remove_lora"));
let args = array[3].as_array().expect("utility args");
assert_eq!(args[0], Value::from(1));
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/load_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "adapter-a",
"lora_path": "org/adapter-a"
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let models = app
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
.await
.expect("call app");
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["data"][1]["id"], "adapter-a");
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "adapter-a",
"prompt": "hello",
"max_tokens": 2
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "adapter-a",
"stream": false,
"messages": [{"role": "user", "content": "hello"}]
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "adapter-a",
"token_ids": [11, 22],
"stream": false,
"sampling_params": {
"max_tokens": 2
}
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/unload_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "adapter-a",
"lora_int_id": 1
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let models = app
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
.await
.expect("call app");
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["data"].as_array().expect("model data").len(), 1);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "adapter-a",
"prompt": "hello",
"max_tokens": 2
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
drop(app);
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("add_lora"));
send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
})
})
.await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/load_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "adapter-a",
"lora_path": "org/adapter-a"
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/unload_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "adapter-a",
"lora_int_id": 99
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let models = app
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
.await
.expect("call app");
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["data"][1]["id"], "adapter-a");
drop(app);
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn load_lora_adapter_rejects_engine_false_result() {
let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| {
boxed_test_future(async move {
let utility = recv_engine_message(dealer).await;
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("add_lora"));
send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await;
})
})
.await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/load_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "adapter-a",
"lora_path": "org/adapter-a"
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let models = app
.call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
.await
.expect("call app");
let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["data"].as_array().expect("model data").len(), 1);
drop(app);
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn load_lora_adapter_rejects_base_model_name_collision() {
let (mut app, engine_task) =
test_admin_app_with_engine_script(|_, _| boxed_test_future(async move {})).await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/v1/load_lora_adapter")
.header("content-type", "application/json")
.body(Body::from(
json!({
"lora_name": "Qwen/Qwen1.5-0.5B-Chat",
"lora_path": "org/adapter-a"
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
drop(app);
engine_task.finish().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn http_metrics_record_list_models_requests() {
@@ -3000,7 +3370,10 @@ async fn prepared_openai_request_streams_text_events() {
"messages": [{"role": "user", "content": "hello"}]
}))
.expect("decode request"),
&["Qwen/Qwen1.5-0.5B-Chat".to_string()],
&LoraModelResolution {
model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
lora_request: None,
},
crate::utils::ResolvedRequestContext::default(),
)
.expect("prepare request");
+49
View File
@@ -5,6 +5,9 @@ use tokio::time::{Duration, Instant, sleep_until};
use tracing::warn;
use vllm_chat::ChatLlm;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
@@ -21,6 +24,8 @@ pub struct AppState {
pub enable_request_id_headers: bool,
/// Number of in-flight inference requests currently owned by this frontend.
server_load: AtomicU64,
/// Dynamic LoRA adapter registry.
lora_manager: LoraManager,
}
impl AppState {
@@ -43,6 +48,7 @@ impl AppState {
enable_log_requests: false,
enable_request_id_headers: false,
server_load: AtomicU64::new(0),
lora_manager: LoraManager::new(),
}
}
@@ -69,6 +75,49 @@ impl AppState {
&self.served_model_names
}
/// Return base served model names plus dynamically loaded LoRA adapter
/// names.
pub async fn served_model_names_with_loras(&self) -> Vec<String> {
self.lora_manager.served_model_names(&self.served_model_names).await
}
/// Resolve the requested model against one dynamic LoRA registry snapshot.
pub async fn resolve_model_with_loras(&self, model_name: Option<&str>) -> LoraModelResolution {
self.lora_manager.resolve_model(&self.served_model_names, model_name).await
}
/// Load one dynamic LoRA adapter and register it as a public model name.
pub async fn load_lora(
&self,
lora_name: String,
lora_path: String,
load_inplace: bool,
is_3d_lora_weight: bool,
) -> Result<LoraRequest, LoadLoraError> {
self.lora_manager
.load_lora(
self.engine_core_client(),
&self.served_model_names,
lora_name,
lora_path,
load_inplace,
is_3d_lora_weight,
)
.await
}
/// Remove one dynamic LoRA adapter from the engine and public model
/// registry.
pub async fn unload_lora(
&self,
lora_name: &str,
lora_int_id: Option<u64>,
) -> Result<LoraRequest, UnloadLoraError> {
self.lora_manager
.unload_lora(self.engine_core_client(), lora_name, lora_int_id)
.await
}
/// Return a reference to the underlying engine core client for utility
/// calls.
pub(crate) fn engine_core_client(&self) -> &EngineCoreClient {
+1 -1
View File
@@ -40,11 +40,11 @@ pub fn lower_text_request(
cache_salt: request.cache_salt.clone(),
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
lora_request: request.lora_request.clone(),
// Fields below are currently placeholders.
arrival_time: None,
trace_headers: None,
reasoning_ended: None,
lora_request: None,
};
Ok(PreparedTextRequest {
+5
View File
@@ -4,6 +4,7 @@ use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::StructuredOutputsParams;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use crate::error::{Error, Result};
@@ -166,6 +167,9 @@ pub struct TextRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
}
impl TextRequest {
@@ -182,6 +186,7 @@ impl TextRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
lora_request: None,
}
}