[Feature] Parse request priority from HTTP header (#51089)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
documents: request.documents,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
@@ -1264,6 +1264,23 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_header_priority_overrides_body() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
|
||||
let request = ChatCompletionRequest {
|
||||
priority: Some(10),
|
||||
..base_request()
|
||||
};
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.priority, -5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_correlation_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
@@ -619,6 +619,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_header_priority_overrides_body() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"priority": 10,
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.priority, -5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::error::ApiError;
|
||||
pub struct ResolvedRequestContext {
|
||||
pub request_id: String,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub priority: Option<i32>,
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
@@ -111,7 +112,8 @@ pub fn convert_logit_bias(
|
||||
}
|
||||
|
||||
/// Extract common request metadata from HTTP headers: the external request ID,
|
||||
/// session ID, and the optional data-parallel rank used for engine routing.
|
||||
/// session ID, priority, and the optional data-parallel rank used for engine
|
||||
/// routing.
|
||||
pub fn resolve_request_context(
|
||||
headers: &HeaderMap,
|
||||
request_id: Option<&str>,
|
||||
@@ -121,6 +123,10 @@ pub fn resolve_request_context(
|
||||
.get("X-data-parallel-rank")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.trim().parse().ok());
|
||||
let priority = headers
|
||||
.get("X-Vllm-Priority")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.trim().parse().ok());
|
||||
|
||||
// Extract request id from header.
|
||||
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
|
||||
@@ -134,6 +140,7 @@ pub fn resolve_request_context(
|
||||
ResolvedRequestContext {
|
||||
request_id,
|
||||
data_parallel_rank,
|
||||
priority,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user