[Feature] Parse request priority from HTTP header (#51089)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-08-05 21:10:29 -07:00
committed by GitHub
parent 2e09247c2d
commit 22170354d8
8 changed files with 81 additions and 8 deletions
@@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
priority: ctx.priority.or(request.priority).unwrap_or(0),
documents: request.documents,
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
@@ -1264,6 +1264,23 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_header_priority_overrides_body() {
let mut headers = HeaderMap::new();
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
let request = ChatCompletionRequest {
priority: Some(10),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.priority, -5);
}
#[test]
fn prepare_chat_request_ignores_correlation_header() {
let mut headers = HeaderMap::new();
@@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
priority: ctx.priority.or(request.priority).unwrap_or(0),
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
@@ -619,6 +619,27 @@ mod tests {
);
}
#[test]
fn prepare_completion_request_header_priority_overrides_body() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"priority": 10,
}))
.expect("parse request");
let mut headers = HeaderMap::new();
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.priority, -5);
}
#[test]
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
let request: CompletionRequest = serde_json::from_value(json!({
+8 -1
View File
@@ -15,6 +15,7 @@ use crate::error::ApiError;
pub struct ResolvedRequestContext {
pub request_id: String,
pub data_parallel_rank: Option<u32>,
pub priority: Option<i32>,
pub session_id: Option<String>,
}
@@ -111,7 +112,8 @@ pub fn convert_logit_bias(
}
/// Extract common request metadata from HTTP headers: the external request ID,
/// session ID, and the optional data-parallel rank used for engine routing.
/// session ID, priority, and the optional data-parallel rank used for engine
/// routing.
pub fn resolve_request_context(
headers: &HeaderMap,
request_id: Option<&str>,
@@ -121,6 +123,10 @@ pub fn resolve_request_context(
.get("X-data-parallel-rank")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.trim().parse().ok());
let priority = headers
.get("X-Vllm-Priority")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse().ok());
// Extract request id from header.
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
@@ -134,6 +140,7 @@ pub fn resolve_request_context(
ResolvedRequestContext {
request_id,
data_parallel_rank,
priority,
session_id,
}
}