[Feature] Parse request priority from HTTP header (#51089)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-08-05 21:10:29 -07:00
committed by GitHub
parent 2e09247c2d
commit 22170354d8
8 changed files with 81 additions and 8 deletions
@@ -83,8 +83,8 @@ completion = client.chat.completions.create(
## Extra HTTP Headers
Only `X-Request-Id` HTTP request header is supported for now. It can be enabled
with `--enable-request-id-headers`.
The `X-Request-Id` HTTP request header can be enabled with
`--enable-request-id-headers`.
??? code
@@ -110,6 +110,19 @@ with `--enable-request-id-headers`.
print(completion._request_id)
```
The Completions, Chat Completions, and Responses APIs also support the
`X-Vllm-Priority` request header. Its value must be an integer and overrides
the `priority` value in the JSON request body. Non-zero priorities require the
server to use priority scheduling.
```python
completion = client.chat.completions.create(
model="NousResearch/Meta-Llama-3-8B-Instruct",
messages=[{"role": "user", "content": "Hello!"}],
extra_headers={"X-Vllm-Priority": "-10"},
)
```
## API Reference
### Completions API
@@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
priority: ctx.priority.or(request.priority).unwrap_or(0),
documents: request.documents,
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
@@ -1264,6 +1264,23 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_header_priority_overrides_body() {
let mut headers = HeaderMap::new();
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
let request = ChatCompletionRequest {
priority: Some(10),
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
)
.expect("request is valid");
assert_eq!(prepared.chat_request.priority, -5);
}
#[test]
fn prepare_chat_request_ignores_correlation_header() {
let mut headers = HeaderMap::new();
@@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
priority: request.priority.unwrap_or(0),
priority: ctx.priority.or(request.priority).unwrap_or(0),
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
@@ -619,6 +619,27 @@ mod tests {
);
}
#[test]
fn prepare_completion_request_header_priority_overrides_body() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"priority": 10,
}))
.expect("parse request");
let mut headers = HeaderMap::new();
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.priority, -5);
}
#[test]
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
let request: CompletionRequest = serde_json::from_value(json!({
+8 -1
View File
@@ -15,6 +15,7 @@ use crate::error::ApiError;
pub struct ResolvedRequestContext {
pub request_id: String,
pub data_parallel_rank: Option<u32>,
pub priority: Option<i32>,
pub session_id: Option<String>,
}
@@ -111,7 +112,8 @@ pub fn convert_logit_bias(
}
/// Extract common request metadata from HTTP headers: the external request ID,
/// session ID, and the optional data-parallel rank used for engine routing.
/// session ID, priority, and the optional data-parallel rank used for engine
/// routing.
pub fn resolve_request_context(
headers: &HeaderMap,
request_id: Option<&str>,
@@ -121,6 +123,10 @@ pub fn resolve_request_context(
.get("X-data-parallel-rank")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.trim().parse().ok());
let priority = headers
.get("X-Vllm-Priority")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse().ok());
// Extract request id from header.
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
@@ -134,6 +140,7 @@ pub fn resolve_request_context(
ResolvedRequestContext {
request_id,
data_parallel_rank,
priority,
session_id,
}
}
+15
View File
@@ -42,6 +42,7 @@ logger = init_logger(__name__)
RequestT = TypeVar("RequestT", bound=AnyRequest)
_T = TypeVar("_T")
SESSION_ID_HEADER = "X-Session-ID"
PRIORITY_HEADER = "X-Vllm-Priority"
def build_per_request_timing_metrics(
@@ -239,6 +240,20 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
return session_id
return None
@staticmethod
def _get_priority(
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
raw_request: Request | None,
) -> int:
if raw_request is not None:
priority = raw_request.headers.get(PRIORITY_HEADER)
if priority is not None:
try:
return int(priority)
except ValueError:
pass
return request.priority
async def _with_kv_transfer_rejection_cleanup(
self,
awaitable: Awaitable[_T],
@@ -348,7 +348,7 @@ class OpenAIServingChat(GenerateBaseServing):
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
session_id=session_id,
reasoning_ended=reasoning_ended,
@@ -212,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
session_id=session_id,
)
+1 -1
View File
@@ -515,7 +515,7 @@ class OpenAIServingResponses(GenerateBaseServing):
sampling_params=sampling_params,
context=context,
lora_request=lora_request,
priority=request.priority,
priority=self._get_priority(request, raw_request),
trace_headers=trace_headers,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs