From 22170354d83a8a9a845a3f50348301e007b9d81a Mon Sep 17 00:00:00 2001 From: Chauncey Date: Thu, 6 Aug 2026 12:10:29 +0800 Subject: [PATCH] [Feature] Parse request priority from HTTP header (#51089) Signed-off-by: chaunceyjiang --- .../openai_compatible_server.md | 17 ++++++++++++-- .../routes/openai/chat_completions/convert.rs | 19 ++++++++++++++- .../src/routes/openai/completions/convert.rs | 23 ++++++++++++++++++- rust/src/server/src/utils.rs | 9 +++++++- vllm/entrypoints/generate/base/serving.py | 15 ++++++++++++ .../openai/chat_completion/serving.py | 2 +- vllm/entrypoints/openai/completion/serving.py | 2 +- vllm/entrypoints/openai/responses/serving.py | 2 +- 8 files changed, 81 insertions(+), 8 deletions(-) diff --git a/docs/serving/online_serving/openai_compatible_server.md b/docs/serving/online_serving/openai_compatible_server.md index e50754aa9c..dfb78245b4 100644 --- a/docs/serving/online_serving/openai_compatible_server.md +++ b/docs/serving/online_serving/openai_compatible_server.md @@ -83,8 +83,8 @@ completion = client.chat.completions.create( ## Extra HTTP Headers -Only `X-Request-Id` HTTP request header is supported for now. It can be enabled -with `--enable-request-id-headers`. +The `X-Request-Id` HTTP request header can be enabled with +`--enable-request-id-headers`. ??? code @@ -110,6 +110,19 @@ with `--enable-request-id-headers`. print(completion._request_id) ``` +The Completions, Chat Completions, and Responses APIs also support the +`X-Vllm-Priority` request header. Its value must be an integer and overrides +the `priority` value in the JSON request body. Non-zero priorities require the +server to use priority scheduling. + +```python +completion = client.chat.completions.create( + model="NousResearch/Meta-Llama-3-8B-Instruct", + messages=[{"role": "user", "content": "Hello!"}], + extra_headers={"X-Vllm-Priority": "-10"}, +) +``` + ## API Reference ### Completions API diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 030f50f972..521643fb69 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request( min_tokens: request.min_tokens.unwrap_or(0), }, intermediate: request.stream, - priority: request.priority.unwrap_or(0), + priority: ctx.priority.or(request.priority).unwrap_or(0), documents: request.documents, cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, @@ -1264,6 +1264,23 @@ mod tests { ); } + #[test] + fn prepare_chat_request_header_priority_overrides_body() { + let mut headers = HeaderMap::new(); + headers.insert("X-Vllm-Priority", "-5".parse().unwrap()); + let request = ChatCompletionRequest { + priority: Some(10), + ..base_request() + }; + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + request_context(&headers, None), + ) + .expect("request is valid"); + assert_eq!(prepared.chat_request.priority, -5); + } + #[test] fn prepare_chat_request_ignores_correlation_header() { let mut headers = HeaderMap::new(); diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 1143da77f2..c2651064eb 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request( min_tokens: request.min_tokens.unwrap_or(0), }, intermediate: request.stream, - priority: request.priority.unwrap_or(0), + priority: ctx.priority.or(request.priority).unwrap_or(0), cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: ctx.data_parallel_rank, @@ -619,6 +619,27 @@ mod tests { ); } + #[test] + fn prepare_completion_request_header_priority_overrides_body() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "priority": 10, + })) + .expect("parse request"); + + let mut headers = HeaderMap::new(); + headers.insert("X-Vllm-Priority", "-5".parse().unwrap()); + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + request_context(&headers, None), + &test_tokenizer(), + ) + .expect("prepare"); + assert_eq!(prepared.text_request.priority, -5); + } + #[test] fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() { let request: CompletionRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/utils.rs b/rust/src/server/src/utils.rs index 2c9e0e5112..b58e89fd3f 100644 --- a/rust/src/server/src/utils.rs +++ b/rust/src/server/src/utils.rs @@ -15,6 +15,7 @@ use crate::error::ApiError; pub struct ResolvedRequestContext { pub request_id: String, pub data_parallel_rank: Option, + pub priority: Option, pub session_id: Option, } @@ -111,7 +112,8 @@ pub fn convert_logit_bias( } /// Extract common request metadata from HTTP headers: the external request ID, -/// session ID, and the optional data-parallel rank used for engine routing. +/// session ID, priority, and the optional data-parallel rank used for engine +/// routing. pub fn resolve_request_context( headers: &HeaderMap, request_id: Option<&str>, @@ -121,6 +123,10 @@ pub fn resolve_request_context( .get("X-data-parallel-rank") .and_then(|v| v.to_str().ok()) .and_then(|s| s.trim().parse().ok()); + let priority = headers + .get("X-Vllm-Priority") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse().ok()); // Extract request id from header. let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok()); @@ -134,6 +140,7 @@ pub fn resolve_request_context( ResolvedRequestContext { request_id, data_parallel_rank, + priority, session_id, } } diff --git a/vllm/entrypoints/generate/base/serving.py b/vllm/entrypoints/generate/base/serving.py index 2497f2f05c..4f27b3bc35 100644 --- a/vllm/entrypoints/generate/base/serving.py +++ b/vllm/entrypoints/generate/base/serving.py @@ -42,6 +42,7 @@ logger = init_logger(__name__) RequestT = TypeVar("RequestT", bound=AnyRequest) _T = TypeVar("_T") SESSION_ID_HEADER = "X-Session-ID" +PRIORITY_HEADER = "X-Vllm-Priority" def build_per_request_timing_metrics( @@ -239,6 +240,20 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin): return session_id return None + @staticmethod + def _get_priority( + request: ChatCompletionRequest | CompletionRequest | ResponsesRequest, + raw_request: Request | None, + ) -> int: + if raw_request is not None: + priority = raw_request.headers.get(PRIORITY_HEADER) + if priority is not None: + try: + return int(priority) + except ValueError: + pass + return request.priority + async def _with_kv_transfer_rejection_cleanup( self, awaitable: Awaitable[_T], diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 3ab4c1ceaa..f2d4936f1e 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -348,7 +348,7 @@ class OpenAIServingChat(GenerateBaseServing): sub_request_id, lora_request=lora_request, trace_headers=trace_headers, - priority=request.priority, + priority=self._get_priority(request, raw_request), data_parallel_rank=data_parallel_rank, session_id=session_id, reasoning_ended=reasoning_ended, diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 7c47e777bf..66b6016d9f 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -212,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing): request_id_item, lora_request=lora_request, trace_headers=trace_headers, - priority=request.priority, + priority=self._get_priority(request, raw_request), data_parallel_rank=data_parallel_rank, session_id=session_id, ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 44b7802957..9d8c5dafb9 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -515,7 +515,7 @@ class OpenAIServingResponses(GenerateBaseServing): sampling_params=sampling_params, context=context, lora_request=lora_request, - priority=request.priority, + priority=self._get_priority(request, raw_request), trace_headers=trace_headers, session_id=session_id, reasoning_parser_kwargs=reasoning_parser_kwargs