[Feature] Parse request priority from HTTP header (#51089)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -83,8 +83,8 @@ completion = client.chat.completions.create(
|
||||
|
||||
## Extra HTTP Headers
|
||||
|
||||
Only `X-Request-Id` HTTP request header is supported for now. It can be enabled
|
||||
with `--enable-request-id-headers`.
|
||||
The `X-Request-Id` HTTP request header can be enabled with
|
||||
`--enable-request-id-headers`.
|
||||
|
||||
??? code
|
||||
|
||||
@@ -110,6 +110,19 @@ with `--enable-request-id-headers`.
|
||||
print(completion._request_id)
|
||||
```
|
||||
|
||||
The Completions, Chat Completions, and Responses APIs also support the
|
||||
`X-Vllm-Priority` request header. Its value must be an integer and overrides
|
||||
the `priority` value in the JSON request body. Non-zero priorities require the
|
||||
server to use priority scheduling.
|
||||
|
||||
```python
|
||||
completion = client.chat.completions.create(
|
||||
model="NousResearch/Meta-Llama-3-8B-Instruct",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
extra_headers={"X-Vllm-Priority": "-10"},
|
||||
)
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Completions API
|
||||
|
||||
@@ -178,7 +178,7 @@ pub(super) fn prepare_chat_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
documents: request.documents,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
@@ -1264,6 +1264,23 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_header_priority_overrides_body() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
|
||||
let request = ChatCompletionRequest {
|
||||
priority: Some(10),
|
||||
..base_request()
|
||||
};
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
)
|
||||
.expect("request is valid");
|
||||
assert_eq!(prepared.chat_request.priority, -5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_ignores_correlation_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -149,7 +149,7 @@ pub(super) fn prepare_completion_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
priority: request.priority.unwrap_or(0),
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
@@ -619,6 +619,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_header_priority_overrides_body() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"priority": 10,
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("X-Vllm-Priority", "-5".parse().unwrap());
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.priority, -5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_uses_vllm_xargs_session_id_fallback() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::error::ApiError;
|
||||
pub struct ResolvedRequestContext {
|
||||
pub request_id: String,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub priority: Option<i32>,
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
@@ -111,7 +112,8 @@ pub fn convert_logit_bias(
|
||||
}
|
||||
|
||||
/// Extract common request metadata from HTTP headers: the external request ID,
|
||||
/// session ID, and the optional data-parallel rank used for engine routing.
|
||||
/// session ID, priority, and the optional data-parallel rank used for engine
|
||||
/// routing.
|
||||
pub fn resolve_request_context(
|
||||
headers: &HeaderMap,
|
||||
request_id: Option<&str>,
|
||||
@@ -121,6 +123,10 @@ pub fn resolve_request_context(
|
||||
.get("X-data-parallel-rank")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.trim().parse().ok());
|
||||
let priority = headers
|
||||
.get("X-Vllm-Priority")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.trim().parse().ok());
|
||||
|
||||
// Extract request id from header.
|
||||
let request_id_header = headers.get("X-Request-Id").and_then(|value| value.to_str().ok());
|
||||
@@ -134,6 +140,7 @@ pub fn resolve_request_context(
|
||||
ResolvedRequestContext {
|
||||
request_id,
|
||||
data_parallel_rank,
|
||||
priority,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ logger = init_logger(__name__)
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
_T = TypeVar("_T")
|
||||
SESSION_ID_HEADER = "X-Session-ID"
|
||||
PRIORITY_HEADER = "X-Vllm-Priority"
|
||||
|
||||
|
||||
def build_per_request_timing_metrics(
|
||||
@@ -239,6 +240,20 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
return session_id
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_priority(
|
||||
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
|
||||
raw_request: Request | None,
|
||||
) -> int:
|
||||
if raw_request is not None:
|
||||
priority = raw_request.headers.get(PRIORITY_HEADER)
|
||||
if priority is not None:
|
||||
try:
|
||||
return int(priority)
|
||||
except ValueError:
|
||||
pass
|
||||
return request.priority
|
||||
|
||||
async def _with_kv_transfer_rejection_cleanup(
|
||||
self,
|
||||
awaitable: Awaitable[_T],
|
||||
|
||||
@@ -348,7 +348,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=reasoning_ended,
|
||||
|
||||
@@ -212,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
@@ -515,7 +515,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
sampling_params=sampling_params,
|
||||
context=context,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
||||
|
||||
Reference in New Issue
Block a user