[Feature] Parse request priority from HTTP header (#51089)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-08-05 21:10:29 -07:00
committed by GitHub
parent 2e09247c2d
commit 22170354d8
8 changed files with 81 additions and 8 deletions
+15
View File
@@ -42,6 +42,7 @@ logger = init_logger(__name__)
RequestT = TypeVar("RequestT", bound=AnyRequest)
_T = TypeVar("_T")
SESSION_ID_HEADER = "X-Session-ID"
PRIORITY_HEADER = "X-Vllm-Priority"
def build_per_request_timing_metrics(
@@ -239,6 +240,20 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
return session_id
return None
@staticmethod
def _get_priority(
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
raw_request: Request | None,
) -> int:
if raw_request is not None:
priority = raw_request.headers.get(PRIORITY_HEADER)
if priority is not None:
try:
return int(priority)
except ValueError:
pass
return request.priority
async def _with_kv_transfer_rejection_cleanup(
self,
awaitable: Awaitable[_T],
@@ -348,7 +348,7 @@ class OpenAIServingChat(GenerateBaseServing):
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
session_id=session_id,
reasoning_ended=reasoning_ended,
@@ -212,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
session_id=session_id,
)
+1 -1
View File
@@ -515,7 +515,7 @@ class OpenAIServingResponses(GenerateBaseServing):
sampling_params=sampling_params,
context=context,
lora_request=lora_request,
priority=request.priority,
priority=self._get_priority(request, raw_request),
trace_headers=trace_headers,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs