[EP/DP][API Server] Enable DP-aware routing in OpenAI API requests (#24945)

Co-authored-by: Cong Chen <prowindy@gmail.com>
This commit is contained in:
cong-meta
2025-10-30 12:10:16 -07:00
committed by GitHub
co-authored by Cong Chen
parent 4574d48bab
commit a2981c4272
4 changed files with 99 additions and 0 deletions
+15
View File
@@ -1298,6 +1298,21 @@ class OpenAIServing:
return raw_request.headers.get("X-Request-Id", default)
@staticmethod
def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
"""Pulls the data parallel rank from a header, if provided"""
if raw_request is None:
return None
rank_str = raw_request.headers.get("X-data-parallel-rank")
if rank_str is None:
return None
try:
return int(rank_str)
except ValueError:
return None
@staticmethod
def _get_decoded_token(
logprob: Logprob,