[Frontend] Forward X-data-parallel-rank header on /inference/v1/generate (#42330)
Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
6f21558da1
commit
cb600d1cdb
@@ -197,6 +197,9 @@ class ServingTokens(OpenAIServing):
|
|||||||
else await self._get_trace_headers(raw_request.headers)
|
else await self._get_trace_headers(raw_request.headers)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Extract data_parallel_rank from header (router can inject it)
|
||||||
|
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||||
|
|
||||||
result_generator = self.engine_client.generate(
|
result_generator = self.engine_client.generate(
|
||||||
engine_input,
|
engine_input,
|
||||||
sampling_params,
|
sampling_params,
|
||||||
@@ -204,6 +207,7 @@ class ServingTokens(OpenAIServing):
|
|||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
trace_headers=trace_headers,
|
trace_headers=trace_headers,
|
||||||
priority=request.priority,
|
priority=request.priority,
|
||||||
|
data_parallel_rank=data_parallel_rank,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result_generator is not None
|
assert result_generator is not None
|
||||||
|
|||||||
Reference in New Issue
Block a user