feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -215,6 +215,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
session_id = self._get_session_id_from_headers(raw_request)
|
||||
|
||||
result_generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
@@ -224,6 +225,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
Reference in New Issue
Block a user