feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -174,6 +174,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(single_request, raw_request)
|
||||
generators.append(
|
||||
self.engine_client.generate(
|
||||
engine_prompt,
|
||||
@@ -183,6 +184,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=None,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -394,6 +394,14 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
session_id: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Stable session identity shared by related requests. Unlike "
|
||||
"request_id, this value is expected to remain stable across "
|
||||
"multiple requests in the same conversation or agent session."
|
||||
),
|
||||
)
|
||||
|
||||
return_tokens_as_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -318,6 +318,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(request, raw_request)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
@@ -326,6 +327,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
else:
|
||||
if not request.include_reasoning:
|
||||
@@ -348,6 +350,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs={
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
|
||||
Reference in New Issue
Block a user