feat(frontend): session id plumbing into requests (#48048)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-08-03 22:18:58 +00:00
committed by GitHub
parent e578de311c
commit f57123aa2d
39 changed files with 438 additions and 4 deletions
@@ -174,6 +174,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
session_id = self._get_session_id(single_request, raw_request)
generators.append(
self.engine_client.generate(
engine_prompt,
@@ -183,6 +184,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
session_id=session_id,
reasoning_ended=None,
)
)
@@ -394,6 +394,14 @@ class ChatCompletionRequest(OpenAIBaseModel):
"through out the inference process and return in response."
),
)
session_id: str | None = Field(
default=None,
description=(
"Stable session identity shared by related requests. Unlike "
"request_id, this value is expected to remain stable across "
"multiple requests in the same conversation or agent session."
),
)
return_tokens_as_token_ids: bool | None = Field(
default=None,
@@ -318,6 +318,7 @@ class OpenAIServingChat(GenerateBaseServing):
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
session_id = self._get_session_id(request, raw_request)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
@@ -326,6 +327,7 @@ class OpenAIServingChat(GenerateBaseServing):
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
session_id=session_id,
)
else:
if not request.include_reasoning:
@@ -348,6 +350,7 @@ class OpenAIServingChat(GenerateBaseServing):
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
session_id=session_id,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs={
"chat_template_kwargs": chat_template_kwargs,
@@ -147,6 +147,14 @@ class CompletionRequest(OpenAIBaseModel):
"through out the inference process and return in response."
),
)
session_id: str | None = Field(
default=None,
description=(
"Stable session identity shared by related requests. Unlike "
"request_id, this value is expected to remain stable across "
"multiple requests in the same conversation or agent session."
),
)
return_tokens_as_token_ids: bool | None = Field(
default=None,
@@ -194,6 +194,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
session_id = self._get_session_id(request, raw_request)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
@@ -202,6 +203,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
session_id=session_id,
)
else:
generator = self.engine_client.generate(
@@ -212,6 +214,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
session_id=session_id,
)
generators.append(generator)
@@ -220,6 +220,14 @@ class ResponsesRequest(OpenAIBaseModel):
"through out the inference process and return in response."
),
)
session_id: str | None = Field(
default=None,
description=(
"Stable session identity shared by related requests. Unlike "
"request_id, this value is expected to remain stable across "
"multiple requests in the same conversation or agent session."
),
)
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
default=None,
description=(
@@ -451,6 +451,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
session_id = self._get_session_id(request, raw_request)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
response_parser = self._make_response_parser(
@@ -516,6 +517,7 @@ class OpenAIServingResponses(GenerateBaseServing):
lora_request=lora_request,
priority=request.priority,
trace_headers=trace_headers,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs
if self.parser and self.parser.reasoning_parser_cls is not None
else None,
@@ -669,6 +671,7 @@ class OpenAIServingResponses(GenerateBaseServing):
lora_request: LoRARequest | None = None,
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
session_id: str | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
):
max_model_len = self.model_config.max_model_len
@@ -693,6 +696,7 @@ class OpenAIServingResponses(GenerateBaseServing):
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)