feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -41,6 +41,7 @@ logger = init_logger(__name__)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
_T = TypeVar("_T")
|
||||
SESSION_ID_HEADER = "X-Session-ID"
|
||||
|
||||
|
||||
def build_per_request_timing_metrics(
|
||||
@@ -215,6 +216,29 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id_from_headers(raw_request: Request | None) -> str | None:
|
||||
if raw_request is None:
|
||||
return None
|
||||
if value := raw_request.headers.get(SESSION_ID_HEADER):
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id(
|
||||
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
|
||||
raw_request: Request | None,
|
||||
) -> str | None:
|
||||
if request.session_id:
|
||||
return request.session_id
|
||||
if value := GenerateBaseServing._get_session_id_from_headers(raw_request):
|
||||
return value
|
||||
if request.vllm_xargs:
|
||||
session_id = request.vllm_xargs.get("session_id")
|
||||
if isinstance(session_id, str) and session_id:
|
||||
return session_id
|
||||
return None
|
||||
|
||||
async def _with_kv_transfer_rejection_cleanup(
|
||||
self,
|
||||
awaitable: Awaitable[_T],
|
||||
|
||||
@@ -32,6 +32,7 @@ class BeamSearchOnlineMixin(ABC):
|
||||
params: BeamSearchParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
beam_width = params.beam_width
|
||||
max_tokens = params.max_tokens
|
||||
@@ -90,6 +91,7 @@ class BeamSearchOnlineMixin(ABC):
|
||||
request_id_item,
|
||||
lora_request=lora_request_item,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -174,6 +174,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(single_request, raw_request)
|
||||
generators.append(
|
||||
self.engine_client.generate(
|
||||
engine_prompt,
|
||||
@@ -183,6 +184,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=None,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -394,6 +394,14 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
session_id: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Stable session identity shared by related requests. Unlike "
|
||||
"request_id, this value is expected to remain stable across "
|
||||
"multiple requests in the same conversation or agent session."
|
||||
),
|
||||
)
|
||||
|
||||
return_tokens_as_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -318,6 +318,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(request, raw_request)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
@@ -326,6 +327,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
else:
|
||||
if not request.include_reasoning:
|
||||
@@ -348,6 +350,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs={
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
|
||||
@@ -147,6 +147,14 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
session_id: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Stable session identity shared by related requests. Unlike "
|
||||
"request_id, this value is expected to remain stable across "
|
||||
"multiple requests in the same conversation or agent session."
|
||||
),
|
||||
)
|
||||
|
||||
return_tokens_as_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -194,6 +194,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(request, raw_request)
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
@@ -202,6 +203,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
@@ -212,6 +214,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
@@ -220,6 +220,14 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
session_id: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Stable session identity shared by related requests. Unlike "
|
||||
"request_id, this value is expected to remain stable across "
|
||||
"multiple requests in the same conversation or agent session."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -451,6 +451,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
session_id = self._get_session_id(request, raw_request)
|
||||
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
response_parser = self._make_response_parser(
|
||||
@@ -516,6 +517,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None
|
||||
else None,
|
||||
@@ -669,6 +671,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
@@ -693,6 +696,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -215,6 +215,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
|
||||
# Extract data_parallel_rank from header (router can inject it)
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
session_id = self._get_session_id_from_headers(raw_request)
|
||||
|
||||
result_generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
@@ -224,6 +225,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
Reference in New Issue
Block a user