feat(frontend): session id plumbing into requests (#48048)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-08-03 22:18:58 +00:00
committed by GitHub
parent e578de311c
commit f57123aa2d
39 changed files with 438 additions and 4 deletions
+24
View File
@@ -41,6 +41,7 @@ logger = init_logger(__name__)
RequestT = TypeVar("RequestT", bound=AnyRequest)
_T = TypeVar("_T")
SESSION_ID_HEADER = "X-Session-ID"
def build_per_request_timing_metrics(
@@ -215,6 +216,29 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
except ValueError:
return None
@staticmethod
def _get_session_id_from_headers(raw_request: Request | None) -> str | None:
if raw_request is None:
return None
if value := raw_request.headers.get(SESSION_ID_HEADER):
return value
return None
@staticmethod
def _get_session_id(
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
raw_request: Request | None,
) -> str | None:
if request.session_id:
return request.session_id
if value := GenerateBaseServing._get_session_id_from_headers(raw_request):
return value
if request.vllm_xargs:
session_id = request.vllm_xargs.get("session_id")
if isinstance(session_id, str) and session_id:
return session_id
return None
async def _with_kv_transfer_rejection_cleanup(
self,
awaitable: Awaitable[_T],