feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -41,6 +41,7 @@ logger = init_logger(__name__)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
_T = TypeVar("_T")
|
||||
SESSION_ID_HEADER = "X-Session-ID"
|
||||
|
||||
|
||||
def build_per_request_timing_metrics(
|
||||
@@ -215,6 +216,29 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id_from_headers(raw_request: Request | None) -> str | None:
|
||||
if raw_request is None:
|
||||
return None
|
||||
if value := raw_request.headers.get(SESSION_ID_HEADER):
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id(
|
||||
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
|
||||
raw_request: Request | None,
|
||||
) -> str | None:
|
||||
if request.session_id:
|
||||
return request.session_id
|
||||
if value := GenerateBaseServing._get_session_id_from_headers(raw_request):
|
||||
return value
|
||||
if request.vllm_xargs:
|
||||
session_id = request.vllm_xargs.get("session_id")
|
||||
if isinstance(session_id, str) and session_id:
|
||||
return session_id
|
||||
return None
|
||||
|
||||
async def _with_kv_transfer_rejection_cleanup(
|
||||
self,
|
||||
awaitable: Awaitable[_T],
|
||||
|
||||
Reference in New Issue
Block a user