feat(frontend): session id plumbing into requests (#48048)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from starlette.requests import Request
|
||||
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
def _raw_request(headers: dict[str, str]) -> Request:
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/v1/chat/completions",
|
||||
"headers": [
|
||||
(key.lower().encode("latin-1"), value.encode("latin-1"))
|
||||
for key, value in headers.items()
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"openai_request",
|
||||
[
|
||||
ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
),
|
||||
CompletionRequest(model="test-model", prompt="hi"),
|
||||
ResponsesRequest(model="test-model", input="hi"),
|
||||
],
|
||||
)
|
||||
def test_get_session_id_accepts_body_field(openai_request):
|
||||
openai_request.session_id = "body-session"
|
||||
|
||||
session_id = GenerateBaseServing._get_session_id(
|
||||
openai_request,
|
||||
_raw_request({"X-Session-ID": "header-session"}),
|
||||
)
|
||||
|
||||
assert session_id == "body-session"
|
||||
|
||||
|
||||
def test_get_session_id_accepts_session_header():
|
||||
request = CompletionRequest(model="test-model", prompt="hi")
|
||||
|
||||
session_id = GenerateBaseServing._get_session_id(
|
||||
request,
|
||||
_raw_request({"X-Session-ID": "header-session"}),
|
||||
)
|
||||
|
||||
assert session_id == "header-session"
|
||||
|
||||
|
||||
def test_get_session_id_ignores_correlation_header():
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="hi",
|
||||
vllm_xargs={"session_id": "xargs-session"},
|
||||
)
|
||||
|
||||
session_id = GenerateBaseServing._get_session_id(
|
||||
request,
|
||||
_raw_request({"X-Correlation-ID": "correlation-session"}),
|
||||
)
|
||||
|
||||
assert session_id == "xargs-session"
|
||||
|
||||
|
||||
def test_get_session_id_keeps_vllm_xargs_as_compatibility_fallback():
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="hi",
|
||||
vllm_xargs={"session_id": "xargs-session"},
|
||||
)
|
||||
|
||||
session_id = GenerateBaseServing._get_session_id(request, None)
|
||||
|
||||
assert session_id == "xargs-session"
|
||||
|
||||
|
||||
def test_get_session_id_ignores_empty_and_non_string_values():
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="hi",
|
||||
session_id="",
|
||||
vllm_xargs={"session_id": 7},
|
||||
)
|
||||
|
||||
session_id = GenerateBaseServing._get_session_id(request, None)
|
||||
|
||||
assert session_id is None
|
||||
@@ -203,6 +203,32 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_tokens_threads_session_id_header_to_engine():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=False,
|
||||
)
|
||||
raw_request = MagicMock()
|
||||
raw_request.headers = {"X-Session-ID": "header-session"}
|
||||
|
||||
await serving.serve_tokens(request, raw_request)
|
||||
|
||||
assert engine.generate.call_args.kwargs["session_id"] == "header-session"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_basic():
|
||||
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from copy import copy
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
@@ -68,6 +70,20 @@ def test_parent_request_to_output_final_only() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_sampling_child_requests_preserve_session_id() -> None:
|
||||
request = make_request(SamplingParams(n=2))
|
||||
request.session_id = "session-1"
|
||||
parent_request = ParentRequest(request)
|
||||
|
||||
for idx in range(parent_request.n):
|
||||
request_id, child_params = parent_request.get_child_info(idx)
|
||||
child_request = request if idx == parent_request.n - 1 else copy(request)
|
||||
child_request.request_id = request_id
|
||||
child_request.sampling_params = child_params
|
||||
|
||||
assert child_request.session_id == "session-1"
|
||||
|
||||
|
||||
def make_request(sampling_params: SamplingParams) -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id="parent_id",
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
|
||||
def test_request_status_fmt_str():
|
||||
@@ -18,3 +21,22 @@ def test_request_status_fmt_str():
|
||||
assert f"{RequestStatus.FINISHED_LENGTH_CAPPED}" == "FINISHED_LENGTH_CAPPED"
|
||||
assert f"{RequestStatus.FINISHED_ABORTED}" == "FINISHED_ABORTED"
|
||||
assert f"{RequestStatus.FINISHED_IGNORED}" == "FINISHED_IGNORED"
|
||||
|
||||
|
||||
def test_request_copies_session_id_from_engine_core_request():
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
session_id="session-1",
|
||||
)
|
||||
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.session_id == "session-1"
|
||||
|
||||
Reference in New Issue
Block a user