[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.kv_hints import KvHintAction, KvHintsEnvelope
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
|
||||
@@ -24,6 +25,7 @@ def test_request_status_fmt_str():
|
||||
|
||||
|
||||
def test_request_copies_session_id_from_engine_core_request():
|
||||
"""Test that Request preserves session ID from EngineCoreRequest."""
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
@@ -40,3 +42,35 @@ def test_request_copies_session_id_from_engine_core_request():
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.session_id == "session-1"
|
||||
|
||||
|
||||
def test_request_copies_kv_hints_from_engine_core_request():
|
||||
"""Test that Request preserves KV hints from EngineCoreRequest."""
|
||||
kv_hints = KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="msg-1",
|
||||
actions=[
|
||||
KvHintAction(
|
||||
action_id="action-1",
|
||||
action_type="example.action",
|
||||
action_version="1.0",
|
||||
payload={},
|
||||
)
|
||||
],
|
||||
)
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.kv_hints == kv_hints
|
||||
|
||||
Reference in New Issue
Block a user