[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -33,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
_create_req_context,
|
||||
get_sliding_window_size_in_chunks,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
@@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
LookupResult,
|
||||
@@ -214,6 +216,7 @@ def _make_partial_tail_request(
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = kv_transfer_params
|
||||
request.kv_hints = None
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
@@ -225,6 +228,21 @@ def _make_partial_tail_request(
|
||||
return request
|
||||
|
||||
|
||||
def test_create_req_context_preserves_kv_hints():
|
||||
kv_hints = KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="message-1",
|
||||
actions=[],
|
||||
)
|
||||
request = MagicMock(
|
||||
request_id="req",
|
||||
kv_transfer_params=None,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
assert _create_req_context(request).kv_hints is kv_hints
|
||||
|
||||
|
||||
def _reduce_kv_connector_stats(runner):
|
||||
reduced: dict[str, int | float] = {}
|
||||
for payload in runner.kv_connector_stats:
|
||||
|
||||
@@ -5,12 +5,13 @@ from collections.abc import Collection, Iterable, Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("kvcr")
|
||||
|
||||
from kvcr import ROUTER_HINT_KEY, KVCRBindings
|
||||
from kvcr import KVCRBindings
|
||||
from kvcr.config import G3Options, KVCRBackendConfigs, KVCRConfig, KVCRGuardConfig
|
||||
from kvcr.policy import FIFOPolicy, G3FIFOPolicy, G3LRUPolicy, LRUPolicy
|
||||
from kvcr.types import (
|
||||
@@ -27,6 +28,7 @@ from kvcr.types import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_hints import KvHintsEnvelope
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
Medium,
|
||||
@@ -271,10 +273,10 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
|
||||
"""Check hint forwarding, key conversion, load descriptors, and cleanup."""
|
||||
kvcr = RecordingKVCR()
|
||||
tier = _make_tier(monkeypatch, kvcr)
|
||||
router_hint = {"opaque": True}
|
||||
kv_hint = KvHintsEnvelope(protocol_version="0.1", message_id="msg", actions=[])
|
||||
ctx = ReqContext(
|
||||
req_id="req",
|
||||
kv_transfer_params={ROUTER_HINT_KEY: router_hint, "unrelated": object()},
|
||||
kv_hints=kv_hint,
|
||||
)
|
||||
key = make_offload_key((123).to_bytes(8, "big"), 0)
|
||||
same_hash_other_group = make_offload_key((123).to_bytes(8, "big"), 7)
|
||||
@@ -282,7 +284,9 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
|
||||
|
||||
tier.on_new_request(ctx)
|
||||
|
||||
assert kvcr.submit_hint_calls == [((), {"request_id": "req", "hints": router_hint})]
|
||||
assert kvcr.submit_hint_calls == [
|
||||
((), {"request_id": "req", "hints": msgspec.to_builtins(kv_hint)})
|
||||
]
|
||||
|
||||
bindings = kvcr.constructor_bindings
|
||||
assert bindings is not None
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.kv_hints import KvHintAction, KvHintsEnvelope
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
|
||||
@@ -24,6 +25,7 @@ def test_request_status_fmt_str():
|
||||
|
||||
|
||||
def test_request_copies_session_id_from_engine_core_request():
|
||||
"""Test that Request preserves session ID from EngineCoreRequest."""
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
@@ -40,3 +42,35 @@ def test_request_copies_session_id_from_engine_core_request():
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.session_id == "session-1"
|
||||
|
||||
|
||||
def test_request_copies_kv_hints_from_engine_core_request():
|
||||
"""Test that Request preserves KV hints from EngineCoreRequest."""
|
||||
kv_hints = KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="msg-1",
|
||||
actions=[
|
||||
KvHintAction(
|
||||
action_id="action-1",
|
||||
action_type="example.action",
|
||||
action_version="1.0",
|
||||
payload={},
|
||||
)
|
||||
],
|
||||
)
|
||||
engine_request = EngineCoreRequest(
|
||||
request_id="request-1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
kv_hints=kv_hints,
|
||||
)
|
||||
|
||||
request = Request.from_engine_core_request(engine_request, block_hasher=None)
|
||||
|
||||
assert request.kv_hints == kv_hints
|
||||
|
||||
Reference in New Issue
Block a user