[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-09-21 09:56:11 -07:00
committed by GitHub
parent 4f14516790
commit 986e217158
30 changed files with 340 additions and 18 deletions
@@ -33,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
OffloadingConnectorScheduler,
RequestOffloadState,
_create_req_context,
get_sliding_window_size_in_chunks,
)
from vllm.v1.core.block_pool import BlockPool
@@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.kv_hints import KvHintsEnvelope
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
LookupResult,
@@ -214,6 +216,7 @@ def _make_partial_tail_request(
request = MagicMock()
request.request_id = "req"
request.kv_transfer_params = kv_transfer_params
request.kv_hints = None
request.num_prompt_tokens = 30
request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
@@ -225,6 +228,21 @@ def _make_partial_tail_request(
return request
def test_create_req_context_preserves_kv_hints():
kv_hints = KvHintsEnvelope(
protocol_version="0.1",
message_id="message-1",
actions=[],
)
request = MagicMock(
request_id="req",
kv_transfer_params=None,
kv_hints=kv_hints,
)
assert _create_req_context(request).kv_hints is kv_hints
def _reduce_kv_connector_stats(runner):
reduced: dict[str, int | float] = {}
for payload in runner.kv_connector_stats:
@@ -5,12 +5,13 @@ from collections.abc import Collection, Iterable, Mapping
from types import SimpleNamespace
from typing import Any
import msgspec
import numpy as np
import pytest
pytest.importorskip("kvcr")
from kvcr import ROUTER_HINT_KEY, KVCRBindings
from kvcr import KVCRBindings
from kvcr.config import G3Options, KVCRBackendConfigs, KVCRConfig, KVCRGuardConfig
from kvcr.policy import FIFOPolicy, G3FIFOPolicy, G3LRUPolicy, LRUPolicy
from kvcr.types import (
@@ -27,6 +28,7 @@ from kvcr.types import (
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_hints import KvHintsEnvelope
from vllm.v1.kv_offload.base import (
LookupResult,
Medium,
@@ -271,10 +273,10 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
"""Check hint forwarding, key conversion, load descriptors, and cleanup."""
kvcr = RecordingKVCR()
tier = _make_tier(monkeypatch, kvcr)
router_hint = {"opaque": True}
kv_hint = KvHintsEnvelope(protocol_version="0.1", message_id="msg", actions=[])
ctx = ReqContext(
req_id="req",
kv_transfer_params={ROUTER_HINT_KEY: router_hint, "unrelated": object()},
kv_hints=kv_hint,
)
key = make_offload_key((123).to_bytes(8, "big"), 0)
same_hash_other_group = make_offload_key((123).to_bytes(8, "big"), 7)
@@ -282,7 +284,9 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
tier.on_new_request(ctx)
assert kvcr.submit_hint_calls == [((), {"request_id": "req", "hints": router_hint})]
assert kvcr.submit_hint_calls == [
((), {"request_id": "req", "hints": msgspec.to_builtins(kv_hint)})
]
bindings = kvcr.constructor_bindings
assert bindings is not None
+34
View File
@@ -3,6 +3,7 @@
from vllm import SamplingParams
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.kv_hints import KvHintAction, KvHintsEnvelope
from vllm.v1.request import Request, RequestStatus
@@ -24,6 +25,7 @@ def test_request_status_fmt_str():
def test_request_copies_session_id_from_engine_core_request():
"""Test that Request preserves session ID from EngineCoreRequest."""
engine_request = EngineCoreRequest(
request_id="request-1",
prompt_token_ids=[1, 2, 3],
@@ -40,3 +42,35 @@ def test_request_copies_session_id_from_engine_core_request():
request = Request.from_engine_core_request(engine_request, block_hasher=None)
assert request.session_id == "session-1"
def test_request_copies_kv_hints_from_engine_core_request():
"""Test that Request preserves KV hints from EngineCoreRequest."""
kv_hints = KvHintsEnvelope(
protocol_version="0.1",
message_id="msg-1",
actions=[
KvHintAction(
action_id="action-1",
action_type="example.action",
action_version="1.0",
payload={},
)
],
)
engine_request = EngineCoreRequest(
request_id="request-1",
prompt_token_ids=[1, 2, 3],
mm_features=None,
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
kv_hints=kv_hints,
)
request = Request.from_engine_core_request(engine_request, block_hasher=None)
assert request.kv_hints == kv_hints