From 17c70c895359f7c941199bdf49b723c602ba75de Mon Sep 17 00:00:00 2001 From: Moein Khazraee <33970824+mkhazraee@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:14:07 -0700 Subject: [PATCH] [KV Offload] Update KVCR adapter to pool-and-index API (#59899) --- .buildkite/scripts/install-kv-offload.sh | 2 +- tests/v1/kv_offload/tiering/test_kvcr_tier.py | 32 ++++++++------- vllm/v1/kv_offload/tiering/kvcr/manager.py | 40 +++++++++---------- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/.buildkite/scripts/install-kv-offload.sh b/.buildkite/scripts/install-kv-offload.sh index 46d756b621..eb8bab9e39 100644 --- a/.buildkite/scripts/install-kv-offload.sh +++ b/.buildkite/scripts/install-kv-offload.sh @@ -10,7 +10,7 @@ fi # TODO: Use the PyPI wheel once it is published. uv pip install --system \ - "kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.0" + "kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.1" # Keep only the NIXL wheel matching the CI image's CUDA runtime. NIXL_VERSION=$(uv pip show --system nixl | sed -n 's/^Version: //p') diff --git a/tests/v1/kv_offload/tiering/test_kvcr_tier.py b/tests/v1/kv_offload/tiering/test_kvcr_tier.py index 5cbc85230f..fc4421eb06 100644 --- a/tests/v1/kv_offload/tiering/test_kvcr_tier.py +++ b/tests/v1/kv_offload/tiering/test_kvcr_tier.py @@ -18,7 +18,7 @@ from kvcr.types import ( BlockKey, CacheTier, InventoryEvent, - MemDescriptor, + MemoryRef, OpEntryResult, OpEntryStatus, OpHandle, @@ -78,11 +78,9 @@ class RecordingKVCR: self.submit_hint_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] self.discard_hint_calls: list[str] = [] self.deliver_calls: list[ - tuple[OpHandle, dict[BlockKey, list[MemDescriptor]], str | None] - ] = [] - self.deposit_calls: list[ - tuple[OpHandle, dict[BlockKey, list[MemDescriptor]]] + tuple[OpHandle, dict[BlockKey, list[MemoryRef]], str | None] ] = [] + self.deposit_calls: list[tuple[OpHandle, dict[BlockKey, list[MemoryRef]]]] = [] self.completed: list[tuple[OpHandle, dict[BlockKey, OpEntryResult]]] = [] self._next_op_handle = 1 @@ -107,7 +105,7 @@ class RecordingKVCR: def deliver( self, - blocks: Mapping[BlockKey, list[MemDescriptor]], + blocks: Mapping[BlockKey, list[MemoryRef]], request_id: str | None = None, ) -> OpHandle: op_handle = self._next_op_handle @@ -118,7 +116,7 @@ class RecordingKVCR: def deposit( self, - blocks: Mapping[BlockKey, list[MemDescriptor]], + blocks: Mapping[BlockKey, list[MemoryRef]], ) -> OpHandle: op_handle = self._next_op_handle self._next_op_handle += 1 @@ -248,7 +246,11 @@ def test_kvcr_tier_configures_service_for_local_dp_rank(monkeypatch): compatibility_digest="Opaque-Digest", ) assert kvcr.config is not None - assert kvcr.config.pool_layouts == [("", tier._primary_row_stride)] + assert kvcr.config.pool_layouts == [("", tier.block_size_bytes)] + (region,) = kvcr.backend_configs.framework_regions + assert region.addr == np.asarray(tier._primary_kv_view).ctypes.data + assert (region.size, region.stride or region.size, region.count) == (16, 16, 4) + assert (region.label, region.mem_type, region.device_Id) == ("", "DRAM", 0) assert kvcr.backend_configs.local_dram is None @@ -306,9 +308,7 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch): assert request_id == "req" assert list(blocks) == [key] (descriptor,) = blocks[key] - assert descriptor.end_point_name == kvcr.nixl_agent_name - assert descriptor.addr == tier._primary_base_addr + 2 * 16 - assert descriptor.size == 16 + assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2) assert list(tier.get_finished_jobs()) == [JobResult(7, True)] # Here we verify that request cleanup discards the request-scoped hint in @@ -351,7 +351,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch): tier = _make_tier(monkeypatch, kvcr) keys = (BlockKey(b"k0"), BlockKey(b"k1"), BlockKey(b"k2")) hit_keys = (keys[0], keys[2]) - chunk_ids = {keys[0]: 1, keys[2]: 5} + chunk_ids = {keys[0]: 1, keys[2]: 3} lifecycle: list[str] = [] class Parent: @@ -388,7 +388,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch): block_descriptors = descriptors[keys[2]] assert block_descriptors is not None (descriptor,) = block_descriptors - assert descriptor.addr == (tier._primary_base_addr + 5 * tier._primary_row_stride) + assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=3) assert lifecycle == ["new", "finished"] polls = 0 @@ -482,14 +482,16 @@ def test_kvcr_tier_stores_and_emits_inventory(monkeypatch): local_dram = kvcr.backend_configs.local_dram assert local_dram is not None assert [(name, size) for name, _, size in local_dram.pools] == [ - ("", 2 * tier._primary_row_stride) + ("", 2 * tier.block_size_bytes) ] key = OffloadKey(b"k0") tier.submit_store(_job(11, ReqContext(req_id="req"), key=key, chunk_id=2)) _, blocks = kvcr.deposit_calls[0] - assert blocks[key][0].addr == tier._primary_base_addr + 2 * 16 + assert blocks[key] == [ + MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2) + ] assert list(tier.get_finished_jobs()) == [JobResult(11, True)] assert kvcr.inventory_sink is not None diff --git a/vllm/v1/kv_offload/tiering/kvcr/manager.py b/vllm/v1/kv_offload/tiering/kvcr/manager.py index 18da2a6c27..4b6f81b3b7 100644 --- a/vllm/v1/kv_offload/tiering/kvcr/manager.py +++ b/vllm/v1/kv_offload/tiering/kvcr/manager.py @@ -22,7 +22,6 @@ from kvcr import ( KVCRBindings, ) from kvcr.config import ( - FrameworkDramInput, G3Options, KeyAdapter, KVCRBackendConfigs, @@ -43,11 +42,12 @@ from kvcr.types import ( BlockKey, CacheTier, InventoryEvent, - MemDescriptor, + MemoryRef, OpHandle, PinRequestId, PinResult, QueryStatus, + RegionDescriptor, ) from typing_extensions import override @@ -177,7 +177,7 @@ class _FrameworkPinAdapter: def _resolve_pin( self, parent: ParentManager, keys: Collection[BlockKey] - ) -> tuple[str, dict[BlockKey, list[MemDescriptor] | None]] | None: + ) -> tuple[str, dict[BlockKey, list[MemoryRef] | None]] | None: offload_keys = tuple(OffloadKey(bytes(key)) for key in keys) request_id = f"kvcr-source:{self._next_request_id}" self._next_request_id += 1 @@ -202,7 +202,7 @@ class _FrameworkPinAdapter: if len(job_keys) != len(chunk_ids) or set(job_keys) != set(hit_keys): return None - descriptors: dict[BlockKey, list[MemDescriptor] | None] = { + descriptors: dict[BlockKey, list[MemoryRef] | None] = { BlockKey(bytes(key)): None for key in offload_keys } descriptors.update( @@ -350,12 +350,6 @@ class KVCRSecondaryTierManager(SecondaryTierManager): with socket.socket() as _s: _s.bind(("", 0)) _nixl_listen_port = _s.getsockname()[1] - self._primary_base_addr = ctypes.addressof( - ctypes.c_char.from_buffer(primary_kv_view) - ) - if primary_kv_view.strides is None: - raise ValueError("primary KV memoryview must expose strides") - self._primary_row_stride = int(primary_kv_view.strides[0]) if secondary_g2_slots < 0: raise ValueError("secondary_g2_slots must be non-negative") if ( @@ -375,7 +369,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager): ) else: local_mapping = mmap.mmap( - -1, secondary_g2_slots * self._primary_row_stride + -1, secondary_g2_slots * self.block_size_bytes ) local_dram = LocalDramOptions( pools=[ @@ -408,7 +402,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager): try: self._kvcr = KVCR( KVCRConfig( - pool_layouts=[("", self._primary_row_stride)], + pool_layouts=[("", self.block_size_bytes)], enable_telemetry=enable_telemetry, operation_timeout_ms=operation_timeout_ms, abandon_timeout_ms=abandon_timeout_ms, @@ -429,9 +423,15 @@ class KVCRSecondaryTierManager(SecondaryTierManager): policy=selected_policy, ), KVCRBackendConfigs( - framework_dram=FrameworkDramInput( - self._primary_base_addr, primary_kv_view.nbytes - ), + framework_regions=[ + RegionDescriptor( + addr=ctypes.addressof( + ctypes.c_char.from_buffer(primary_kv_view) + ), + size=self.block_size_bytes, + count=len(primary_kv_view), + ) + ], local_dram=local_dram, g3=g3_config, remote_fw_dram=RemoteFWDramOptions( @@ -621,12 +621,8 @@ class KVCRSecondaryTierManager(SecondaryTierManager): ) ) - def _make_descriptor(self, chunk_id: int) -> MemDescriptor: - return MemDescriptor( + def _make_descriptor(self, chunk_id: int) -> MemoryRef: + return MemoryRef( end_point_name=self._kvcr.config.nixl_agent_name, - mem_type="DRAM", - addr=self._primary_base_addr + chunk_id * self._primary_row_stride, - size=self._primary_row_stride, - device_Id=0, - info="", + element_index=chunk_id, )