[KV Offload] Update KVCR adapter to pool-and-index API (#59899)

This commit is contained in:
Moein Khazraee
2026-10-07 21:14:07 +02:00
committed by GitHub
parent b9767299db
commit 17c70c8953
3 changed files with 36 additions and 38 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ fi
# TODO: Use the PyPI wheel once it is published.
uv pip install --system \
"kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.0"
"kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.1"
# Keep only the NIXL wheel matching the CI image's CUDA runtime.
NIXL_VERSION=$(uv pip show --system nixl | sed -n 's/^Version: //p')
+17 -15
View File
@@ -18,7 +18,7 @@ from kvcr.types import (
BlockKey,
CacheTier,
InventoryEvent,
MemDescriptor,
MemoryRef,
OpEntryResult,
OpEntryStatus,
OpHandle,
@@ -78,11 +78,9 @@ class RecordingKVCR:
self.submit_hint_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
self.discard_hint_calls: list[str] = []
self.deliver_calls: list[
tuple[OpHandle, dict[BlockKey, list[MemDescriptor]], str | None]
] = []
self.deposit_calls: list[
tuple[OpHandle, dict[BlockKey, list[MemDescriptor]]]
tuple[OpHandle, dict[BlockKey, list[MemoryRef]], str | None]
] = []
self.deposit_calls: list[tuple[OpHandle, dict[BlockKey, list[MemoryRef]]]] = []
self.completed: list[tuple[OpHandle, dict[BlockKey, OpEntryResult]]] = []
self._next_op_handle = 1
@@ -107,7 +105,7 @@ class RecordingKVCR:
def deliver(
self,
blocks: Mapping[BlockKey, list[MemDescriptor]],
blocks: Mapping[BlockKey, list[MemoryRef]],
request_id: str | None = None,
) -> OpHandle:
op_handle = self._next_op_handle
@@ -118,7 +116,7 @@ class RecordingKVCR:
def deposit(
self,
blocks: Mapping[BlockKey, list[MemDescriptor]],
blocks: Mapping[BlockKey, list[MemoryRef]],
) -> OpHandle:
op_handle = self._next_op_handle
self._next_op_handle += 1
@@ -248,7 +246,11 @@ def test_kvcr_tier_configures_service_for_local_dp_rank(monkeypatch):
compatibility_digest="Opaque-Digest",
)
assert kvcr.config is not None
assert kvcr.config.pool_layouts == [("", tier._primary_row_stride)]
assert kvcr.config.pool_layouts == [("", tier.block_size_bytes)]
(region,) = kvcr.backend_configs.framework_regions
assert region.addr == np.asarray(tier._primary_kv_view).ctypes.data
assert (region.size, region.stride or region.size, region.count) == (16, 16, 4)
assert (region.label, region.mem_type, region.device_Id) == ("", "DRAM", 0)
assert kvcr.backend_configs.local_dram is None
@@ -306,9 +308,7 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
assert request_id == "req"
assert list(blocks) == [key]
(descriptor,) = blocks[key]
assert descriptor.end_point_name == kvcr.nixl_agent_name
assert descriptor.addr == tier._primary_base_addr + 2 * 16
assert descriptor.size == 16
assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2)
assert list(tier.get_finished_jobs()) == [JobResult(7, True)]
# Here we verify that request cleanup discards the request-scoped hint in
@@ -351,7 +351,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch):
tier = _make_tier(monkeypatch, kvcr)
keys = (BlockKey(b"k0"), BlockKey(b"k1"), BlockKey(b"k2"))
hit_keys = (keys[0], keys[2])
chunk_ids = {keys[0]: 1, keys[2]: 5}
chunk_ids = {keys[0]: 1, keys[2]: 3}
lifecycle: list[str] = []
class Parent:
@@ -388,7 +388,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch):
block_descriptors = descriptors[keys[2]]
assert block_descriptors is not None
(descriptor,) = block_descriptors
assert descriptor.addr == (tier._primary_base_addr + 5 * tier._primary_row_stride)
assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=3)
assert lifecycle == ["new", "finished"]
polls = 0
@@ -482,14 +482,16 @@ def test_kvcr_tier_stores_and_emits_inventory(monkeypatch):
local_dram = kvcr.backend_configs.local_dram
assert local_dram is not None
assert [(name, size) for name, _, size in local_dram.pools] == [
("", 2 * tier._primary_row_stride)
("", 2 * tier.block_size_bytes)
]
key = OffloadKey(b"k0")
tier.submit_store(_job(11, ReqContext(req_id="req"), key=key, chunk_id=2))
_, blocks = kvcr.deposit_calls[0]
assert blocks[key][0].addr == tier._primary_base_addr + 2 * 16
assert blocks[key] == [
MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2)
]
assert list(tier.get_finished_jobs()) == [JobResult(11, True)]
assert kvcr.inventory_sink is not None
+18 -22
View File
@@ -22,7 +22,6 @@ from kvcr import (
KVCRBindings,
)
from kvcr.config import (
FrameworkDramInput,
G3Options,
KeyAdapter,
KVCRBackendConfigs,
@@ -43,11 +42,12 @@ from kvcr.types import (
BlockKey,
CacheTier,
InventoryEvent,
MemDescriptor,
MemoryRef,
OpHandle,
PinRequestId,
PinResult,
QueryStatus,
RegionDescriptor,
)
from typing_extensions import override
@@ -177,7 +177,7 @@ class _FrameworkPinAdapter:
def _resolve_pin(
self, parent: ParentManager, keys: Collection[BlockKey]
) -> tuple[str, dict[BlockKey, list[MemDescriptor] | None]] | None:
) -> tuple[str, dict[BlockKey, list[MemoryRef] | None]] | None:
offload_keys = tuple(OffloadKey(bytes(key)) for key in keys)
request_id = f"kvcr-source:{self._next_request_id}"
self._next_request_id += 1
@@ -202,7 +202,7 @@ class _FrameworkPinAdapter:
if len(job_keys) != len(chunk_ids) or set(job_keys) != set(hit_keys):
return None
descriptors: dict[BlockKey, list[MemDescriptor] | None] = {
descriptors: dict[BlockKey, list[MemoryRef] | None] = {
BlockKey(bytes(key)): None for key in offload_keys
}
descriptors.update(
@@ -350,12 +350,6 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
with socket.socket() as _s:
_s.bind(("", 0))
_nixl_listen_port = _s.getsockname()[1]
self._primary_base_addr = ctypes.addressof(
ctypes.c_char.from_buffer(primary_kv_view)
)
if primary_kv_view.strides is None:
raise ValueError("primary KV memoryview must expose strides")
self._primary_row_stride = int(primary_kv_view.strides[0])
if secondary_g2_slots < 0:
raise ValueError("secondary_g2_slots must be non-negative")
if (
@@ -375,7 +369,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
)
else:
local_mapping = mmap.mmap(
-1, secondary_g2_slots * self._primary_row_stride
-1, secondary_g2_slots * self.block_size_bytes
)
local_dram = LocalDramOptions(
pools=[
@@ -408,7 +402,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
try:
self._kvcr = KVCR(
KVCRConfig(
pool_layouts=[("", self._primary_row_stride)],
pool_layouts=[("", self.block_size_bytes)],
enable_telemetry=enable_telemetry,
operation_timeout_ms=operation_timeout_ms,
abandon_timeout_ms=abandon_timeout_ms,
@@ -429,9 +423,15 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
policy=selected_policy,
),
KVCRBackendConfigs(
framework_dram=FrameworkDramInput(
self._primary_base_addr, primary_kv_view.nbytes
),
framework_regions=[
RegionDescriptor(
addr=ctypes.addressof(
ctypes.c_char.from_buffer(primary_kv_view)
),
size=self.block_size_bytes,
count=len(primary_kv_view),
)
],
local_dram=local_dram,
g3=g3_config,
remote_fw_dram=RemoteFWDramOptions(
@@ -621,12 +621,8 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
)
)
def _make_descriptor(self, chunk_id: int) -> MemDescriptor:
return MemDescriptor(
def _make_descriptor(self, chunk_id: int) -> MemoryRef:
return MemoryRef(
end_point_name=self._kvcr.config.nixl_agent_name,
mem_type="DRAM",
addr=self._primary_base_addr + chunk_id * self._primary_row_stride,
size=self._primary_row_stride,
device_Id=0,
info="",
element_index=chunk_id,
)