[KV Offload] Update KVCR adapter to pool-and-index API (#59899)
This commit is contained in:
@@ -10,7 +10,7 @@ fi
|
||||
|
||||
# TODO: Use the PyPI wheel once it is published.
|
||||
uv pip install --system \
|
||||
"kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.0"
|
||||
"kvcr @ git+https://github.com/ai-dynamo/kvcr.git@v0.1.1"
|
||||
|
||||
# Keep only the NIXL wheel matching the CI image's CUDA runtime.
|
||||
NIXL_VERSION=$(uv pip show --system nixl | sed -n 's/^Version: //p')
|
||||
|
||||
@@ -18,7 +18,7 @@ from kvcr.types import (
|
||||
BlockKey,
|
||||
CacheTier,
|
||||
InventoryEvent,
|
||||
MemDescriptor,
|
||||
MemoryRef,
|
||||
OpEntryResult,
|
||||
OpEntryStatus,
|
||||
OpHandle,
|
||||
@@ -78,11 +78,9 @@ class RecordingKVCR:
|
||||
self.submit_hint_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
|
||||
self.discard_hint_calls: list[str] = []
|
||||
self.deliver_calls: list[
|
||||
tuple[OpHandle, dict[BlockKey, list[MemDescriptor]], str | None]
|
||||
] = []
|
||||
self.deposit_calls: list[
|
||||
tuple[OpHandle, dict[BlockKey, list[MemDescriptor]]]
|
||||
tuple[OpHandle, dict[BlockKey, list[MemoryRef]], str | None]
|
||||
] = []
|
||||
self.deposit_calls: list[tuple[OpHandle, dict[BlockKey, list[MemoryRef]]]] = []
|
||||
self.completed: list[tuple[OpHandle, dict[BlockKey, OpEntryResult]]] = []
|
||||
self._next_op_handle = 1
|
||||
|
||||
@@ -107,7 +105,7 @@ class RecordingKVCR:
|
||||
|
||||
def deliver(
|
||||
self,
|
||||
blocks: Mapping[BlockKey, list[MemDescriptor]],
|
||||
blocks: Mapping[BlockKey, list[MemoryRef]],
|
||||
request_id: str | None = None,
|
||||
) -> OpHandle:
|
||||
op_handle = self._next_op_handle
|
||||
@@ -118,7 +116,7 @@ class RecordingKVCR:
|
||||
|
||||
def deposit(
|
||||
self,
|
||||
blocks: Mapping[BlockKey, list[MemDescriptor]],
|
||||
blocks: Mapping[BlockKey, list[MemoryRef]],
|
||||
) -> OpHandle:
|
||||
op_handle = self._next_op_handle
|
||||
self._next_op_handle += 1
|
||||
@@ -248,7 +246,11 @@ def test_kvcr_tier_configures_service_for_local_dp_rank(monkeypatch):
|
||||
compatibility_digest="Opaque-Digest",
|
||||
)
|
||||
assert kvcr.config is not None
|
||||
assert kvcr.config.pool_layouts == [("", tier._primary_row_stride)]
|
||||
assert kvcr.config.pool_layouts == [("", tier.block_size_bytes)]
|
||||
(region,) = kvcr.backend_configs.framework_regions
|
||||
assert region.addr == np.asarray(tier._primary_kv_view).ctypes.data
|
||||
assert (region.size, region.stride or region.size, region.count) == (16, 16, 4)
|
||||
assert (region.label, region.mem_type, region.device_Id) == ("", "DRAM", 0)
|
||||
assert kvcr.backend_configs.local_dram is None
|
||||
|
||||
|
||||
@@ -306,9 +308,7 @@ def test_kvcr_tier_adapts_request_and_load(monkeypatch):
|
||||
assert request_id == "req"
|
||||
assert list(blocks) == [key]
|
||||
(descriptor,) = blocks[key]
|
||||
assert descriptor.end_point_name == kvcr.nixl_agent_name
|
||||
assert descriptor.addr == tier._primary_base_addr + 2 * 16
|
||||
assert descriptor.size == 16
|
||||
assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2)
|
||||
assert list(tier.get_finished_jobs()) == [JobResult(7, True)]
|
||||
|
||||
# Here we verify that request cleanup discards the request-scoped hint in
|
||||
@@ -351,7 +351,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch):
|
||||
tier = _make_tier(monkeypatch, kvcr)
|
||||
keys = (BlockKey(b"k0"), BlockKey(b"k1"), BlockKey(b"k2"))
|
||||
hit_keys = (keys[0], keys[2])
|
||||
chunk_ids = {keys[0]: 1, keys[2]: 5}
|
||||
chunk_ids = {keys[0]: 1, keys[2]: 3}
|
||||
lifecycle: list[str] = []
|
||||
|
||||
class Parent:
|
||||
@@ -388,7 +388,7 @@ def test_kvcr_tier_serves_primary_pin_request(monkeypatch):
|
||||
block_descriptors = descriptors[keys[2]]
|
||||
assert block_descriptors is not None
|
||||
(descriptor,) = block_descriptors
|
||||
assert descriptor.addr == (tier._primary_base_addr + 5 * tier._primary_row_stride)
|
||||
assert descriptor == MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=3)
|
||||
assert lifecycle == ["new", "finished"]
|
||||
|
||||
polls = 0
|
||||
@@ -482,14 +482,16 @@ def test_kvcr_tier_stores_and_emits_inventory(monkeypatch):
|
||||
local_dram = kvcr.backend_configs.local_dram
|
||||
assert local_dram is not None
|
||||
assert [(name, size) for name, _, size in local_dram.pools] == [
|
||||
("", 2 * tier._primary_row_stride)
|
||||
("", 2 * tier.block_size_bytes)
|
||||
]
|
||||
|
||||
key = OffloadKey(b"k0")
|
||||
tier.submit_store(_job(11, ReqContext(req_id="req"), key=key, chunk_id=2))
|
||||
|
||||
_, blocks = kvcr.deposit_calls[0]
|
||||
assert blocks[key][0].addr == tier._primary_base_addr + 2 * 16
|
||||
assert blocks[key] == [
|
||||
MemoryRef(end_point_name=kvcr.nixl_agent_name, element_index=2)
|
||||
]
|
||||
assert list(tier.get_finished_jobs()) == [JobResult(11, True)]
|
||||
|
||||
assert kvcr.inventory_sink is not None
|
||||
|
||||
@@ -22,7 +22,6 @@ from kvcr import (
|
||||
KVCRBindings,
|
||||
)
|
||||
from kvcr.config import (
|
||||
FrameworkDramInput,
|
||||
G3Options,
|
||||
KeyAdapter,
|
||||
KVCRBackendConfigs,
|
||||
@@ -43,11 +42,12 @@ from kvcr.types import (
|
||||
BlockKey,
|
||||
CacheTier,
|
||||
InventoryEvent,
|
||||
MemDescriptor,
|
||||
MemoryRef,
|
||||
OpHandle,
|
||||
PinRequestId,
|
||||
PinResult,
|
||||
QueryStatus,
|
||||
RegionDescriptor,
|
||||
)
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -177,7 +177,7 @@ class _FrameworkPinAdapter:
|
||||
|
||||
def _resolve_pin(
|
||||
self, parent: ParentManager, keys: Collection[BlockKey]
|
||||
) -> tuple[str, dict[BlockKey, list[MemDescriptor] | None]] | None:
|
||||
) -> tuple[str, dict[BlockKey, list[MemoryRef] | None]] | None:
|
||||
offload_keys = tuple(OffloadKey(bytes(key)) for key in keys)
|
||||
request_id = f"kvcr-source:{self._next_request_id}"
|
||||
self._next_request_id += 1
|
||||
@@ -202,7 +202,7 @@ class _FrameworkPinAdapter:
|
||||
if len(job_keys) != len(chunk_ids) or set(job_keys) != set(hit_keys):
|
||||
return None
|
||||
|
||||
descriptors: dict[BlockKey, list[MemDescriptor] | None] = {
|
||||
descriptors: dict[BlockKey, list[MemoryRef] | None] = {
|
||||
BlockKey(bytes(key)): None for key in offload_keys
|
||||
}
|
||||
descriptors.update(
|
||||
@@ -350,12 +350,6 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
with socket.socket() as _s:
|
||||
_s.bind(("", 0))
|
||||
_nixl_listen_port = _s.getsockname()[1]
|
||||
self._primary_base_addr = ctypes.addressof(
|
||||
ctypes.c_char.from_buffer(primary_kv_view)
|
||||
)
|
||||
if primary_kv_view.strides is None:
|
||||
raise ValueError("primary KV memoryview must expose strides")
|
||||
self._primary_row_stride = int(primary_kv_view.strides[0])
|
||||
if secondary_g2_slots < 0:
|
||||
raise ValueError("secondary_g2_slots must be non-negative")
|
||||
if (
|
||||
@@ -375,7 +369,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
)
|
||||
else:
|
||||
local_mapping = mmap.mmap(
|
||||
-1, secondary_g2_slots * self._primary_row_stride
|
||||
-1, secondary_g2_slots * self.block_size_bytes
|
||||
)
|
||||
local_dram = LocalDramOptions(
|
||||
pools=[
|
||||
@@ -408,7 +402,7 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
try:
|
||||
self._kvcr = KVCR(
|
||||
KVCRConfig(
|
||||
pool_layouts=[("", self._primary_row_stride)],
|
||||
pool_layouts=[("", self.block_size_bytes)],
|
||||
enable_telemetry=enable_telemetry,
|
||||
operation_timeout_ms=operation_timeout_ms,
|
||||
abandon_timeout_ms=abandon_timeout_ms,
|
||||
@@ -429,9 +423,15 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
policy=selected_policy,
|
||||
),
|
||||
KVCRBackendConfigs(
|
||||
framework_dram=FrameworkDramInput(
|
||||
self._primary_base_addr, primary_kv_view.nbytes
|
||||
),
|
||||
framework_regions=[
|
||||
RegionDescriptor(
|
||||
addr=ctypes.addressof(
|
||||
ctypes.c_char.from_buffer(primary_kv_view)
|
||||
),
|
||||
size=self.block_size_bytes,
|
||||
count=len(primary_kv_view),
|
||||
)
|
||||
],
|
||||
local_dram=local_dram,
|
||||
g3=g3_config,
|
||||
remote_fw_dram=RemoteFWDramOptions(
|
||||
@@ -621,12 +621,8 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
)
|
||||
)
|
||||
|
||||
def _make_descriptor(self, chunk_id: int) -> MemDescriptor:
|
||||
return MemDescriptor(
|
||||
def _make_descriptor(self, chunk_id: int) -> MemoryRef:
|
||||
return MemoryRef(
|
||||
end_point_name=self._kvcr.config.nixl_agent_name,
|
||||
mem_type="DRAM",
|
||||
addr=self._primary_base_addr + chunk_id * self._primary_row_stride,
|
||||
size=self._primary_row_stride,
|
||||
device_Id=0,
|
||||
info="",
|
||||
element_index=chunk_id,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user