[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Chang Guo
2026-07-17 10:31:52 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5
parent 26c909ed74
commit 426d48bfa1
11 changed files with 326 additions and 10 deletions
+6
View File
@@ -74,6 +74,8 @@ class BlockStored(KVCacheEvent):
# filter groups as they are learned. Remove events only need group_idx+hash.
kv_cache_spec_kind: str | None = None
kv_cache_spec_sliding_window: int | None = None
locality: str | None = None
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
def __hash__(self) -> int:
return hash(
@@ -88,6 +90,7 @@ class BlockStored(KVCacheEvent):
self.group_idx,
self.kv_cache_spec_kind,
self.kv_cache_spec_sliding_window,
self.locality,
)
)
@@ -96,6 +99,8 @@ class BlockRemoved(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
medium: str | None
group_idx: int | None = None
locality: str | None = None
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
def __hash__(self) -> int:
return hash(
@@ -103,6 +108,7 @@ class BlockRemoved(KVCacheEvent):
tuple(self.block_hashes),
self.medium,
self.group_idx,
self.locality,
)
)
@@ -198,7 +198,12 @@ class OffloadingEventsTracker:
kv_cache_spec=group_config.kv_event_group_spec,
)
def _placeholder_stored(self, key: OffloadKey, medium: str) -> BlockStored:
def _placeholder_stored(
self,
key: OffloadKey,
medium: str,
locality: str | None,
) -> BlockStored:
return BlockStored(
block_hashes=[
maybe_convert_block_hash(BlockHash(get_offload_block_hash(key)))
@@ -210,12 +215,14 @@ class OffloadingEventsTracker:
medium=medium,
lora_name=None,
group_idx=get_offload_group_idx(key),
locality=locality,
)
def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Metadata is read, NOT popped: the entry must survive until the
# eviction event so BlockRemoved can fan out to the same hashes.
# Events are self-contained (own parent), so key order is free.
locality = event.locality.value if event.locality is not None else None
for key in event.keys:
meta = self._pending_event_metadata.get(key)
if meta is None:
@@ -227,7 +234,7 @@ class OffloadingEventsTracker:
"placeholder payload. Expected for non-full-attention "
"groups; otherwise indicates a missing populate path."
)
yield self._placeholder_stored(key, event.medium)
yield self._placeholder_stored(key, event.medium, locality)
continue
yield BlockStored(
@@ -252,10 +259,12 @@ class OffloadingEventsTracker:
kv_cache_spec_sliding_window=(
meta.kv_cache_spec.kv_cache_spec_sliding_window
),
locality=locality,
)
def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Keep group_idx unambiguous if a manager batch spans groups.
locality = event.locality.value if event.locality is not None else None
by_group: dict[int, list] = {}
for key in event.keys:
meta = self._pending_event_metadata.pop(key, None)
@@ -283,4 +292,5 @@ class OffloadingEventsTracker:
block_hashes=hashes,
medium=event.medium,
group_idx=group_idx,
locality=locality,
)