[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
parent
26c909ed74
commit
426d48bfa1
@@ -74,6 +74,8 @@ class BlockStored(KVCacheEvent):
|
||||
# filter groups as they are learned. Remove events only need group_idx+hash.
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
locality: str | None = None
|
||||
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
@@ -88,6 +90,7 @@ class BlockStored(KVCacheEvent):
|
||||
self.group_idx,
|
||||
self.kv_cache_spec_kind,
|
||||
self.kv_cache_spec_sliding_window,
|
||||
self.locality,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -96,6 +99,8 @@ class BlockRemoved(KVCacheEvent):
|
||||
block_hashes: list[ExternalBlockHash]
|
||||
medium: str | None
|
||||
group_idx: int | None = None
|
||||
locality: str | None = None
|
||||
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
@@ -103,6 +108,7 @@ class BlockRemoved(KVCacheEvent):
|
||||
tuple(self.block_hashes),
|
||||
self.medium,
|
||||
self.group_idx,
|
||||
self.locality,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -198,7 +198,12 @@ class OffloadingEventsTracker:
|
||||
kv_cache_spec=group_config.kv_event_group_spec,
|
||||
)
|
||||
|
||||
def _placeholder_stored(self, key: OffloadKey, medium: str) -> BlockStored:
|
||||
def _placeholder_stored(
|
||||
self,
|
||||
key: OffloadKey,
|
||||
medium: str,
|
||||
locality: str | None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[
|
||||
maybe_convert_block_hash(BlockHash(get_offload_block_hash(key)))
|
||||
@@ -210,12 +215,14 @@ class OffloadingEventsTracker:
|
||||
medium=medium,
|
||||
lora_name=None,
|
||||
group_idx=get_offload_group_idx(key),
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
|
||||
# Metadata is read, NOT popped: the entry must survive until the
|
||||
# eviction event so BlockRemoved can fan out to the same hashes.
|
||||
# Events are self-contained (own parent), so key order is free.
|
||||
locality = event.locality.value if event.locality is not None else None
|
||||
for key in event.keys:
|
||||
meta = self._pending_event_metadata.get(key)
|
||||
if meta is None:
|
||||
@@ -227,7 +234,7 @@ class OffloadingEventsTracker:
|
||||
"placeholder payload. Expected for non-full-attention "
|
||||
"groups; otherwise indicates a missing populate path."
|
||||
)
|
||||
yield self._placeholder_stored(key, event.medium)
|
||||
yield self._placeholder_stored(key, event.medium, locality)
|
||||
continue
|
||||
|
||||
yield BlockStored(
|
||||
@@ -252,10 +259,12 @@ class OffloadingEventsTracker:
|
||||
kv_cache_spec_sliding_window=(
|
||||
meta.kv_cache_spec.kv_cache_spec_sliding_window
|
||||
),
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
|
||||
# Keep group_idx unambiguous if a manager batch spans groups.
|
||||
locality = event.locality.value if event.locality is not None else None
|
||||
by_group: dict[int, list] = {}
|
||||
for key in event.keys:
|
||||
meta = self._pending_event_metadata.pop(key, None)
|
||||
@@ -283,4 +292,5 @@ class OffloadingEventsTracker:
|
||||
block_hashes=hashes,
|
||||
medium=event.medium,
|
||||
group_idx=group_idx,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user