[KV Offload] Forward ownership in KV cache events (#52067)
Signed-off-by: Moein Khazraee <moein@nvidia.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Cyrus Leung
parent
8e92248f79
commit
f9d666f917
@@ -141,12 +141,14 @@ def _stored_event(
|
||||
keys: list[OffloadKey],
|
||||
medium: Medium = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
ownership: str | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
medium=medium,
|
||||
removed=False,
|
||||
locality=locality,
|
||||
ownership=ownership,
|
||||
)
|
||||
|
||||
|
||||
@@ -154,12 +156,14 @@ def _removed_event(
|
||||
keys: list[OffloadKey],
|
||||
medium: Medium = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
ownership: str | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
medium=medium,
|
||||
removed=True,
|
||||
locality=locality,
|
||||
ownership=ownership,
|
||||
)
|
||||
|
||||
|
||||
@@ -534,8 +538,12 @@ def test_secondary_stored_event_does_not_mutate_cpu_metadata():
|
||||
tracker, _, _, key = _lookup_chunk()
|
||||
expected_metadata = dict(tracker._pending_event_metadata)
|
||||
|
||||
stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)]))
|
||||
stored = list(
|
||||
tracker.take_events([_stored_event([key], Medium.STORAGE, ownership="custom")])
|
||||
)
|
||||
|
||||
assert stored[0].token_ids == [1, 2, 3, 4]
|
||||
assert stored[0].ownership == "custom"
|
||||
assert tracker._pending_event_metadata == expected_metadata
|
||||
|
||||
|
||||
@@ -548,9 +556,12 @@ def test_take_events_groups_removed_hashes_by_kv_group():
|
||||
key0 = _record_chunks(tracker, req0, group0_config, num_chunks=1)[0]
|
||||
key1 = _record_chunks(tracker, req1, group1_config, num_chunks=1)[0]
|
||||
|
||||
removed = list(tracker.take_events([_removed_event([key0, key1])]))
|
||||
removed = list(
|
||||
tracker.take_events([_removed_event([key0, key1], ownership="custom")])
|
||||
)
|
||||
|
||||
assert len(removed) == 2
|
||||
assert {event.ownership for event in removed} == {"custom"}
|
||||
by_group = {event.group_idx: event.block_hashes for event in removed}
|
||||
assert by_group == {
|
||||
0: [_wire_hash(_hash(0)), _wire_hash(_hash(1))],
|
||||
|
||||
@@ -53,28 +53,31 @@ class BlockStored(KVCacheEvent):
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
|
||||
lora_id: int | None
|
||||
"""Deprecated: use `lora_name` for KV block key hash.
|
||||
Retained for backward compatibility.
|
||||
"""
|
||||
lora_id: int | None
|
||||
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
"""Extra keys used in block hash computation, one entry per block in
|
||||
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
|
||||
prompt embedding hashes, etc. for that specific block. Exposed for external
|
||||
KV cache consumers to reconstruct block hashes.
|
||||
"""
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
|
||||
"""Store events carry cache-spec metadata so consumers can classify and
|
||||
filter groups as they are learned. Remove events only need group_idx+hash.
|
||||
"""
|
||||
group_idx: int | None = None
|
||||
# Store events carry cache-spec metadata so consumers can classify and
|
||||
# filter groups as they are learned. Remove events only need group_idx+hash.
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
locality: str | None = None
|
||||
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
|
||||
locality: str | None = None
|
||||
"""Secondary offloading tier identifier, if generated by one."""
|
||||
ownership: str | None = None
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
@@ -90,6 +93,7 @@ class BlockStored(KVCacheEvent):
|
||||
self.kv_cache_spec_kind,
|
||||
self.kv_cache_spec_sliding_window,
|
||||
self.locality,
|
||||
self.ownership,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -98,8 +102,10 @@ class BlockRemoved(KVCacheEvent):
|
||||
block_hashes: list[ExternalBlockHash]
|
||||
medium: str | None
|
||||
group_idx: int | None = None
|
||||
locality: str | None = None
|
||||
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
|
||||
locality: str | None = None
|
||||
"""Secondary offloading tier identifier, if generated by one."""
|
||||
ownership: str | None = None
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(
|
||||
@@ -108,6 +114,7 @@ class BlockRemoved(KVCacheEvent):
|
||||
self.medium,
|
||||
self.group_idx,
|
||||
self.locality,
|
||||
self.ownership,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ def get_offloading_event_group_spec(
|
||||
class _OffloadEventMetadata:
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured while the
|
||||
Request is available and kept until the matching eviction event. ``medium``
|
||||
is forwarded from the OffloadingEvent."""
|
||||
and ``ownership`` are forwarded from the OffloadingEvent."""
|
||||
|
||||
# The chunk's constituent block hashes; the last one is the OffloadKey.
|
||||
block_hashes: tuple[BlockHash, ...]
|
||||
@@ -319,6 +319,7 @@ class OffloadingEventsTracker:
|
||||
key: OffloadKey,
|
||||
medium: Medium,
|
||||
locality: str | None,
|
||||
ownership: str | None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[
|
||||
@@ -332,6 +333,7 @@ class OffloadingEventsTracker:
|
||||
lora_name=None,
|
||||
group_idx=get_offload_group_idx(key),
|
||||
locality=locality,
|
||||
ownership=ownership,
|
||||
)
|
||||
|
||||
def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
|
||||
@@ -351,7 +353,9 @@ class OffloadingEventsTracker:
|
||||
"groups and promotions not observed as a primary-tier "
|
||||
"hit before translation."
|
||||
)
|
||||
yield self._placeholder_stored(key, event.medium, locality)
|
||||
yield self._placeholder_stored(
|
||||
key, event.medium, locality, event.ownership
|
||||
)
|
||||
continue
|
||||
|
||||
yield BlockStored(
|
||||
@@ -377,6 +381,7 @@ class OffloadingEventsTracker:
|
||||
meta.kv_cache_spec.kv_cache_spec_sliding_window
|
||||
),
|
||||
locality=locality,
|
||||
ownership=event.ownership,
|
||||
)
|
||||
|
||||
def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
|
||||
@@ -410,4 +415,5 @@ class OffloadingEventsTracker:
|
||||
medium=_MEDIUM_TO_EVENT_STR[event.medium],
|
||||
group_idx=group_idx,
|
||||
locality=locality,
|
||||
ownership=event.ownership,
|
||||
)
|
||||
|
||||
@@ -157,6 +157,8 @@ class OffloadingEvent:
|
||||
# True if blocks are removed, False if stored
|
||||
removed: bool
|
||||
locality: Locality | None = None
|
||||
# Secondary tier identifier that generated the event, or None for primary.
|
||||
ownership: str | None = None
|
||||
|
||||
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user