From f9d666f917941852b7d48ec20069f37480fafc28 Mon Sep 17 00:00:00 2001 From: Moein Khazraee <33970824+mkhazraee@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:28:41 -0700 Subject: [PATCH] [KV Offload] Forward ownership in KV cache events (#52067) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moein Khazraee Co-authored-by: Nicolò Lucchesi Co-authored-by: Cyrus Leung --- .../unit/offloading_connector/test_events.py | 15 +++++++++++++-- vllm/distributed/kv_events.py | 19 +++++++++++++------ .../kv_connector/v1/offloading/events.py | 10 ++++++++-- vllm/v1/kv_offload/base.py | 2 ++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index de3d3b0495..29fc29b299 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -141,12 +141,14 @@ def _stored_event( keys: list[OffloadKey], medium: Medium = _CPU_MEDIUM, locality: Locality | None = None, + ownership: str | None = None, ) -> OffloadingEvent: return OffloadingEvent( keys=keys, medium=medium, removed=False, locality=locality, + ownership=ownership, ) @@ -154,12 +156,14 @@ def _removed_event( keys: list[OffloadKey], medium: Medium = _CPU_MEDIUM, locality: Locality | None = None, + ownership: str | None = None, ) -> OffloadingEvent: return OffloadingEvent( keys=keys, medium=medium, removed=True, locality=locality, + ownership=ownership, ) @@ -534,8 +538,12 @@ def test_secondary_stored_event_does_not_mutate_cpu_metadata(): tracker, _, _, key = _lookup_chunk() expected_metadata = dict(tracker._pending_event_metadata) - stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)])) + stored = list( + tracker.take_events([_stored_event([key], Medium.STORAGE, ownership="custom")]) + ) + assert stored[0].token_ids == [1, 2, 3, 4] + assert stored[0].ownership == "custom" assert tracker._pending_event_metadata == expected_metadata @@ -548,9 +556,12 @@ def test_take_events_groups_removed_hashes_by_kv_group(): key0 = _record_chunks(tracker, req0, group0_config, num_chunks=1)[0] key1 = _record_chunks(tracker, req1, group1_config, num_chunks=1)[0] - removed = list(tracker.take_events([_removed_event([key0, key1])])) + removed = list( + tracker.take_events([_removed_event([key0, key1], ownership="custom")]) + ) assert len(removed) == 2 + assert {event.ownership for event in removed} == {"custom"} by_group = {event.group_idx: event.block_hashes for event in removed} assert by_group == { 0: [_wire_hash(_hash(0)), _wire_hash(_hash(1))], diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index cd222567a3..f55c5634cd 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -53,28 +53,31 @@ class BlockStored(KVCacheEvent): token_ids: list[int] block_size: int - lora_id: int | None """Deprecated: use `lora_name` for KV block key hash. Retained for backward compatibility. """ + lora_id: int | None medium: str | None lora_name: str | None - extra_keys: list[tuple[Any, ...] | None] | None = None """Extra keys used in block hash computation, one entry per block in block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt, prompt embedding hashes, etc. for that specific block. Exposed for external KV cache consumers to reconstruct block hashes. """ + extra_keys: list[tuple[Any, ...] | None] | None = None + """Store events carry cache-spec metadata so consumers can classify and + filter groups as they are learned. Remove events only need group_idx+hash. + """ group_idx: int | None = None - # Store events carry cache-spec metadata so consumers can classify and - # filter groups as they are learned. Remove events only need group_idx+hash. kv_cache_spec_kind: str | None = None kv_cache_spec_sliding_window: int | None = None - locality: str | None = None """LOCAL or REMOTE relative to the publisher; None means unspecified.""" + locality: str | None = None + """Secondary offloading tier identifier, if generated by one.""" + ownership: str | None = None def __hash__(self) -> int: return hash( @@ -90,6 +93,7 @@ class BlockStored(KVCacheEvent): self.kv_cache_spec_kind, self.kv_cache_spec_sliding_window, self.locality, + self.ownership, ) ) @@ -98,8 +102,10 @@ class BlockRemoved(KVCacheEvent): block_hashes: list[ExternalBlockHash] medium: str | None group_idx: int | None = None - locality: str | None = None """LOCAL or REMOTE relative to the publisher; None means unspecified.""" + locality: str | None = None + """Secondary offloading tier identifier, if generated by one.""" + ownership: str | None = None def __hash__(self) -> int: return hash( @@ -108,6 +114,7 @@ class BlockRemoved(KVCacheEvent): self.medium, self.group_idx, self.locality, + self.ownership, ) ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 9bd43a890c..9c1b6542e6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -79,7 +79,7 @@ def get_offloading_event_group_spec( class _OffloadEventMetadata: """BlockStored payload snapshot for one OffloadKey, captured while the Request is available and kept until the matching eviction event. ``medium`` - is forwarded from the OffloadingEvent.""" + and ``ownership`` are forwarded from the OffloadingEvent.""" # The chunk's constituent block hashes; the last one is the OffloadKey. block_hashes: tuple[BlockHash, ...] @@ -319,6 +319,7 @@ class OffloadingEventsTracker: key: OffloadKey, medium: Medium, locality: str | None, + ownership: str | None, ) -> BlockStored: return BlockStored( block_hashes=[ @@ -332,6 +333,7 @@ class OffloadingEventsTracker: lora_name=None, group_idx=get_offload_group_idx(key), locality=locality, + ownership=ownership, ) def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]: @@ -351,7 +353,9 @@ class OffloadingEventsTracker: "groups and promotions not observed as a primary-tier " "hit before translation." ) - yield self._placeholder_stored(key, event.medium, locality) + yield self._placeholder_stored( + key, event.medium, locality, event.ownership + ) continue yield BlockStored( @@ -377,6 +381,7 @@ class OffloadingEventsTracker: meta.kv_cache_spec.kv_cache_spec_sliding_window ), locality=locality, + ownership=event.ownership, ) def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]: @@ -410,4 +415,5 @@ class OffloadingEventsTracker: medium=_MEDIUM_TO_EVENT_STR[event.medium], group_idx=group_idx, locality=locality, + ownership=event.ownership, ) diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 90138fc250..b3b4df8fd7 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -157,6 +157,8 @@ class OffloadingEvent: # True if blocks are removed, False if stored removed: bool locality: Locality | None = None + # Secondary tier identifier that generated the event, or None for primary. + ownership: str | None = None """