From 70fc359d257a984903b7ab9e123fbc45085c5254 Mon Sep 17 00:00:00 2001 From: Kapil Arya Date: Wed, 23 Sep 2026 21:09:12 -0700 Subject: [PATCH] [Bugfix][KV Offload] Retain offload event metadata through batch translation (#57453) Signed-off-by: Kapil Arya Signed-off-by: Kapil Arya Co-authored-by: Or Ozeri --- .../unit/offloading_connector/test_events.py | 38 +++++++++++++++++++ .../kv_connector/v1/offloading/events.py | 12 +++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index c409fd125d..0121ed95bf 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -506,6 +506,44 @@ def test_event_metadata_skips_non_full_attention_group( assert events[0].block_size == 0 +@pytest.mark.parametrize("secondary_removal_in_same_batch", [False, True]) +def test_primary_removal_preserves_metadata_for_same_batch_kvcr_store( + secondary_removal_in_same_batch, +): + """A removal must not drop the payload a later store in the same batch + still needs, and must still drop it once no residency is left.""" + tracker = _tracker() + group_config = _group_config(block_size=64, blocks_per_chunk=2) + req = _request(block_hashes=[_hash(i) for i in range(4)], token_count=256) + key = _record_chunks(tracker, req, group_config, num_chunks=2)[1] + secondary_removal = _removed_event([key], ownership="kvcr") + + batch = [ + _removed_event([key]), + _stored_event([key], ownership="kvcr", removal_expected=True), + ] + if secondary_removal_in_same_batch: + batch.append(secondary_removal) + events = list(tracker.take_events(batch)) + if not secondary_removal_in_same_batch: + events += tracker.take_events([secondary_removal]) + removed, stored, final_removal = events + + hashes = [_wire_hash(_hash(2)), _wire_hash(_hash(3))] + assert isinstance(removed, BlockRemoved) + assert removed.block_hashes == hashes + assert stored.block_size == 64 + assert stored.token_ids == list(range(129, 257)) + assert stored.parent_block_hash == _wire_hash(_hash(1)) + assert stored.block_hashes == hashes + assert stored.ownership == "kvcr" + assert final_removal.block_hashes == hashes + + # Nothing holds a residency now, so the payload is gone. + [after_eviction] = tracker.take_events([_stored_event([key], ownership="kvcr")]) + assert after_eviction.block_size == 0 + + def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit(): tracker = _tracker() block_hashes = [_hash(0), _hash(1)] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 6dcfcb5ec0..68318a16da 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -247,12 +247,22 @@ class OffloadingEventsTracker: the underlying :class:`OffloadingEvent` stream. """ + removed_keys: set[OffloadKey] = set() for event in events: if event.removed: + if self.self_describing_enabled: + removed_keys.update(event.keys) yield from self._take_removed_event(event) else: yield from self._take_stored_event(event) + # A primary removal can precede a queued secondary store in this batch. + # Keep its payload until all stores have registered their residencies. + for key in removed_keys: + meta = self._pending_event_metadata.get(key) + if meta is not None and not meta.active_residencies: + self._pending_event_metadata.pop(key) + def reset(self) -> None: """Drop all tracked state; pending payloads are stale after a manager cache reset.""" @@ -407,8 +417,6 @@ class OffloadingEventsTracker: maybe_convert_block_hash(h) for h in meta.block_hashes ) meta.active_residencies.discard((event.medium, event.ownership)) - if not meta.active_residencies: - self._pending_event_metadata.pop(key) else: if self.self_describing_enabled: logger.warning_once(