[Bugfix][KV Offload] Retain offload event metadata through batch translation (#57453)

Signed-off-by: Kapil Arya <kapila@nvidia.com>
Signed-off-by: Kapil Arya <kapil.arya.17@gmail.com>
Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
Kapil Arya
2026-09-24 07:09:12 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 4fb767ff0a
commit 70fc359d25
2 changed files with 48 additions and 2 deletions
@@ -506,6 +506,44 @@ def test_event_metadata_skips_non_full_attention_group(
assert events[0].block_size == 0
@pytest.mark.parametrize("secondary_removal_in_same_batch", [False, True])
def test_primary_removal_preserves_metadata_for_same_batch_kvcr_store(
secondary_removal_in_same_batch,
):
"""A removal must not drop the payload a later store in the same batch
still needs, and must still drop it once no residency is left."""
tracker = _tracker()
group_config = _group_config(block_size=64, blocks_per_chunk=2)
req = _request(block_hashes=[_hash(i) for i in range(4)], token_count=256)
key = _record_chunks(tracker, req, group_config, num_chunks=2)[1]
secondary_removal = _removed_event([key], ownership="kvcr")
batch = [
_removed_event([key]),
_stored_event([key], ownership="kvcr", removal_expected=True),
]
if secondary_removal_in_same_batch:
batch.append(secondary_removal)
events = list(tracker.take_events(batch))
if not secondary_removal_in_same_batch:
events += tracker.take_events([secondary_removal])
removed, stored, final_removal = events
hashes = [_wire_hash(_hash(2)), _wire_hash(_hash(3))]
assert isinstance(removed, BlockRemoved)
assert removed.block_hashes == hashes
assert stored.block_size == 64
assert stored.token_ids == list(range(129, 257))
assert stored.parent_block_hash == _wire_hash(_hash(1))
assert stored.block_hashes == hashes
assert stored.ownership == "kvcr"
assert final_removal.block_hashes == hashes
# Nothing holds a residency now, so the payload is gone.
[after_eviction] = tracker.take_events([_stored_event([key], ownership="kvcr")])
assert after_eviction.block_size == 0
def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit():
tracker = _tracker()
block_hashes = [_hash(0), _hash(1)]
@@ -247,12 +247,22 @@ class OffloadingEventsTracker:
the underlying :class:`OffloadingEvent` stream.
"""
removed_keys: set[OffloadKey] = set()
for event in events:
if event.removed:
if self.self_describing_enabled:
removed_keys.update(event.keys)
yield from self._take_removed_event(event)
else:
yield from self._take_stored_event(event)
# A primary removal can precede a queued secondary store in this batch.
# Keep its payload until all stores have registered their residencies.
for key in removed_keys:
meta = self._pending_event_metadata.get(key)
if meta is not None and not meta.active_residencies:
self._pending_event_metadata.pop(key)
def reset(self) -> None:
"""Drop all tracked state; pending payloads are stale after a
manager cache reset."""
@@ -407,8 +417,6 @@ class OffloadingEventsTracker:
maybe_convert_block_hash(h) for h in meta.block_hashes
)
meta.active_residencies.discard((event.medium, event.ownership))
if not meta.active_residencies:
self._pending_event_metadata.pop(key)
else:
if self.self_describing_enabled:
logger.warning_once(