[KV Offload] Forward ownership in KV cache events (#52067)

Signed-off-by: Moein Khazraee <moein@nvidia.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Moein Khazraee
2026-08-31 10:28:41 +03:00
committed by GitHub
co-authored by Nicolò Lucchesi Cyrus Leung
parent 8e92248f79
commit f9d666f917
4 changed files with 36 additions and 10 deletions
@@ -141,12 +141,14 @@ def _stored_event(
keys: list[OffloadKey],
medium: Medium = _CPU_MEDIUM,
locality: Locality | None = None,
ownership: str | None = None,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=False,
locality=locality,
ownership=ownership,
)
@@ -154,12 +156,14 @@ def _removed_event(
keys: list[OffloadKey],
medium: Medium = _CPU_MEDIUM,
locality: Locality | None = None,
ownership: str | None = None,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=True,
locality=locality,
ownership=ownership,
)
@@ -534,8 +538,12 @@ def test_secondary_stored_event_does_not_mutate_cpu_metadata():
tracker, _, _, key = _lookup_chunk()
expected_metadata = dict(tracker._pending_event_metadata)
stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)]))
stored = list(
tracker.take_events([_stored_event([key], Medium.STORAGE, ownership="custom")])
)
assert stored[0].token_ids == [1, 2, 3, 4]
assert stored[0].ownership == "custom"
assert tracker._pending_event_metadata == expected_metadata
@@ -548,9 +556,12 @@ def test_take_events_groups_removed_hashes_by_kv_group():
key0 = _record_chunks(tracker, req0, group0_config, num_chunks=1)[0]
key1 = _record_chunks(tracker, req1, group1_config, num_chunks=1)[0]
removed = list(tracker.take_events([_removed_event([key0, key1])]))
removed = list(
tracker.take_events([_removed_event([key0, key1], ownership="custom")])
)
assert len(removed) == 2
assert {event.ownership for event in removed} == {"custom"}
by_group = {event.group_idx: event.block_hashes for event in removed}
assert by_group == {
0: [_wire_hash(_hash(0)), _wire_hash(_hash(1))],