[kv_offload] Establish tier-owned KV event handling (#46544)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <changg@nvidia.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chang Guo
2026-07-07 16:55:20 +03:00
committed by GitHub
co-authored by Claude Opus 4.8 Or Ozeri
parent 65dcde1695
commit c46ced1ee3
11 changed files with 61 additions and 45 deletions
@@ -7,7 +7,7 @@ import torch
from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import BlockRemoved, BlockStored
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
OffloadingEventGroupSpec,
OffloadingEventsTracker,
@@ -28,10 +28,9 @@ from vllm.v1.kv_offload.base import (
OffloadKey,
make_offload_key,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
_CPU_MEDIUM = CPULoadStoreSpec.medium()
_CPU_MEDIUM = MEDIUM_CPU
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
kv_cache_spec_sliding_window=None,
@@ -73,10 +73,6 @@ class MockLoadStoreSpec(LoadStoreSpec):
def __init__(self, offload_keys: Iterable[OffloadKey]):
self.offload_keys: list[OffloadKey] = list(offload_keys)
@staticmethod
def medium() -> str:
return "Mock"
def __repr__(self) -> str:
return repr(self.offload_keys)
+21 -1
View File
@@ -6,6 +6,7 @@ from dataclasses import dataclass
import numpy as np
import pytest
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
@@ -99,7 +100,7 @@ def verify_events(
stores: list[set[OffloadKey]] = []
evictions: list[set[OffloadKey]] = []
for event in events:
assert event.medium == CPULoadStoreSpec.medium()
assert event.medium == MEDIUM_CPU
if event.removed:
evictions.append(set(event.keys))
else:
@@ -114,6 +115,25 @@ def verify_events(
assert tuple(stores) == to_key_sets(expected_stores)
def test_cpu_eviction_removed_precedes_stored():
"""An eviction is announced before the store that reuses its capacity."""
manager = make_cpu_manager(num_blocks=2, enable_events=True)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
list(manager.take_events())
manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
events = list(manager.take_events())
removed_idx = [i for i, event in enumerate(events) if event.removed]
stored_idx = [i for i, event in enumerate(events) if not event.removed]
assert removed_idx and stored_idx, events
assert max(removed_idx) < min(stored_idx)
assert all(event.medium == manager.medium for event in events)
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
"""
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingCounterMetadata,
OffloadingEvent,
OffloadKey,
OffloadPolicy,
ReqContext,
@@ -242,6 +243,24 @@ class TestTieringOffloadingManager:
if req_context.req_id not in self.manager._req_state:
self.manager.on_new_request(req_context)
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
self.secondary_tier2.take_events = MagicMock(return_value=[secondary_event2])
assert list(self.manager.take_events()) == [
primary_event,
secondary_event1,
secondary_event2,
]
self.primary_tier.take_events.assert_called_once_with()
self.secondary_tier1.take_events.assert_called_once_with()
self.secondary_tier2.take_events.assert_called_once_with()
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))