[kv_offload] Establish tier-owned KV event handling (#46544)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <changg@nvidia.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Or Ozeri
parent
65dcde1695
commit
c46ced1ee3
@@ -7,7 +7,7 @@ import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
|
||||
OffloadingEventGroupSpec,
|
||||
OffloadingEventsTracker,
|
||||
@@ -28,10 +28,9 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadKey,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
|
||||
_CPU_MEDIUM = CPULoadStoreSpec.medium()
|
||||
_CPU_MEDIUM = MEDIUM_CPU
|
||||
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
|
||||
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
|
||||
kv_cache_spec_sliding_window=None,
|
||||
|
||||
@@ -73,10 +73,6 @@ class MockLoadStoreSpec(LoadStoreSpec):
|
||||
def __init__(self, offload_keys: Iterable[OffloadKey]):
|
||||
self.offload_keys: list[OffloadKey] = list(offload_keys)
|
||||
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "Mock"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return repr(self.offload_keys)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
@@ -99,7 +100,7 @@ def verify_events(
|
||||
stores: list[set[OffloadKey]] = []
|
||||
evictions: list[set[OffloadKey]] = []
|
||||
for event in events:
|
||||
assert event.medium == CPULoadStoreSpec.medium()
|
||||
assert event.medium == MEDIUM_CPU
|
||||
if event.removed:
|
||||
evictions.append(set(event.keys))
|
||||
else:
|
||||
@@ -114,6 +115,25 @@ def verify_events(
|
||||
assert tuple(stores) == to_key_sets(expected_stores)
|
||||
|
||||
|
||||
def test_cpu_eviction_removed_precedes_stored():
|
||||
"""An eviction is announced before the store that reuses its capacity."""
|
||||
manager = make_cpu_manager(num_blocks=2, enable_events=True)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
list(manager.take_events())
|
||||
|
||||
manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
|
||||
events = list(manager.take_events())
|
||||
removed_idx = [i for i, event in enumerate(events) if event.removed]
|
||||
stored_idx = [i for i, event in enumerate(events) if not event.removed]
|
||||
assert removed_idx and stored_idx, events
|
||||
assert max(removed_idx) < min(stored_idx)
|
||||
assert all(event.medium == manager.medium for event in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
||||
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
"""
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
@@ -242,6 +243,24 @@ class TestTieringOffloadingManager:
|
||||
if req_context.req_id not in self.manager._req_state:
|
||||
self.manager.on_new_request(req_context)
|
||||
|
||||
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
|
||||
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
|
||||
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
|
||||
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
|
||||
|
||||
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
|
||||
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
|
||||
self.secondary_tier2.take_events = MagicMock(return_value=[secondary_event2])
|
||||
|
||||
assert list(self.manager.take_events()) == [
|
||||
primary_event,
|
||||
secondary_event1,
|
||||
secondary_event2,
|
||||
]
|
||||
self.primary_tier.take_events.assert_called_once_with()
|
||||
self.secondary_tier1.take_events.assert_called_once_with()
|
||||
self.secondary_tier2.take_events.assert_called_once_with()
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
Reference in New Issue
Block a user