diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index 9e5f564bba..beb639724e 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -7,7 +7,7 @@ import torch from tests.v1.kv_connector.unit.utils import create_vllm_config from vllm.config import KVEventsConfig, KVTransferConfig -from vllm.distributed.kv_events import BlockRemoved, BlockStored +from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import ( OffloadingEventGroupSpec, OffloadingEventsTracker, @@ -28,10 +28,9 @@ from vllm.v1.kv_offload.base import ( OffloadKey, make_offload_key, ) -from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec -_CPU_MEDIUM = CPULoadStoreSpec.medium() +_CPU_MEDIUM = MEDIUM_CPU _FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec( kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value, kv_cache_spec_sliding_window=None, diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index 0d152a2038..73ea5e2be1 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -73,10 +73,6 @@ class MockLoadStoreSpec(LoadStoreSpec): def __init__(self, offload_keys: Iterable[OffloadKey]): self.offload_keys: list[OffloadKey] = list(offload_keys) - @staticmethod - def medium() -> str: - return "Mock" - def __repr__(self) -> str: return repr(self.offload_keys) diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 15cac05158..999f7b38bc 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import numpy as np import pytest +from vllm.distributed.kv_events import MEDIUM_CPU from vllm.v1.kv_offload.base import ( LoadStoreSpec, LookupResult, @@ -99,7 +100,7 @@ def verify_events( stores: list[set[OffloadKey]] = [] evictions: list[set[OffloadKey]] = [] for event in events: - assert event.medium == CPULoadStoreSpec.medium() + assert event.medium == MEDIUM_CPU if event.removed: evictions.append(set(event.keys)) else: @@ -114,6 +115,25 @@ def verify_events( assert tuple(stores) == to_key_sets(expected_stores) +def test_cpu_eviction_removed_precedes_stored(): + """An eviction is announced before the store that reuses its capacity.""" + manager = make_cpu_manager(num_blocks=2, enable_events=True) + + manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX) + manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX) + list(manager.take_events()) + + manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX) + manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX) + + events = list(manager.take_events()) + removed_idx = [i for i, event in enumerate(events) if event.removed] + stored_idx = [i for i, event in enumerate(events) if not event.removed] + assert removed_idx and stored_idx, events + assert max(removed_idx) < min(stored_idx) + assert all(event.medium == manager.medium for event in events) + + @pytest.mark.parametrize("eviction_policy", ["lru", "arc"]) def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy): """ diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index fe9283a811..13460f68d7 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( from vllm.v1.kv_offload.base import ( LookupResult, OffloadingCounterMetadata, + OffloadingEvent, OffloadKey, OffloadPolicy, ReqContext, @@ -242,6 +243,24 @@ class TestTieringOffloadingManager: if req_context.req_id not in self.manager._req_state: self.manager.on_new_request(req_context) + def test_take_events_aggregates_tier_owned_events(self, manager_setup): + primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False) + secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False) + secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True) + + self.primary_tier.take_events = MagicMock(return_value=[primary_event]) + self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1]) + self.secondary_tier2.take_events = MagicMock(return_value=[secondary_event2]) + + assert list(self.manager.take_events()) == [ + primary_event, + secondary_event1, + secondary_event2, + ] + self.primary_tier.take_events.assert_called_once_with() + self.secondary_tier1.take_events.assert_called_once_with() + self.secondary_tier2.take_events.assert_called_once_with() + def test_basic_store_to_primary(self, manager_setup): """Test basic store operation to primary tier.""" blocks = to_keys(range(3)) diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index adc8b08269..2f363f0bc0 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -43,6 +43,7 @@ class KVCacheEvent( MEDIUM_GPU = "GPU" +MEDIUM_CPU = "CPU" class BlockStored(KVCacheEvent): diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 48838599d6..5a2e3c184d 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple, NewType import numpy as np import torch -from typing_extensions import override from vllm.logger import init_logger from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes @@ -85,21 +84,12 @@ class ScheduleEndContext(NamedTuple): preempted_req_ids: Collection[str] -class LoadStoreSpec(ABC): +class LoadStoreSpec: """ - Abstract metadata that encapsulates information allowing a worker + Metadata that encapsulates information allowing a worker to load, and optionally also to store, blocks of KV data. """ - @staticmethod - @abstractmethod - def medium() -> str: - """ - Returns a string representation of the medium type - this store/load targets. - """ - pass - @dataclass class PrepareStoreOutput: @@ -313,6 +303,10 @@ class OffloadingManager(ABC): """ Take the offloading events from the manager. + A tier manager emits only events for storage state it owns. A + composing manager may aggregate child event streams, but should not + synthesize events on behalf of a child tier. + Yields: New OffloadingEvents collected since the last call. """ @@ -392,11 +386,6 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec): self.group_sizes: Sequence[int] = group_sizes self.block_indices: Sequence[int] = block_indices - @staticmethod - @override - def medium() -> str: - return "GPU" - @dataclass class CanonicalKVCacheTensor: diff --git a/vllm/v1/kv_offload/cpu/common.py b/vllm/v1/kv_offload/cpu/common.py index b6a2c2256a..29f95e1b40 100644 --- a/vllm/v1/kv_offload/cpu/common.py +++ b/vllm/v1/kv_offload/cpu/common.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing_extensions import override - from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec @@ -15,8 +13,3 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec): """ Spec for loading/storing a KV block to CPU memory. """ - - @staticmethod - @override - def medium() -> str: - return "CPU" diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index fa5a9e04e6..f0a3d9dead 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -6,6 +6,7 @@ from typing import Literal from typing_extensions import override +from vllm.distributed.kv_events import MEDIUM_CPU from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, ) @@ -51,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager): store_threshold: int = 1, max_tracker_size: int = 64_000, ): - self.medium: str = CPULoadStoreSpec.medium() + self.medium: str = MEDIUM_CPU self._num_blocks: int = num_blocks self._num_allocated_blocks: int = 0 self._free_list: list[int] = [] diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index 9e3d19644e..f83113e137 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -13,6 +13,7 @@ import numpy as np from vllm.v1.kv_offload.base import ( LookupResult, + OffloadingEvent, OffloadingMetricMetadata, OffloadKey, ReqContext, @@ -208,6 +209,10 @@ class SecondaryTierManager(ABC): """ return False + def take_events(self) -> Iterable[OffloadingEvent]: + """Take KV events for storage state owned by this tier.""" + return () + def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): """ Mark blocks as recently used for eviction policy. diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 156f5a96e1..728ac8dc86 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -170,7 +170,6 @@ class TieringOffloadingManager(OffloadingManager): self, primary_tier: CPUPrimaryTierOffloadingManager, secondary_tiers: list[SecondaryTierManager] | None = None, - enable_events: bool = False, ): """ Initialize the TieringOffloadingManager. @@ -179,14 +178,11 @@ class TieringOffloadingManager(OffloadingManager): primary_tier: The primary tier manager (CPU-based). secondary_tiers: List of secondary tier managers (e.g., Storage, Network). Can be None or empty list. - enable_events: Whether to track offloading events """ self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier self.secondary_tiers = secondary_tiers or [] self._job_id_counter: int = 0 - self.events: list[OffloadingEvent] | None = [] if enable_events else None - # Job tracking: maps job_id to metadata for all in-flight transfers. # JobMetadata.is_promotion distinguishes direction: # True: secondary → primary (promotion) @@ -539,7 +535,7 @@ class TieringOffloadingManager(OffloadingManager): keys: Collection[OffloadKey], req_context: ReqContext, success: bool = True, - ): + ) -> None: """ Mark blocks as done storing from GPU to primary tier. @@ -707,16 +703,14 @@ class TieringOffloadingManager(OffloadingManager): @override def take_events(self) -> Iterable[OffloadingEvent]: - """Yield offloading events collected since the last call. + """Yield events owned by the primary and secondary tiers. Yields: - New OffloadingEvents collected since the last call. + New OffloadingEvents collected by each tier since the last call. """ - if self.events is not None: - yield from self.events - self.events.clear() - yield from self.primary_tier.take_events() + for tier in self.secondary_tiers: + yield from tier.take_events() @override def reset_cache(self) -> None: diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 406c94e3d7..3dc31a3622 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -168,7 +168,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec): tiering_manager = TieringOffloadingManager( primary_tier=primary_tier, secondary_tiers=secondary_tiers, - enable_events=self.kv_events_config.enable_kv_cache_events, ) if int(self.extra_config.get("store_threshold", 0)) >= 2: raise ValueError(