[kv_offload] Establish tier-owned KV event handling (#46544)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <changg@nvidia.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chang Guo
2026-07-07 16:55:20 +03:00
committed by GitHub
co-authored by Claude Opus 4.8 Or Ozeri
parent 65dcde1695
commit c46ced1ee3
11 changed files with 61 additions and 45 deletions
@@ -7,7 +7,7 @@ import torch
from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import BlockRemoved, BlockStored
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
OffloadingEventGroupSpec,
OffloadingEventsTracker,
@@ -28,10 +28,9 @@ from vllm.v1.kv_offload.base import (
OffloadKey,
make_offload_key,
)
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
_CPU_MEDIUM = CPULoadStoreSpec.medium()
_CPU_MEDIUM = MEDIUM_CPU
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
kv_cache_spec_sliding_window=None,
@@ -73,10 +73,6 @@ class MockLoadStoreSpec(LoadStoreSpec):
def __init__(self, offload_keys: Iterable[OffloadKey]):
self.offload_keys: list[OffloadKey] = list(offload_keys)
@staticmethod
def medium() -> str:
return "Mock"
def __repr__(self) -> str:
return repr(self.offload_keys)
+21 -1
View File
@@ -6,6 +6,7 @@ from dataclasses import dataclass
import numpy as np
import pytest
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
@@ -99,7 +100,7 @@ def verify_events(
stores: list[set[OffloadKey]] = []
evictions: list[set[OffloadKey]] = []
for event in events:
assert event.medium == CPULoadStoreSpec.medium()
assert event.medium == MEDIUM_CPU
if event.removed:
evictions.append(set(event.keys))
else:
@@ -114,6 +115,25 @@ def verify_events(
assert tuple(stores) == to_key_sets(expected_stores)
def test_cpu_eviction_removed_precedes_stored():
"""An eviction is announced before the store that reuses its capacity."""
manager = make_cpu_manager(num_blocks=2, enable_events=True)
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
list(manager.take_events())
manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
events = list(manager.take_events())
removed_idx = [i for i, event in enumerate(events) if event.removed]
stored_idx = [i for i, event in enumerate(events) if not event.removed]
assert removed_idx and stored_idx, events
assert max(removed_idx) < min(stored_idx)
assert all(event.medium == manager.medium for event in events)
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
"""
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingCounterMetadata,
OffloadingEvent,
OffloadKey,
OffloadPolicy,
ReqContext,
@@ -242,6 +243,24 @@ class TestTieringOffloadingManager:
if req_context.req_id not in self.manager._req_state:
self.manager.on_new_request(req_context)
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
self.secondary_tier2.take_events = MagicMock(return_value=[secondary_event2])
assert list(self.manager.take_events()) == [
primary_event,
secondary_event1,
secondary_event2,
]
self.primary_tier.take_events.assert_called_once_with()
self.secondary_tier1.take_events.assert_called_once_with()
self.secondary_tier2.take_events.assert_called_once_with()
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
+1
View File
@@ -43,6 +43,7 @@ class KVCacheEvent(
MEDIUM_GPU = "GPU"
MEDIUM_CPU = "CPU"
class BlockStored(KVCacheEvent):
+6 -17
View File
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple, NewType
import numpy as np
import torch
from typing_extensions import override
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
@@ -85,21 +84,12 @@ class ScheduleEndContext(NamedTuple):
preempted_req_ids: Collection[str]
class LoadStoreSpec(ABC):
class LoadStoreSpec:
"""
Abstract metadata that encapsulates information allowing a worker
Metadata that encapsulates information allowing a worker
to load, and optionally also to store, blocks of KV data.
"""
@staticmethod
@abstractmethod
def medium() -> str:
"""
Returns a string representation of the medium type
this store/load targets.
"""
pass
@dataclass
class PrepareStoreOutput:
@@ -313,6 +303,10 @@ class OffloadingManager(ABC):
"""
Take the offloading events from the manager.
A tier manager emits only events for storage state it owns. A
composing manager may aggregate child event streams, but should not
synthesize events on behalf of a child tier.
Yields:
New OffloadingEvents collected since the last call.
"""
@@ -392,11 +386,6 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec):
self.group_sizes: Sequence[int] = group_sizes
self.block_indices: Sequence[int] = block_indices
@staticmethod
@override
def medium() -> str:
return "GPU"
@dataclass
class CanonicalKVCacheTensor:
-7
View File
@@ -1,7 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing_extensions import override
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
@@ -15,8 +13,3 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
"""
Spec for loading/storing a KV block to CPU memory.
"""
@staticmethod
@override
def medium() -> str:
return "CPU"
+2 -1
View File
@@ -6,6 +6,7 @@ from typing import Literal
from typing_extensions import override
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
@@ -51,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager):
store_threshold: int = 1,
max_tracker_size: int = 64_000,
):
self.medium: str = CPULoadStoreSpec.medium()
self.medium: str = MEDIUM_CPU
self._num_blocks: int = num_blocks
self._num_allocated_blocks: int = 0
self._free_list: list[int] = []
+5
View File
@@ -13,6 +13,7 @@ import numpy as np
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingEvent,
OffloadingMetricMetadata,
OffloadKey,
ReqContext,
@@ -208,6 +209,10 @@ class SecondaryTierManager(ABC):
"""
return False
def take_events(self) -> Iterable[OffloadingEvent]:
"""Take KV events for storage state owned by this tier."""
return ()
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""
Mark blocks as recently used for eviction policy.
+5 -11
View File
@@ -170,7 +170,6 @@ class TieringOffloadingManager(OffloadingManager):
self,
primary_tier: CPUPrimaryTierOffloadingManager,
secondary_tiers: list[SecondaryTierManager] | None = None,
enable_events: bool = False,
):
"""
Initialize the TieringOffloadingManager.
@@ -179,14 +178,11 @@ class TieringOffloadingManager(OffloadingManager):
primary_tier: The primary tier manager (CPU-based).
secondary_tiers: List of secondary tier managers (e.g., Storage,
Network). Can be None or empty list.
enable_events: Whether to track offloading events
"""
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
self.secondary_tiers = secondary_tiers or []
self._job_id_counter: int = 0
self.events: list[OffloadingEvent] | None = [] if enable_events else None
# Job tracking: maps job_id to metadata for all in-flight transfers.
# JobMetadata.is_promotion distinguishes direction:
# True: secondary → primary (promotion)
@@ -539,7 +535,7 @@ class TieringOffloadingManager(OffloadingManager):
keys: Collection[OffloadKey],
req_context: ReqContext,
success: bool = True,
):
) -> None:
"""
Mark blocks as done storing from GPU to primary tier.
@@ -707,16 +703,14 @@ class TieringOffloadingManager(OffloadingManager):
@override
def take_events(self) -> Iterable[OffloadingEvent]:
"""Yield offloading events collected since the last call.
"""Yield events owned by the primary and secondary tiers.
Yields:
New OffloadingEvents collected since the last call.
New OffloadingEvents collected by each tier since the last call.
"""
if self.events is not None:
yield from self.events
self.events.clear()
yield from self.primary_tier.take_events()
for tier in self.secondary_tiers:
yield from tier.take_events()
@override
def reset_cache(self) -> None:
-1
View File
@@ -168,7 +168,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
tiering_manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=secondary_tiers,
enable_events=self.kv_events_config.enable_kv_cache_events,
)
if int(self.extra_config.get("store_threshold", 0)) >= 2:
raise ValueError(