[kv_offload] Establish tier-owned KV event handling (#46544)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <changg@nvidia.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Or Ozeri
parent
65dcde1695
commit
c46ced1ee3
@@ -7,7 +7,7 @@ import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.events import (
|
||||
OffloadingEventGroupSpec,
|
||||
OffloadingEventsTracker,
|
||||
@@ -28,10 +28,9 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadKey,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
|
||||
_CPU_MEDIUM = CPULoadStoreSpec.medium()
|
||||
_CPU_MEDIUM = MEDIUM_CPU
|
||||
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
|
||||
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
|
||||
kv_cache_spec_sliding_window=None,
|
||||
|
||||
@@ -73,10 +73,6 @@ class MockLoadStoreSpec(LoadStoreSpec):
|
||||
def __init__(self, offload_keys: Iterable[OffloadKey]):
|
||||
self.offload_keys: list[OffloadKey] = list(offload_keys)
|
||||
|
||||
@staticmethod
|
||||
def medium() -> str:
|
||||
return "Mock"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return repr(self.offload_keys)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
@@ -99,7 +100,7 @@ def verify_events(
|
||||
stores: list[set[OffloadKey]] = []
|
||||
evictions: list[set[OffloadKey]] = []
|
||||
for event in events:
|
||||
assert event.medium == CPULoadStoreSpec.medium()
|
||||
assert event.medium == MEDIUM_CPU
|
||||
if event.removed:
|
||||
evictions.append(set(event.keys))
|
||||
else:
|
||||
@@ -114,6 +115,25 @@ def verify_events(
|
||||
assert tuple(stores) == to_key_sets(expected_stores)
|
||||
|
||||
|
||||
def test_cpu_eviction_removed_precedes_stored():
|
||||
"""An eviction is announced before the store that reuses its capacity."""
|
||||
manager = make_cpu_manager(num_blocks=2, enable_events=True)
|
||||
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
list(manager.take_events())
|
||||
|
||||
manager.prepare_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
manager.complete_store(to_keys([3]), _EMPTY_REQ_CTX)
|
||||
|
||||
events = list(manager.take_events())
|
||||
removed_idx = [i for i, event in enumerate(events) if event.removed]
|
||||
stored_idx = [i for i, event in enumerate(events) if not event.removed]
|
||||
assert removed_idx and stored_idx, events
|
||||
assert max(removed_idx) < min(stored_idx)
|
||||
assert all(event.medium == manager.medium for event in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eviction_policy", ["lru", "arc"])
|
||||
def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
|
||||
"""
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
@@ -242,6 +243,24 @@ class TestTieringOffloadingManager:
|
||||
if req_context.req_id not in self.manager._req_state:
|
||||
self.manager.on_new_request(req_context)
|
||||
|
||||
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
|
||||
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
|
||||
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
|
||||
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
|
||||
|
||||
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
|
||||
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
|
||||
self.secondary_tier2.take_events = MagicMock(return_value=[secondary_event2])
|
||||
|
||||
assert list(self.manager.take_events()) == [
|
||||
primary_event,
|
||||
secondary_event1,
|
||||
secondary_event2,
|
||||
]
|
||||
self.primary_tier.take_events.assert_called_once_with()
|
||||
self.secondary_tier1.take_events.assert_called_once_with()
|
||||
self.secondary_tier2.take_events.assert_called_once_with()
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
@@ -43,6 +43,7 @@ class KVCacheEvent(
|
||||
|
||||
|
||||
MEDIUM_GPU = "GPU"
|
||||
MEDIUM_CPU = "CPU"
|
||||
|
||||
|
||||
class BlockStored(KVCacheEvent):
|
||||
|
||||
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple, NewType
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
@@ -85,21 +84,12 @@ class ScheduleEndContext(NamedTuple):
|
||||
preempted_req_ids: Collection[str]
|
||||
|
||||
|
||||
class LoadStoreSpec(ABC):
|
||||
class LoadStoreSpec:
|
||||
"""
|
||||
Abstract metadata that encapsulates information allowing a worker
|
||||
Metadata that encapsulates information allowing a worker
|
||||
to load, and optionally also to store, blocks of KV data.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def medium() -> str:
|
||||
"""
|
||||
Returns a string representation of the medium type
|
||||
this store/load targets.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrepareStoreOutput:
|
||||
@@ -313,6 +303,10 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
Take the offloading events from the manager.
|
||||
|
||||
A tier manager emits only events for storage state it owns. A
|
||||
composing manager may aggregate child event streams, but should not
|
||||
synthesize events on behalf of a child tier.
|
||||
|
||||
Yields:
|
||||
New OffloadingEvents collected since the last call.
|
||||
"""
|
||||
@@ -392,11 +386,6 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
self.group_sizes: Sequence[int] = group_sizes
|
||||
self.block_indices: Sequence[int] = block_indices
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
def medium() -> str:
|
||||
return "GPU"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CanonicalKVCacheTensor:
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
|
||||
|
||||
@@ -15,8 +13,3 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
"""
|
||||
Spec for loading/storing a KV block to CPU memory.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
def medium() -> str:
|
||||
return "CPU"
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Literal
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
@@ -51,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
store_threshold: int = 1,
|
||||
max_tracker_size: int = 64_000,
|
||||
):
|
||||
self.medium: str = CPULoadStoreSpec.medium()
|
||||
self.medium: str = MEDIUM_CPU
|
||||
self._num_blocks: int = num_blocks
|
||||
self._num_allocated_blocks: int = 0
|
||||
self._free_list: list[int] = []
|
||||
|
||||
@@ -13,6 +13,7 @@ import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
@@ -208,6 +209,10 @@ class SecondaryTierManager(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
"""Take KV events for storage state owned by this tier."""
|
||||
return ()
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used for eviction policy.
|
||||
|
||||
@@ -170,7 +170,6 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self,
|
||||
primary_tier: CPUPrimaryTierOffloadingManager,
|
||||
secondary_tiers: list[SecondaryTierManager] | None = None,
|
||||
enable_events: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the TieringOffloadingManager.
|
||||
@@ -179,14 +178,11 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
primary_tier: The primary tier manager (CPU-based).
|
||||
secondary_tiers: List of secondary tier managers (e.g., Storage,
|
||||
Network). Can be None or empty list.
|
||||
enable_events: Whether to track offloading events
|
||||
"""
|
||||
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
|
||||
self.secondary_tiers = secondary_tiers or []
|
||||
|
||||
self._job_id_counter: int = 0
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
|
||||
# Job tracking: maps job_id to metadata for all in-flight transfers.
|
||||
# JobMetadata.is_promotion distinguishes direction:
|
||||
# True: secondary → primary (promotion)
|
||||
@@ -539,7 +535,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
success: bool = True,
|
||||
):
|
||||
) -> None:
|
||||
"""
|
||||
Mark blocks as done storing from GPU to primary tier.
|
||||
|
||||
@@ -707,16 +703,14 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
@override
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
"""Yield offloading events collected since the last call.
|
||||
"""Yield events owned by the primary and secondary tiers.
|
||||
|
||||
Yields:
|
||||
New OffloadingEvents collected since the last call.
|
||||
New OffloadingEvents collected by each tier since the last call.
|
||||
"""
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
yield from self.primary_tier.take_events()
|
||||
for tier in self.secondary_tiers:
|
||||
yield from tier.take_events()
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
|
||||
@@ -168,7 +168,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
enable_events=self.kv_events_config.enable_kv_cache_events,
|
||||
)
|
||||
if int(self.extra_config.get("store_threshold", 0)) >= 2:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user