From 77cba0259f21dfc6f4f6298f54f3e43c9cc824c3 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Mon, 27 Jul 2026 13:29:57 +0300 Subject: [PATCH] [KV Offloading] Per-request tier filtering with TierFilter/TierMatcher (#48123) Signed-off-by: Ronen Schaffer --- .../unit/offloading_connector/test_events.py | 24 ++- .../offloading_connector/test_scheduler.py | 7 +- tests/v1/kv_offload/cpu/test_manager.py | 4 +- tests/v1/kv_offload/tiering/test_fs_tier.py | 7 +- tests/v1/kv_offload/tiering/test_obj_tier.py | 4 +- .../tiering/test_tiering_offloading.py | 139 +++++++++++++++++- vllm/distributed/kv_events.py | 3 +- .../kv_connector/v1/offloading/events.py | 22 ++- .../kv_connector/v1/offloading/scheduler.py | 61 +++++++- vllm/v1/kv_offload/base.py | 55 +++++-- vllm/v1/kv_offload/cpu/manager.py | 4 +- vllm/v1/kv_offload/tiering/base.py | 8 +- vllm/v1/kv_offload/tiering/example/manager.py | 5 +- vllm/v1/kv_offload/tiering/fs/manager.py | 4 +- vllm/v1/kv_offload/tiering/manager.py | 2 + vllm/v1/kv_offload/tiering/obj/manager.py | 4 +- 16 files changed, 301 insertions(+), 52 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index 5c172454b5..a6bab14e06 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -9,8 +9,6 @@ from tests.v1.kv_connector.unit.utils import create_vllm_config from vllm.config import KVEventsConfig, KVTransferConfig from vllm.distributed.kv_events import ( MEDIUM_CPU, - MEDIUM_FS, - MEDIUM_OBJ, BlockRemoved, BlockStored, ) @@ -33,6 +31,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.kv_offload.base import ( Locality, + Medium, OffloadingEvent, OffloadingKVEventsConfig, OffloadKey, @@ -40,7 +39,7 @@ from vllm.v1.kv_offload.base import ( ) from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec -_CPU_MEDIUM = MEDIUM_CPU +_CPU_MEDIUM = Medium.CPU _FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec( kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value, kv_cache_spec_sliding_window=None, @@ -135,7 +134,7 @@ def _record_lookup_chunks( def _stored_event( keys: list[OffloadKey], - medium: str = _CPU_MEDIUM, + medium: Medium = _CPU_MEDIUM, locality: Locality | None = None, ) -> OffloadingEvent: return OffloadingEvent( @@ -148,7 +147,7 @@ def _stored_event( def _removed_event( keys: list[OffloadKey], - medium: str = _CPU_MEDIUM, + medium: Medium = _CPU_MEDIUM, locality: Locality | None = None, ) -> OffloadingEvent: return OffloadingEvent( @@ -181,7 +180,7 @@ def test_take_events_forwards_locality_to_rich_store(): events = list( tracker.take_events( - [_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)] + [_stored_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)] ) ) @@ -199,7 +198,7 @@ def test_take_events_forwards_locality_to_placeholder_store(): events = list( tracker.take_events( - [_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)] + [_stored_event([key], locality=Locality.REMOTE, medium=Medium.STORAGE)] ) ) @@ -216,7 +215,7 @@ def test_take_events_forwards_locality_to_remove(): events = list( tracker.take_events( - [_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)] + [_removed_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)] ) ) @@ -240,7 +239,7 @@ def test_take_events_publishes_routable_block_stored(): for i, event in enumerate(batch1): assert isinstance(event, BlockStored) - assert event.medium == _CPU_MEDIUM + assert event.medium == _CPU_MEDIUM.value assert event.block_hashes == [_wire_hash(_hash(i))] assert event.block_size == block_size assert event.token_ids == list( @@ -324,7 +323,7 @@ def test_lookup_promotion_factor_gt_1_store_and_remove(): assert len(removed) == 1 assert isinstance(removed[0], BlockRemoved) assert removed[0].block_hashes == expected_hashes - assert removed[0].medium == _CPU_MEDIUM + assert removed[0].medium == _CPU_MEDIUM.value assert removed[0].group_idx == 0 assert not tracker._pending_event_metadata @@ -444,12 +443,11 @@ def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit(): ] -@pytest.mark.parametrize("medium", [MEDIUM_FS, MEDIUM_OBJ]) -def test_secondary_stored_event_does_not_mutate_cpu_metadata(medium: str): +def test_secondary_stored_event_does_not_mutate_cpu_metadata(): tracker, _, _, key = _lookup_chunk() expected_metadata = dict(tracker._pending_event_metadata) - stored = list(tracker.take_events([_stored_event([key], medium)])) + stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)])) assert stored[0].token_ids == [1, 2, 3, 4] assert tracker._pending_event_metadata == expected_metadata diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index d1463aebb4..661078041f 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.kv_offload.base import ( LookupResult, + Medium, OffloadingEvent, OffloadingManager, OffloadPolicy, @@ -299,7 +300,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal( assert not tracker._pending_event_metadata - raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False)) + raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False)) events = list(runner.connector_scheduler.take_events()) assert len(events) == 1 assert isinstance(events[0], BlockStored) @@ -320,7 +321,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal( ) assert key in tracker._pending_event_metadata - raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=True)) + raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=True)) [event] = runner.connector_scheduler.take_events() assert isinstance(event, BlockRemoved) assert event.medium == MEDIUM_CPU @@ -355,7 +356,7 @@ def test_promotion_hit_precedes_stored_event_translation( raw_events: list[OffloadingEvent] = [] def lookup(key, req_context): - raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False)) + raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False)) return LookupResult.HIT def take_raw_events(): diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 97ed5168f7..6520a93fd1 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -6,10 +6,10 @@ from dataclasses import dataclass import numpy as np import pytest -from vllm.distributed.kv_events import MEDIUM_CPU from vllm.v1.kv_offload.base import ( LoadStoreSpec, LookupResult, + Medium, OffloadingEvent, OffloadKey, PrepareStoreOutput, @@ -115,7 +115,7 @@ def verify_events( stores: list[set[OffloadKey]] = [] evictions: list[set[OffloadKey]] = [] for event in events: - assert event.medium == MEDIUM_CPU + assert event.medium == Medium.CPU if event.removed: evictions.append(set(event.keys)) else: diff --git a/tests/v1/kv_offload/tiering/test_fs_tier.py b/tests/v1/kv_offload/tiering/test_fs_tier.py index 589d7dda77..68b5512a3e 100644 --- a/tests/v1/kv_offload/tiering/test_fs_tier.py +++ b/tests/v1/kv_offload/tiering/test_fs_tier.py @@ -18,10 +18,10 @@ import numpy as np import pytest import torch -from vllm.distributed.kv_events import MEDIUM_FS from vllm.v1.kv_offload.base import ( Locality, LookupResult, + Medium, OffloadingEvent, OffloadingKVEventsConfig, OffloadKey, @@ -514,8 +514,7 @@ def test_successful_store_emits_stored_event(fs_tier_with_events): events = list(tier.take_events()) assert len(events) == 1 assert events[0].keys == keys - # Literal medium pins the wire contract, not just the constant choice. - assert events[0].medium == "FS" + assert events[0].medium == Medium.STORAGE assert events[0].locality is Locality.LOCAL assert not events[0].removed # take_events drains the buffer. @@ -685,7 +684,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path): events.extend(manager.take_events()) time.sleep(0.01) - fs_events = [e for e in events if e.medium == MEDIUM_FS] + fs_events = [e for e in events if e.medium == Medium.STORAGE] assert len(fs_events) == 1 assert set(fs_events[0].keys) == set(keys) assert not fs_events[0].removed diff --git a/tests/v1/kv_offload/tiering/test_obj_tier.py b/tests/v1/kv_offload/tiering/test_obj_tier.py index 7500df8b4b..fc30e1437a 100644 --- a/tests/v1/kv_offload/tiering/test_obj_tier.py +++ b/tests/v1/kv_offload/tiering/test_obj_tier.py @@ -21,6 +21,7 @@ import torch from vllm.v1.kv_offload.base import ( Locality, LookupResult, + Medium, OffloadingKVEventsConfig, OffloadKey, ReqContext, @@ -473,8 +474,7 @@ class TestObjTierKVEvents: events = list(self.tier.take_events()) assert len(events) == 1 assert events[0].keys == keys - # Literal medium pins the wire contract, not just the constant choice. - assert events[0].medium == "OBJ" + assert events[0].medium == Medium.STORAGE assert events[0].locality is Locality.REMOTE assert not events[0].removed # take_events drains the buffer. diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index cf546b304c..b19a270efb 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -20,8 +20,13 @@ import torch from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, ) +from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( + _parse_tier_filter, +) from vllm.v1.kv_offload.base import ( + Locality, LookupResult, + Medium, OffloadingCounterMetadata, OffloadingEvent, OffloadKey, @@ -29,6 +34,8 @@ from vllm.v1.kv_offload.base import ( ReqContext, RequestOffloadingContext, ScheduleEndContext, + TierFilter, + TierMatcher, make_offload_key, ) from vllm.v1.kv_offload.tiering.base import ( @@ -245,9 +252,9 @@ class TestTieringOffloadingManager: self.manager.on_new_request(req_context) def test_take_events_aggregates_tier_owned_events(self, manager_setup): - primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False) - secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False) - secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True) + primary_event = OffloadingEvent(to_keys([1]), Medium.CPU, removed=False) + secondary_event1 = OffloadingEvent(to_keys([2]), Medium.STORAGE, removed=False) + secondary_event2 = OffloadingEvent(to_keys([3]), Medium.STORAGE, removed=True) self.primary_tier.take_events = MagicMock(return_value=[primary_event]) self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1]) @@ -973,6 +980,56 @@ class TestTieringOffloadingManager: self.secondary_tier2.drain_jobs.assert_called_once() assert self.manager._transfer_jobs == {} + @pytest.mark.parametrize( + "load_tier_filter", + [ + TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)), + TierFilter(matchers=()), + ], + ids=["non_matching_medium", "empty_no_load"], + ) + def test_tier_filter_skips_filtered_secondary( + self, manager_setup, load_tier_filter + ): + """Filter excluding secondary medium returns MISS from secondaries + even when they hold the block; primary is unaffected.""" + blocks = to_keys(range(2)) + # Put one block in primary, one only in secondary + self._start_request() + self.manager.prepare_store(blocks[:1], _CTX) + self.manager.complete_store(blocks[:1], _CTX, success=True) + self.secondary_tier1.blocks[blocks[1]] = True + + # Secondaries have medium=CPU, so load_tier_filter skips them. + self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup) + + ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter) + assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT + assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS + self.secondary_tier1.lookup.assert_not_called() + + @pytest.mark.parametrize( + "load_tier_filter", + [ + TierFilter.ALL, + TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)), + TierFilter(matchers=(TierMatcher(),)), + ], + ids=["all", "explicit_cpu", "unconstrained_matcher"], + ) + def test_tier_filter_allows_matching_secondary( + self, manager_setup, load_tier_filter + ): + """Filter that matches the secondary's medium allows lookup.""" + blocks = to_keys(range(1)) + self.secondary_tier1.blocks[blocks[0]] = True + + self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup) + + ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter) + assert self.manager.lookup(blocks[0], ctx) is LookupResult.RETRY + self.secondary_tier1.lookup.assert_called() + class TestTieringOffloadingWithoutSecondaryTiers: """Test TieringOffloadingManager with no secondary tiers (backward compat).""" @@ -999,5 +1056,81 @@ class TestTieringOffloadingWithoutSecondaryTiers: assert count_hits(manager, blocks) == 3 +@pytest.mark.parametrize( + "raw,expected", + [ + ( + [{"medium": "storage"}], + TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)), + ), + ( + [{"medium": "CPU"}], + TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)), + ), + ( + [{}], + TierFilter(matchers=(TierMatcher(),)), + ), + ( + [{"medium": "storage", "locality": "local"}], + TierFilter( + matchers=(TierMatcher(medium=Medium.STORAGE, locality=Locality.LOCAL),) + ), + ), + ( + [{"medium": "cpu"}, {"medium": "storage"}], + TierFilter( + matchers=( + TierMatcher(medium=Medium.CPU), + TierMatcher(medium=Medium.STORAGE), + ) + ), + ), + ( + [], + TierFilter(matchers=()), + ), + ], + ids=[ + "medium_storage", + "medium_cpu_uppercase", + "unconstrained", + "with_locality", + "multiple_matchers", + "empty_list_deny_all", + ], +) +def test_parse_tier_filter_valid(raw, expected): + assert _parse_tier_filter(raw) == expected + + +@pytest.mark.parametrize( + "raw", + [ + "not a list", + [{"medium": "unknown"}], + [{"locality": "nowhere"}], + ], + ids=["non_list", "invalid_medium", "invalid_locality"], +) +def test_parse_tier_filter_invalid_returns_all(raw): + assert _parse_tier_filter(raw) is TierFilter.ALL + + +def test_parse_tier_filter_skips_bad_entries(): + result = _parse_tier_filter( + [ + {"medium": "storage"}, + "not a dict", + {"medium": "bogus"}, + {"medium": "cpu"}, + ] + ) + assert result.matchers == ( + TierMatcher(medium=Medium.STORAGE), + TierMatcher(medium=Medium.CPU), + ) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index be7e7363fb..88cb3541f2 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -44,8 +44,7 @@ class KVCacheEvent( MEDIUM_GPU = "GPU" MEDIUM_CPU = "CPU" -MEDIUM_FS = "FS" -MEDIUM_OBJ = "OBJ" +MEDIUM_STORAGE = "STORAGE" class BlockStored(KVCacheEvent): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 3c5a9956ca..67ada00e03 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -19,7 +19,13 @@ from collections.abc import Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, NamedTuple -from vllm.distributed.kv_events import BlockRemoved, BlockStored, KVCacheEvent +from vllm.distributed.kv_events import ( + MEDIUM_CPU, + MEDIUM_STORAGE, + BlockRemoved, + BlockStored, + KVCacheEvent, +) from vllm.logger import init_logger from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash from vllm.v1.kv_cache_interface import ( @@ -28,6 +34,7 @@ from vllm.v1.kv_cache_interface import ( get_kv_cache_spec_sliding_window, ) from vllm.v1.kv_offload.base import ( + Medium, OffloadingEvent, OffloadingKVEventsConfig, OffloadKey, @@ -43,6 +50,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_MEDIUM_TO_EVENT_STR: dict[Medium, str] = { + Medium.CPU: MEDIUM_CPU, + Medium.STORAGE: MEDIUM_STORAGE, +} + class OffloadingEventGroupSpec(NamedTuple): kv_cache_spec_kind: str | None @@ -218,7 +230,7 @@ class OffloadingEventsTracker: def _placeholder_stored( self, key: OffloadKey, - medium: str, + medium: Medium, locality: str | None, ) -> BlockStored: return BlockStored( @@ -229,7 +241,7 @@ class OffloadingEventsTracker: token_ids=[], lora_id=None, block_size=0, - medium=medium, + medium=_MEDIUM_TO_EVENT_STR[medium], lora_name=None, group_idx=get_offload_group_idx(key), locality=locality, @@ -267,7 +279,7 @@ class OffloadingEventsTracker: token_ids=list(meta.token_ids), block_size=meta.block_size, lora_id=meta.lora_id, - medium=event.medium, + medium=_MEDIUM_TO_EVENT_STR[event.medium], lora_name=meta.lora_name, extra_keys=( list(meta.extra_keys) if meta.extra_keys is not None else None @@ -308,7 +320,7 @@ class OffloadingEventsTracker: for group_idx, hashes in by_group.items(): yield BlockRemoved( block_hashes=hashes, - medium=event.medium, + medium=_MEDIUM_TO_EVENT_STR[event.medium], group_idx=group_idx, locality=locality, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index f809dca5ba..fecce3d28f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -39,7 +39,9 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.kv_offload.base import ( GPULoadStoreSpec, + Locality, LookupResult, + Medium, OffloadingManager, OffloadingSpec, OffloadKey, @@ -47,6 +49,8 @@ from vllm.v1.kv_offload.base import ( ReqContext, RequestOffloadingContext, ScheduleEndContext, + TierFilter, + TierMatcher, make_offload_key, ) from vllm.v1.outputs import KVConnectorOutput @@ -54,6 +58,10 @@ from vllm.v1.request import Request, RequestStatus logger = init_logger(__name__) +KV_LOAD_TIERS_KEY = "kv_load_tiers" +MATCHER_MEDIUM_KEY = "medium" +MATCHER_LOCALITY_KEY = "locality" + @dataclass(slots=True) class TransferJobStatus: @@ -376,10 +384,61 @@ class RequestOffloadState: ) +def _parse_tier_filter(raw: Any) -> TierFilter: + """Parse raw kv_transfer_params tier matchers into a TierFilter.""" + if not isinstance(raw, list): + logger.warning( + "_parse_tier_filter: expected list, got %s; ignoring", + type(raw).__name__, + ) + return TierFilter.ALL + matchers: list[TierMatcher] = [] + for entry in raw: + if not isinstance(entry, dict): + logger.warning("_parse_tier_filter: entry is not a dict; skipping") + continue + medium: Medium | None = None + locality: Locality | None = None + raw_medium = entry.get(MATCHER_MEDIUM_KEY) + if raw_medium is not None: + try: + medium = Medium(raw_medium.upper()) + except (ValueError, AttributeError): + logger.warning( + "_parse_tier_filter: unknown medium %r; skipping entry", + raw_medium, + ) + continue + raw_locality = entry.get(MATCHER_LOCALITY_KEY) + if raw_locality is not None: + try: + locality = Locality(raw_locality.upper()) + except (ValueError, AttributeError): + logger.warning( + "_parse_tier_filter: unknown locality %r; skipping entry", + raw_locality, + ) + continue + matchers.append(TierMatcher(medium=medium, locality=locality)) + if not matchers: + if not raw: # input was [] — user explicitly wants nothing + return TierFilter(matchers=()) + # all entries were invalid — fall back to ALL + return TierFilter.ALL + return TierFilter(matchers=tuple(matchers)) + + def _create_req_context(req: Request) -> ReqContext: + params = req.kv_transfer_params + load_filter = TierFilter.ALL + if params: + raw = params.get(KV_LOAD_TIERS_KEY) + if raw is not None: + load_filter = _parse_tier_filter(raw) return ReqContext( req_id=req.request_id, - kv_transfer_params=req.kv_transfer_params, + kv_transfer_params=params, + load_tier_filter=load_filter, ) diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index cb90599cf8..6ff09f23d3 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -8,7 +8,7 @@ from abc import ABC, abstractmethod from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass, field from enum import Enum, auto -from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar import numpy as np import torch @@ -48,10 +48,54 @@ def get_offload_group_idx(key: OffloadKey) -> int: _T = TypeVar("_T") +class Medium(Enum): + """Storage medium of an offloading tier.""" + + CPU = "CPU" + STORAGE = "STORAGE" + + +class Locality(Enum): + """Locality of a tier's storage relative to the publishing instance.""" + + LOCAL = "LOCAL" + REMOTE = "REMOTE" + + +class TierMatcher(NamedTuple): + medium: Medium | None = None + locality: Locality | None = None + + def matches(self, medium: Medium | None, locality: Locality | None) -> bool: + medium_matches = self.medium is None or medium is None or self.medium == medium + locality_matches = ( + self.locality is None or locality is None or self.locality == locality + ) + return medium_matches and locality_matches + + +@dataclass(frozen=True) +class TierFilter: + """Per-request filter controlling which tiers participate.""" + + matchers: tuple[TierMatcher, ...] = () + + ALL: ClassVar["TierFilter"] + + def allows(self, medium: Medium | None, locality: Locality | None) -> bool: + if self is TierFilter.ALL: + return True + return any(m.matches(medium, locality) for m in self.matchers) + + +TierFilter.ALL = TierFilter(matchers=(TierMatcher(),)) + + @dataclass class ReqContext: req_id: str kv_transfer_params: dict[str, Any] | None = None + load_tier_filter: TierFilter = TierFilter.ALL # Per-request scratch space keyed by value type, so a tier can parse # kv_transfer_params once (in on_new_request) and read the result back # on later calls for the same request. @@ -110,17 +154,10 @@ class PrepareStoreOutput: evicted_keys: list[OffloadKey] -class Locality(Enum): - """Locality of a tier's storage relative to the publishing instance.""" - - LOCAL = "LOCAL" - REMOTE = "REMOTE" - - @dataclass class OffloadingEvent: keys: list[OffloadKey] - medium: str + medium: Medium # True if blocks are removed, False if stored removed: bool locality: Locality | None = None diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 297c94aabb..c2ec4170b8 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -6,13 +6,13 @@ from typing import Literal from typing_extensions import override -from vllm.distributed.kv_events import MEDIUM_CPU from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, ) from vllm.v1.kv_offload.base import ( LoadStoreSpec, LookupResult, + Medium, OffloadingEvent, OffloadingManager, OffloadKey, @@ -52,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager): store_threshold: int = 1, max_tracker_size: int = 64_000, ): - self.medium: str = MEDIUM_CPU + self.medium: Medium = Medium.CPU self._num_blocks: int = num_blocks self._num_allocated_blocks: int = 0 self._free_list: list[int] = [] diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index f561417196..3577b4fa0e 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -7,12 +7,14 @@ Abstract interfaces and data types for the secondary tiering layer. from abc import ABC, abstractmethod from collections.abc import Collection, Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import numpy as np from vllm.v1.kv_offload.base import ( + Locality, LookupResult, + Medium, OffloadingEvent, OffloadingMetricMetadata, OffloadKey, @@ -27,6 +29,7 @@ if TYPE_CHECKING: ) from vllm.v1.kv_offload.base import OffloadingSpec + # Type alias for job IDs used in async transfer tracking JobId = int @@ -108,6 +111,8 @@ class SecondaryTierManager(ABC): async jobs; get_finished_jobs() polls for completion. """ + medium: ClassVar[Medium | None] = None + def __init__( self, offloading_spec: "OffloadingSpec", @@ -124,6 +129,7 @@ class SecondaryTierManager(ABC): self._offloading_spec = offloading_spec self._primary_kv_view: memoryview = primary_kv_view self.tier_type = tier_type + self.locality: Locality | None = None @abstractmethod def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: diff --git a/vllm/v1/kv_offload/tiering/example/manager.py b/vllm/v1/kv_offload/tiering/example/manager.py index a9e4e4f689..1a78a3feb7 100644 --- a/vllm/v1/kv_offload/tiering/example/manager.py +++ b/vllm/v1/kv_offload/tiering/example/manager.py @@ -11,12 +11,13 @@ TieringOffloadingManager without requiring actual storage or network backends. import logging from collections.abc import Iterable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from typing_extensions import override from vllm.v1.kv_offload.base import ( LookupResult, + Medium, OffloadKey, ReqContext, RequestOffloadingContext, @@ -42,6 +43,8 @@ class ExampleSecondaryTierManager(SecondaryTierManager): - Completes transfers immediately (synchronous) """ + medium: ClassVar[Medium] = Medium.CPU + def __init__( self, offloading_spec: "OffloadingSpec", diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index 4fde24d760..4fe200629f 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -30,11 +30,11 @@ except ImportError: from typing_extensions import override -from vllm.distributed.kv_events import MEDIUM_FS from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( Locality, LookupResult, + Medium, OffloadingEvent, OffloadKey, ReqContext, @@ -100,7 +100,7 @@ class FileSystemTierManager(SecondaryTierManager): content. """ - medium: ClassVar[str] = MEDIUM_FS + medium: ClassVar[Medium] = Medium.STORAGE def __init__( self, diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 2f5b52e222..c673809613 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -324,6 +324,8 @@ class TieringOffloadingManager(OffloadingManager): for tier in self.secondary_tiers: if tier is exclude_tier: continue + if not req_context.load_tier_filter.allows(tier.medium, tier.locality): + continue result = tier.lookup(key, req_context) if result is LookupResult.HIT: promoted = self._initiate_promotion(tier, key, req_context) diff --git a/vllm/v1/kv_offload/tiering/obj/manager.py b/vllm/v1/kv_offload/tiering/obj/manager.py index f4af8c44c5..c7e0d4c4be 100644 --- a/vllm/v1/kv_offload/tiering/obj/manager.py +++ b/vllm/v1/kv_offload/tiering/obj/manager.py @@ -7,13 +7,13 @@ import time from collections.abc import Iterable from typing import TYPE_CHECKING, ClassVar, NamedTuple -from vllm.distributed.kv_events import MEDIUM_OBJ from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent from vllm.distributed.nixl_utils import nixl_agent_config from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( Locality, LookupResult, + Medium, OffloadingEvent, OffloadKey, ReqContext, @@ -98,7 +98,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager): primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``. """ - medium: ClassVar[str] = MEDIUM_OBJ + medium: ClassVar[Medium] = Medium.STORAGE def __init__( self,