[KV Offloading] Per-request tier filtering with TierFilter/TierMatcher (#48123)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-07-27 13:29:57 +03:00
committed by GitHub
parent 30fbd05537
commit 77cba0259f
16 changed files with 301 additions and 52 deletions
@@ -9,8 +9,6 @@ from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import (
MEDIUM_CPU,
MEDIUM_FS,
MEDIUM_OBJ,
BlockRemoved,
BlockStored,
)
@@ -33,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.kv_offload.base import (
Locality,
Medium,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
@@ -40,7 +39,7 @@ from vllm.v1.kv_offload.base import (
)
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
_CPU_MEDIUM = MEDIUM_CPU
_CPU_MEDIUM = Medium.CPU
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
kv_cache_spec_sliding_window=None,
@@ -135,7 +134,7 @@ def _record_lookup_chunks(
def _stored_event(
keys: list[OffloadKey],
medium: str = _CPU_MEDIUM,
medium: Medium = _CPU_MEDIUM,
locality: Locality | None = None,
) -> OffloadingEvent:
return OffloadingEvent(
@@ -148,7 +147,7 @@ def _stored_event(
def _removed_event(
keys: list[OffloadKey],
medium: str = _CPU_MEDIUM,
medium: Medium = _CPU_MEDIUM,
locality: Locality | None = None,
) -> OffloadingEvent:
return OffloadingEvent(
@@ -181,7 +180,7 @@ def test_take_events_forwards_locality_to_rich_store():
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
[_stored_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)]
)
)
@@ -199,7 +198,7 @@ def test_take_events_forwards_locality_to_placeholder_store():
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)]
[_stored_event([key], locality=Locality.REMOTE, medium=Medium.STORAGE)]
)
)
@@ -216,7 +215,7 @@ def test_take_events_forwards_locality_to_remove():
events = list(
tracker.take_events(
[_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
[_removed_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)]
)
)
@@ -240,7 +239,7 @@ def test_take_events_publishes_routable_block_stored():
for i, event in enumerate(batch1):
assert isinstance(event, BlockStored)
assert event.medium == _CPU_MEDIUM
assert event.medium == _CPU_MEDIUM.value
assert event.block_hashes == [_wire_hash(_hash(i))]
assert event.block_size == block_size
assert event.token_ids == list(
@@ -324,7 +323,7 @@ def test_lookup_promotion_factor_gt_1_store_and_remove():
assert len(removed) == 1
assert isinstance(removed[0], BlockRemoved)
assert removed[0].block_hashes == expected_hashes
assert removed[0].medium == _CPU_MEDIUM
assert removed[0].medium == _CPU_MEDIUM.value
assert removed[0].group_idx == 0
assert not tracker._pending_event_metadata
@@ -444,12 +443,11 @@ def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit():
]
@pytest.mark.parametrize("medium", [MEDIUM_FS, MEDIUM_OBJ])
def test_secondary_stored_event_does_not_mutate_cpu_metadata(medium: str):
def test_secondary_stored_event_does_not_mutate_cpu_metadata():
tracker, _, _, key = _lookup_chunk()
expected_metadata = dict(tracker._pending_event_metadata)
stored = list(tracker.take_events([_stored_event([key], medium)]))
stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)]))
assert stored[0].token_ids == [1, 2, 3, 4]
assert tracker._pending_event_metadata == expected_metadata
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.kv_offload.base import (
LookupResult,
Medium,
OffloadingEvent,
OffloadingManager,
OffloadPolicy,
@@ -299,7 +300,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal(
assert not tracker._pending_event_metadata
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False))
events = list(runner.connector_scheduler.take_events())
assert len(events) == 1
assert isinstance(events[0], BlockStored)
@@ -320,7 +321,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal(
)
assert key in tracker._pending_event_metadata
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=True))
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=True))
[event] = runner.connector_scheduler.take_events()
assert isinstance(event, BlockRemoved)
assert event.medium == MEDIUM_CPU
@@ -355,7 +356,7 @@ def test_promotion_hit_precedes_stored_event_translation(
raw_events: list[OffloadingEvent] = []
def lookup(key, req_context):
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False))
return LookupResult.HIT
def take_raw_events():
+2 -2
View File
@@ -6,10 +6,10 @@ from dataclasses import dataclass
import numpy as np
import pytest
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
Medium,
OffloadingEvent,
OffloadKey,
PrepareStoreOutput,
@@ -115,7 +115,7 @@ def verify_events(
stores: list[set[OffloadKey]] = []
evictions: list[set[OffloadKey]] = []
for event in events:
assert event.medium == MEDIUM_CPU
assert event.medium == Medium.CPU
if event.removed:
evictions.append(set(event.keys))
else:
+3 -4
View File
@@ -18,10 +18,10 @@ import numpy as np
import pytest
import torch
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
@@ -514,8 +514,7 @@ def test_successful_store_emits_stored_event(fs_tier_with_events):
events = list(tier.take_events())
assert len(events) == 1
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "FS"
assert events[0].medium == Medium.STORAGE
assert events[0].locality is Locality.LOCAL
assert not events[0].removed
# take_events drains the buffer.
@@ -685,7 +684,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
events.extend(manager.take_events())
time.sleep(0.01)
fs_events = [e for e in events if e.medium == MEDIUM_FS]
fs_events = [e for e in events if e.medium == Medium.STORAGE]
assert len(fs_events) == 1
assert set(fs_events[0].keys) == set(keys)
assert not fs_events[0].removed
+2 -2
View File
@@ -21,6 +21,7 @@ import torch
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingKVEventsConfig,
OffloadKey,
ReqContext,
@@ -473,8 +474,7 @@ class TestObjTierKVEvents:
events = list(self.tier.take_events())
assert len(events) == 1
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "OBJ"
assert events[0].medium == Medium.STORAGE
assert events[0].locality is Locality.REMOTE
assert not events[0].removed
# take_events drains the buffer.
@@ -20,8 +20,13 @@ import torch
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
_parse_tier_filter,
)
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingCounterMetadata,
OffloadingEvent,
OffloadKey,
@@ -29,6 +34,8 @@ from vllm.v1.kv_offload.base import (
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
TierFilter,
TierMatcher,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import (
@@ -245,9 +252,9 @@ class TestTieringOffloadingManager:
self.manager.on_new_request(req_context)
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
primary_event = OffloadingEvent(to_keys([1]), Medium.CPU, removed=False)
secondary_event1 = OffloadingEvent(to_keys([2]), Medium.STORAGE, removed=False)
secondary_event2 = OffloadingEvent(to_keys([3]), Medium.STORAGE, removed=True)
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
@@ -973,6 +980,56 @@ class TestTieringOffloadingManager:
self.secondary_tier2.drain_jobs.assert_called_once()
assert self.manager._transfer_jobs == {}
@pytest.mark.parametrize(
"load_tier_filter",
[
TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)),
TierFilter(matchers=()),
],
ids=["non_matching_medium", "empty_no_load"],
)
def test_tier_filter_skips_filtered_secondary(
self, manager_setup, load_tier_filter
):
"""Filter excluding secondary medium returns MISS from secondaries
even when they hold the block; primary is unaffected."""
blocks = to_keys(range(2))
# Put one block in primary, one only in secondary
self._start_request()
self.manager.prepare_store(blocks[:1], _CTX)
self.manager.complete_store(blocks[:1], _CTX, success=True)
self.secondary_tier1.blocks[blocks[1]] = True
# Secondaries have medium=CPU, so load_tier_filter skips them.
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
self.secondary_tier1.lookup.assert_not_called()
@pytest.mark.parametrize(
"load_tier_filter",
[
TierFilter.ALL,
TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)),
TierFilter(matchers=(TierMatcher(),)),
],
ids=["all", "explicit_cpu", "unconstrained_matcher"],
)
def test_tier_filter_allows_matching_secondary(
self, manager_setup, load_tier_filter
):
"""Filter that matches the secondary's medium allows lookup."""
blocks = to_keys(range(1))
self.secondary_tier1.blocks[blocks[0]] = True
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
assert self.manager.lookup(blocks[0], ctx) is LookupResult.RETRY
self.secondary_tier1.lookup.assert_called()
class TestTieringOffloadingWithoutSecondaryTiers:
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
@@ -999,5 +1056,81 @@ class TestTieringOffloadingWithoutSecondaryTiers:
assert count_hits(manager, blocks) == 3
@pytest.mark.parametrize(
"raw,expected",
[
(
[{"medium": "storage"}],
TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)),
),
(
[{"medium": "CPU"}],
TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)),
),
(
[{}],
TierFilter(matchers=(TierMatcher(),)),
),
(
[{"medium": "storage", "locality": "local"}],
TierFilter(
matchers=(TierMatcher(medium=Medium.STORAGE, locality=Locality.LOCAL),)
),
),
(
[{"medium": "cpu"}, {"medium": "storage"}],
TierFilter(
matchers=(
TierMatcher(medium=Medium.CPU),
TierMatcher(medium=Medium.STORAGE),
)
),
),
(
[],
TierFilter(matchers=()),
),
],
ids=[
"medium_storage",
"medium_cpu_uppercase",
"unconstrained",
"with_locality",
"multiple_matchers",
"empty_list_deny_all",
],
)
def test_parse_tier_filter_valid(raw, expected):
assert _parse_tier_filter(raw) == expected
@pytest.mark.parametrize(
"raw",
[
"not a list",
[{"medium": "unknown"}],
[{"locality": "nowhere"}],
],
ids=["non_list", "invalid_medium", "invalid_locality"],
)
def test_parse_tier_filter_invalid_returns_all(raw):
assert _parse_tier_filter(raw) is TierFilter.ALL
def test_parse_tier_filter_skips_bad_entries():
result = _parse_tier_filter(
[
{"medium": "storage"},
"not a dict",
{"medium": "bogus"},
{"medium": "cpu"},
]
)
assert result.matchers == (
TierMatcher(medium=Medium.STORAGE),
TierMatcher(medium=Medium.CPU),
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+1 -2
View File
@@ -44,8 +44,7 @@ class KVCacheEvent(
MEDIUM_GPU = "GPU"
MEDIUM_CPU = "CPU"
MEDIUM_FS = "FS"
MEDIUM_OBJ = "OBJ"
MEDIUM_STORAGE = "STORAGE"
class BlockStored(KVCacheEvent):
@@ -19,7 +19,13 @@ from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NamedTuple
from vllm.distributed.kv_events import BlockRemoved, BlockStored, KVCacheEvent
from vllm.distributed.kv_events import (
MEDIUM_CPU,
MEDIUM_STORAGE,
BlockRemoved,
BlockStored,
KVCacheEvent,
)
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
@@ -28,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
get_kv_cache_spec_sliding_window,
)
from vllm.v1.kv_offload.base import (
Medium,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
@@ -43,6 +50,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
_MEDIUM_TO_EVENT_STR: dict[Medium, str] = {
Medium.CPU: MEDIUM_CPU,
Medium.STORAGE: MEDIUM_STORAGE,
}
class OffloadingEventGroupSpec(NamedTuple):
kv_cache_spec_kind: str | None
@@ -218,7 +230,7 @@ class OffloadingEventsTracker:
def _placeholder_stored(
self,
key: OffloadKey,
medium: str,
medium: Medium,
locality: str | None,
) -> BlockStored:
return BlockStored(
@@ -229,7 +241,7 @@ class OffloadingEventsTracker:
token_ids=[],
lora_id=None,
block_size=0,
medium=medium,
medium=_MEDIUM_TO_EVENT_STR[medium],
lora_name=None,
group_idx=get_offload_group_idx(key),
locality=locality,
@@ -267,7 +279,7 @@ class OffloadingEventsTracker:
token_ids=list(meta.token_ids),
block_size=meta.block_size,
lora_id=meta.lora_id,
medium=event.medium,
medium=_MEDIUM_TO_EVENT_STR[event.medium],
lora_name=meta.lora_name,
extra_keys=(
list(meta.extra_keys) if meta.extra_keys is not None else None
@@ -308,7 +320,7 @@ class OffloadingEventsTracker:
for group_idx, hashes in by_group.items():
yield BlockRemoved(
block_hashes=hashes,
medium=event.medium,
medium=_MEDIUM_TO_EVENT_STR[event.medium],
group_idx=group_idx,
locality=locality,
)
@@ -39,7 +39,9 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
Locality,
LookupResult,
Medium,
OffloadingManager,
OffloadingSpec,
OffloadKey,
@@ -47,6 +49,8 @@ from vllm.v1.kv_offload.base import (
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
TierFilter,
TierMatcher,
make_offload_key,
)
from vllm.v1.outputs import KVConnectorOutput
@@ -54,6 +58,10 @@ from vllm.v1.request import Request, RequestStatus
logger = init_logger(__name__)
KV_LOAD_TIERS_KEY = "kv_load_tiers"
MATCHER_MEDIUM_KEY = "medium"
MATCHER_LOCALITY_KEY = "locality"
@dataclass(slots=True)
class TransferJobStatus:
@@ -376,10 +384,61 @@ class RequestOffloadState:
)
def _parse_tier_filter(raw: Any) -> TierFilter:
"""Parse raw kv_transfer_params tier matchers into a TierFilter."""
if not isinstance(raw, list):
logger.warning(
"_parse_tier_filter: expected list, got %s; ignoring",
type(raw).__name__,
)
return TierFilter.ALL
matchers: list[TierMatcher] = []
for entry in raw:
if not isinstance(entry, dict):
logger.warning("_parse_tier_filter: entry is not a dict; skipping")
continue
medium: Medium | None = None
locality: Locality | None = None
raw_medium = entry.get(MATCHER_MEDIUM_KEY)
if raw_medium is not None:
try:
medium = Medium(raw_medium.upper())
except (ValueError, AttributeError):
logger.warning(
"_parse_tier_filter: unknown medium %r; skipping entry",
raw_medium,
)
continue
raw_locality = entry.get(MATCHER_LOCALITY_KEY)
if raw_locality is not None:
try:
locality = Locality(raw_locality.upper())
except (ValueError, AttributeError):
logger.warning(
"_parse_tier_filter: unknown locality %r; skipping entry",
raw_locality,
)
continue
matchers.append(TierMatcher(medium=medium, locality=locality))
if not matchers:
if not raw: # input was [] — user explicitly wants nothing
return TierFilter(matchers=())
# all entries were invalid — fall back to ALL
return TierFilter.ALL
return TierFilter(matchers=tuple(matchers))
def _create_req_context(req: Request) -> ReqContext:
params = req.kv_transfer_params
load_filter = TierFilter.ALL
if params:
raw = params.get(KV_LOAD_TIERS_KEY)
if raw is not None:
load_filter = _parse_tier_filter(raw)
return ReqContext(
req_id=req.request_id,
kv_transfer_params=req.kv_transfer_params,
kv_transfer_params=params,
load_tier_filter=load_filter,
)
+46 -9
View File
@@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeVar
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar
import numpy as np
import torch
@@ -48,10 +48,54 @@ def get_offload_group_idx(key: OffloadKey) -> int:
_T = TypeVar("_T")
class Medium(Enum):
"""Storage medium of an offloading tier."""
CPU = "CPU"
STORAGE = "STORAGE"
class Locality(Enum):
"""Locality of a tier's storage relative to the publishing instance."""
LOCAL = "LOCAL"
REMOTE = "REMOTE"
class TierMatcher(NamedTuple):
medium: Medium | None = None
locality: Locality | None = None
def matches(self, medium: Medium | None, locality: Locality | None) -> bool:
medium_matches = self.medium is None or medium is None or self.medium == medium
locality_matches = (
self.locality is None or locality is None or self.locality == locality
)
return medium_matches and locality_matches
@dataclass(frozen=True)
class TierFilter:
"""Per-request filter controlling which tiers participate."""
matchers: tuple[TierMatcher, ...] = ()
ALL: ClassVar["TierFilter"]
def allows(self, medium: Medium | None, locality: Locality | None) -> bool:
if self is TierFilter.ALL:
return True
return any(m.matches(medium, locality) for m in self.matchers)
TierFilter.ALL = TierFilter(matchers=(TierMatcher(),))
@dataclass
class ReqContext:
req_id: str
kv_transfer_params: dict[str, Any] | None = None
load_tier_filter: TierFilter = TierFilter.ALL
# Per-request scratch space keyed by value type, so a tier can parse
# kv_transfer_params once (in on_new_request) and read the result back
# on later calls for the same request.
@@ -110,17 +154,10 @@ class PrepareStoreOutput:
evicted_keys: list[OffloadKey]
class Locality(Enum):
"""Locality of a tier's storage relative to the publishing instance."""
LOCAL = "LOCAL"
REMOTE = "REMOTE"
@dataclass
class OffloadingEvent:
keys: list[OffloadKey]
medium: str
medium: Medium
# True if blocks are removed, False if stored
removed: bool
locality: Locality | None = None
+2 -2
View File
@@ -6,13 +6,13 @@ from typing import Literal
from typing_extensions import override
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
Medium,
OffloadingEvent,
OffloadingManager,
OffloadKey,
@@ -52,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager):
store_threshold: int = 1,
max_tracker_size: int = 64_000,
):
self.medium: str = MEDIUM_CPU
self.medium: Medium = Medium.CPU
self._num_blocks: int = num_blocks
self._num_allocated_blocks: int = 0
self._free_list: list[int] = []
+7 -1
View File
@@ -7,12 +7,14 @@ Abstract interfaces and data types for the secondary tiering layer.
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingEvent,
OffloadingMetricMetadata,
OffloadKey,
@@ -27,6 +29,7 @@ if TYPE_CHECKING:
)
from vllm.v1.kv_offload.base import OffloadingSpec
# Type alias for job IDs used in async transfer tracking
JobId = int
@@ -108,6 +111,8 @@ class SecondaryTierManager(ABC):
async jobs; get_finished_jobs() polls for completion.
"""
medium: ClassVar[Medium | None] = None
def __init__(
self,
offloading_spec: "OffloadingSpec",
@@ -124,6 +129,7 @@ class SecondaryTierManager(ABC):
self._offloading_spec = offloading_spec
self._primary_kv_view: memoryview = primary_kv_view
self.tier_type = tier_type
self.locality: Locality | None = None
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
@@ -11,12 +11,13 @@ TieringOffloadingManager without requiring actual storage or network backends.
import logging
from collections.abc import Iterable
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, ClassVar
from typing_extensions import override
from vllm.v1.kv_offload.base import (
LookupResult,
Medium,
OffloadKey,
ReqContext,
RequestOffloadingContext,
@@ -42,6 +43,8 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
- Completes transfers immediately (synchronous)
"""
medium: ClassVar[Medium] = Medium.CPU
def __init__(
self,
offloading_spec: "OffloadingSpec",
+2 -2
View File
@@ -30,11 +30,11 @@ except ImportError:
from typing_extensions import override
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingEvent,
OffloadKey,
ReqContext,
@@ -100,7 +100,7 @@ class FileSystemTierManager(SecondaryTierManager):
content.
"""
medium: ClassVar[str] = MEDIUM_FS
medium: ClassVar[Medium] = Medium.STORAGE
def __init__(
self,
+2
View File
@@ -324,6 +324,8 @@ class TieringOffloadingManager(OffloadingManager):
for tier in self.secondary_tiers:
if tier is exclude_tier:
continue
if not req_context.load_tier_filter.allows(tier.medium, tier.locality):
continue
result = tier.lookup(key, req_context)
if result is LookupResult.HIT:
promoted = self._initiate_promotion(tier, key, req_context)
+2 -2
View File
@@ -7,13 +7,13 @@ import time
from collections.abc import Iterable
from typing import TYPE_CHECKING, ClassVar, NamedTuple
from vllm.distributed.kv_events import MEDIUM_OBJ
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
from vllm.distributed.nixl_utils import nixl_agent_config
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
Medium,
OffloadingEvent,
OffloadKey,
ReqContext,
@@ -98,7 +98,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``.
"""
medium: ClassVar[str] = MEDIUM_OBJ
medium: ClassVar[Medium] = Medium.STORAGE
def __init__(
self,