[KV Offloading] Per-request tier filtering with TierFilter/TierMatcher (#48123)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -9,8 +9,6 @@ from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_CPU,
|
||||
MEDIUM_FS,
|
||||
MEDIUM_OBJ,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
)
|
||||
@@ -33,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
@@ -40,7 +39,7 @@ from vllm.v1.kv_offload.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
|
||||
_CPU_MEDIUM = MEDIUM_CPU
|
||||
_CPU_MEDIUM = Medium.CPU
|
||||
_FULL_ATTENTION_EVENT_SPEC = OffloadingEventGroupSpec(
|
||||
kv_cache_spec_kind=KVCacheSpecKind.FULL_ATTENTION.value,
|
||||
kv_cache_spec_sliding_window=None,
|
||||
@@ -135,7 +134,7 @@ def _record_lookup_chunks(
|
||||
|
||||
def _stored_event(
|
||||
keys: list[OffloadKey],
|
||||
medium: str = _CPU_MEDIUM,
|
||||
medium: Medium = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
@@ -148,7 +147,7 @@ def _stored_event(
|
||||
|
||||
def _removed_event(
|
||||
keys: list[OffloadKey],
|
||||
medium: str = _CPU_MEDIUM,
|
||||
medium: Medium = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
@@ -181,7 +180,7 @@ def test_take_events_forwards_locality_to_rich_store():
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
|
||||
[_stored_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)]
|
||||
)
|
||||
)
|
||||
|
||||
@@ -199,7 +198,7 @@ def test_take_events_forwards_locality_to_placeholder_store():
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)]
|
||||
[_stored_event([key], locality=Locality.REMOTE, medium=Medium.STORAGE)]
|
||||
)
|
||||
)
|
||||
|
||||
@@ -216,7 +215,7 @@ def test_take_events_forwards_locality_to_remove():
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
|
||||
[_removed_event([key], locality=Locality.LOCAL, medium=Medium.STORAGE)]
|
||||
)
|
||||
)
|
||||
|
||||
@@ -240,7 +239,7 @@ def test_take_events_publishes_routable_block_stored():
|
||||
|
||||
for i, event in enumerate(batch1):
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.medium == _CPU_MEDIUM
|
||||
assert event.medium == _CPU_MEDIUM.value
|
||||
assert event.block_hashes == [_wire_hash(_hash(i))]
|
||||
assert event.block_size == block_size
|
||||
assert event.token_ids == list(
|
||||
@@ -324,7 +323,7 @@ def test_lookup_promotion_factor_gt_1_store_and_remove():
|
||||
assert len(removed) == 1
|
||||
assert isinstance(removed[0], BlockRemoved)
|
||||
assert removed[0].block_hashes == expected_hashes
|
||||
assert removed[0].medium == _CPU_MEDIUM
|
||||
assert removed[0].medium == _CPU_MEDIUM.value
|
||||
assert removed[0].group_idx == 0
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
@@ -444,12 +443,11 @@ def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("medium", [MEDIUM_FS, MEDIUM_OBJ])
|
||||
def test_secondary_stored_event_does_not_mutate_cpu_metadata(medium: str):
|
||||
def test_secondary_stored_event_does_not_mutate_cpu_metadata():
|
||||
tracker, _, _, key = _lookup_chunk()
|
||||
expected_metadata = dict(tracker._pending_event_metadata)
|
||||
|
||||
stored = list(tracker.take_events([_stored_event([key], medium)]))
|
||||
stored = list(tracker.take_events([_stored_event([key], Medium.STORAGE)]))
|
||||
assert stored[0].token_ids == [1, 2, 3, 4]
|
||||
assert tracker._pending_event_metadata == expected_metadata
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadPolicy,
|
||||
@@ -299,7 +300,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal(
|
||||
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False))
|
||||
events = list(runner.connector_scheduler.take_events())
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
@@ -320,7 +321,7 @@ def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal(
|
||||
)
|
||||
assert key in tracker._pending_event_metadata
|
||||
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=True))
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=True))
|
||||
[event] = runner.connector_scheduler.take_events()
|
||||
assert isinstance(event, BlockRemoved)
|
||||
assert event.medium == MEDIUM_CPU
|
||||
@@ -355,7 +356,7 @@ def test_promotion_hit_precedes_stored_event_translation(
|
||||
raw_events: list[OffloadingEvent] = []
|
||||
|
||||
def lookup(key, req_context):
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=Medium.CPU, removed=False))
|
||||
return LookupResult.HIT
|
||||
|
||||
def take_raw_events():
|
||||
|
||||
@@ -6,10 +6,10 @@ from dataclasses import dataclass
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
@@ -115,7 +115,7 @@ def verify_events(
|
||||
stores: list[set[OffloadKey]] = []
|
||||
evictions: list[set[OffloadKey]] = []
|
||||
for event in events:
|
||||
assert event.medium == MEDIUM_CPU
|
||||
assert event.medium == Medium.CPU
|
||||
if event.removed:
|
||||
evictions.append(set(event.keys))
|
||||
else:
|
||||
|
||||
@@ -18,10 +18,10 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
@@ -514,8 +514,7 @@ def test_successful_store_emits_stored_event(fs_tier_with_events):
|
||||
events = list(tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "FS"
|
||||
assert events[0].medium == Medium.STORAGE
|
||||
assert events[0].locality is Locality.LOCAL
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
@@ -685,7 +684,7 @@ def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
events.extend(manager.take_events())
|
||||
time.sleep(0.01)
|
||||
|
||||
fs_events = [e for e in events if e.medium == MEDIUM_FS]
|
||||
fs_events = [e for e in events if e.medium == Medium.STORAGE]
|
||||
assert len(fs_events) == 1
|
||||
assert set(fs_events[0].keys) == set(keys)
|
||||
assert not fs_events[0].removed
|
||||
|
||||
@@ -21,6 +21,7 @@ import torch
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
@@ -473,8 +474,7 @@ class TestObjTierKVEvents:
|
||||
events = list(self.tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "OBJ"
|
||||
assert events[0].medium == Medium.STORAGE
|
||||
assert events[0].locality is Locality.REMOTE
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
|
||||
@@ -20,8 +20,13 @@ import torch
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
_parse_tier_filter,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
@@ -29,6 +34,8 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
TierFilter,
|
||||
TierMatcher,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
@@ -245,9 +252,9 @@ class TestTieringOffloadingManager:
|
||||
self.manager.on_new_request(req_context)
|
||||
|
||||
def test_take_events_aggregates_tier_owned_events(self, manager_setup):
|
||||
primary_event = OffloadingEvent(to_keys([1]), "CPU", removed=False)
|
||||
secondary_event1 = OffloadingEvent(to_keys([2]), "tier-1", removed=False)
|
||||
secondary_event2 = OffloadingEvent(to_keys([3]), "tier-2", removed=True)
|
||||
primary_event = OffloadingEvent(to_keys([1]), Medium.CPU, removed=False)
|
||||
secondary_event1 = OffloadingEvent(to_keys([2]), Medium.STORAGE, removed=False)
|
||||
secondary_event2 = OffloadingEvent(to_keys([3]), Medium.STORAGE, removed=True)
|
||||
|
||||
self.primary_tier.take_events = MagicMock(return_value=[primary_event])
|
||||
self.secondary_tier1.take_events = MagicMock(return_value=[secondary_event1])
|
||||
@@ -973,6 +980,56 @@ class TestTieringOffloadingManager:
|
||||
self.secondary_tier2.drain_jobs.assert_called_once()
|
||||
assert self.manager._transfer_jobs == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_tier_filter",
|
||||
[
|
||||
TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)),
|
||||
TierFilter(matchers=()),
|
||||
],
|
||||
ids=["non_matching_medium", "empty_no_load"],
|
||||
)
|
||||
def test_tier_filter_skips_filtered_secondary(
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter excluding secondary medium returns MISS from secondaries
|
||||
even when they hold the block; primary is unaffected."""
|
||||
blocks = to_keys(range(2))
|
||||
# Put one block in primary, one only in secondary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:1], _CTX)
|
||||
self.manager.complete_store(blocks[:1], _CTX, success=True)
|
||||
self.secondary_tier1.blocks[blocks[1]] = True
|
||||
|
||||
# Secondaries have medium=CPU, so load_tier_filter skips them.
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r1", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.HIT
|
||||
assert self.manager.lookup(blocks[1], ctx) is LookupResult.MISS
|
||||
self.secondary_tier1.lookup.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_tier_filter",
|
||||
[
|
||||
TierFilter.ALL,
|
||||
TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)),
|
||||
TierFilter(matchers=(TierMatcher(),)),
|
||||
],
|
||||
ids=["all", "explicit_cpu", "unconstrained_matcher"],
|
||||
)
|
||||
def test_tier_filter_allows_matching_secondary(
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter that matches the secondary's medium allows lookup."""
|
||||
blocks = to_keys(range(1))
|
||||
self.secondary_tier1.blocks[blocks[0]] = True
|
||||
|
||||
self.secondary_tier1.lookup = MagicMock(wraps=self.secondary_tier1.lookup)
|
||||
|
||||
ctx = ReqContext(req_id="r2", load_tier_filter=load_tier_filter)
|
||||
assert self.manager.lookup(blocks[0], ctx) is LookupResult.RETRY
|
||||
self.secondary_tier1.lookup.assert_called()
|
||||
|
||||
|
||||
class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
|
||||
@@ -999,5 +1056,81 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
assert count_hits(manager, blocks) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw,expected",
|
||||
[
|
||||
(
|
||||
[{"medium": "storage"}],
|
||||
TierFilter(matchers=(TierMatcher(medium=Medium.STORAGE),)),
|
||||
),
|
||||
(
|
||||
[{"medium": "CPU"}],
|
||||
TierFilter(matchers=(TierMatcher(medium=Medium.CPU),)),
|
||||
),
|
||||
(
|
||||
[{}],
|
||||
TierFilter(matchers=(TierMatcher(),)),
|
||||
),
|
||||
(
|
||||
[{"medium": "storage", "locality": "local"}],
|
||||
TierFilter(
|
||||
matchers=(TierMatcher(medium=Medium.STORAGE, locality=Locality.LOCAL),)
|
||||
),
|
||||
),
|
||||
(
|
||||
[{"medium": "cpu"}, {"medium": "storage"}],
|
||||
TierFilter(
|
||||
matchers=(
|
||||
TierMatcher(medium=Medium.CPU),
|
||||
TierMatcher(medium=Medium.STORAGE),
|
||||
)
|
||||
),
|
||||
),
|
||||
(
|
||||
[],
|
||||
TierFilter(matchers=()),
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"medium_storage",
|
||||
"medium_cpu_uppercase",
|
||||
"unconstrained",
|
||||
"with_locality",
|
||||
"multiple_matchers",
|
||||
"empty_list_deny_all",
|
||||
],
|
||||
)
|
||||
def test_parse_tier_filter_valid(raw, expected):
|
||||
assert _parse_tier_filter(raw) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"not a list",
|
||||
[{"medium": "unknown"}],
|
||||
[{"locality": "nowhere"}],
|
||||
],
|
||||
ids=["non_list", "invalid_medium", "invalid_locality"],
|
||||
)
|
||||
def test_parse_tier_filter_invalid_returns_all(raw):
|
||||
assert _parse_tier_filter(raw) is TierFilter.ALL
|
||||
|
||||
|
||||
def test_parse_tier_filter_skips_bad_entries():
|
||||
result = _parse_tier_filter(
|
||||
[
|
||||
{"medium": "storage"},
|
||||
"not a dict",
|
||||
{"medium": "bogus"},
|
||||
{"medium": "cpu"},
|
||||
]
|
||||
)
|
||||
assert result.matchers == (
|
||||
TierMatcher(medium=Medium.STORAGE),
|
||||
TierMatcher(medium=Medium.CPU),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -44,8 +44,7 @@ class KVCacheEvent(
|
||||
|
||||
MEDIUM_GPU = "GPU"
|
||||
MEDIUM_CPU = "CPU"
|
||||
MEDIUM_FS = "FS"
|
||||
MEDIUM_OBJ = "OBJ"
|
||||
MEDIUM_STORAGE = "STORAGE"
|
||||
|
||||
|
||||
class BlockStored(KVCacheEvent):
|
||||
|
||||
@@ -19,7 +19,13 @@ from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored, KVCacheEvent
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_CPU,
|
||||
MEDIUM_STORAGE,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -28,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
get_kv_cache_spec_sliding_window,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
@@ -43,6 +50,11 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MEDIUM_TO_EVENT_STR: dict[Medium, str] = {
|
||||
Medium.CPU: MEDIUM_CPU,
|
||||
Medium.STORAGE: MEDIUM_STORAGE,
|
||||
}
|
||||
|
||||
|
||||
class OffloadingEventGroupSpec(NamedTuple):
|
||||
kv_cache_spec_kind: str | None
|
||||
@@ -218,7 +230,7 @@ class OffloadingEventsTracker:
|
||||
def _placeholder_stored(
|
||||
self,
|
||||
key: OffloadKey,
|
||||
medium: str,
|
||||
medium: Medium,
|
||||
locality: str | None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
@@ -229,7 +241,7 @@ class OffloadingEventsTracker:
|
||||
token_ids=[],
|
||||
lora_id=None,
|
||||
block_size=0,
|
||||
medium=medium,
|
||||
medium=_MEDIUM_TO_EVENT_STR[medium],
|
||||
lora_name=None,
|
||||
group_idx=get_offload_group_idx(key),
|
||||
locality=locality,
|
||||
@@ -267,7 +279,7 @@ class OffloadingEventsTracker:
|
||||
token_ids=list(meta.token_ids),
|
||||
block_size=meta.block_size,
|
||||
lora_id=meta.lora_id,
|
||||
medium=event.medium,
|
||||
medium=_MEDIUM_TO_EVENT_STR[event.medium],
|
||||
lora_name=meta.lora_name,
|
||||
extra_keys=(
|
||||
list(meta.extra_keys) if meta.extra_keys is not None else None
|
||||
@@ -308,7 +320,7 @@ class OffloadingEventsTracker:
|
||||
for group_idx, hashes in by_group.items():
|
||||
yield BlockRemoved(
|
||||
block_hashes=hashes,
|
||||
medium=event.medium,
|
||||
medium=_MEDIUM_TO_EVENT_STR[event.medium],
|
||||
group_idx=group_idx,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
@@ -39,7 +39,9 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadKey,
|
||||
@@ -47,6 +49,8 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
TierFilter,
|
||||
TierMatcher,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -54,6 +58,10 @@ from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
KV_LOAD_TIERS_KEY = "kv_load_tiers"
|
||||
MATCHER_MEDIUM_KEY = "medium"
|
||||
MATCHER_LOCALITY_KEY = "locality"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TransferJobStatus:
|
||||
@@ -376,10 +384,61 @@ class RequestOffloadState:
|
||||
)
|
||||
|
||||
|
||||
def _parse_tier_filter(raw: Any) -> TierFilter:
|
||||
"""Parse raw kv_transfer_params tier matchers into a TierFilter."""
|
||||
if not isinstance(raw, list):
|
||||
logger.warning(
|
||||
"_parse_tier_filter: expected list, got %s; ignoring",
|
||||
type(raw).__name__,
|
||||
)
|
||||
return TierFilter.ALL
|
||||
matchers: list[TierMatcher] = []
|
||||
for entry in raw:
|
||||
if not isinstance(entry, dict):
|
||||
logger.warning("_parse_tier_filter: entry is not a dict; skipping")
|
||||
continue
|
||||
medium: Medium | None = None
|
||||
locality: Locality | None = None
|
||||
raw_medium = entry.get(MATCHER_MEDIUM_KEY)
|
||||
if raw_medium is not None:
|
||||
try:
|
||||
medium = Medium(raw_medium.upper())
|
||||
except (ValueError, AttributeError):
|
||||
logger.warning(
|
||||
"_parse_tier_filter: unknown medium %r; skipping entry",
|
||||
raw_medium,
|
||||
)
|
||||
continue
|
||||
raw_locality = entry.get(MATCHER_LOCALITY_KEY)
|
||||
if raw_locality is not None:
|
||||
try:
|
||||
locality = Locality(raw_locality.upper())
|
||||
except (ValueError, AttributeError):
|
||||
logger.warning(
|
||||
"_parse_tier_filter: unknown locality %r; skipping entry",
|
||||
raw_locality,
|
||||
)
|
||||
continue
|
||||
matchers.append(TierMatcher(medium=medium, locality=locality))
|
||||
if not matchers:
|
||||
if not raw: # input was [] — user explicitly wants nothing
|
||||
return TierFilter(matchers=())
|
||||
# all entries were invalid — fall back to ALL
|
||||
return TierFilter.ALL
|
||||
return TierFilter(matchers=tuple(matchers))
|
||||
|
||||
|
||||
def _create_req_context(req: Request) -> ReqContext:
|
||||
params = req.kv_transfer_params
|
||||
load_filter = TierFilter.ALL
|
||||
if params:
|
||||
raw = params.get(KV_LOAD_TIERS_KEY)
|
||||
if raw is not None:
|
||||
load_filter = _parse_tier_filter(raw)
|
||||
return ReqContext(
|
||||
req_id=req.request_id,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
kv_transfer_params=params,
|
||||
load_tier_filter=load_filter,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeVar
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -48,10 +48,54 @@ def get_offload_group_idx(key: OffloadKey) -> int:
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class Medium(Enum):
|
||||
"""Storage medium of an offloading tier."""
|
||||
|
||||
CPU = "CPU"
|
||||
STORAGE = "STORAGE"
|
||||
|
||||
|
||||
class Locality(Enum):
|
||||
"""Locality of a tier's storage relative to the publishing instance."""
|
||||
|
||||
LOCAL = "LOCAL"
|
||||
REMOTE = "REMOTE"
|
||||
|
||||
|
||||
class TierMatcher(NamedTuple):
|
||||
medium: Medium | None = None
|
||||
locality: Locality | None = None
|
||||
|
||||
def matches(self, medium: Medium | None, locality: Locality | None) -> bool:
|
||||
medium_matches = self.medium is None or medium is None or self.medium == medium
|
||||
locality_matches = (
|
||||
self.locality is None or locality is None or self.locality == locality
|
||||
)
|
||||
return medium_matches and locality_matches
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TierFilter:
|
||||
"""Per-request filter controlling which tiers participate."""
|
||||
|
||||
matchers: tuple[TierMatcher, ...] = ()
|
||||
|
||||
ALL: ClassVar["TierFilter"]
|
||||
|
||||
def allows(self, medium: Medium | None, locality: Locality | None) -> bool:
|
||||
if self is TierFilter.ALL:
|
||||
return True
|
||||
return any(m.matches(medium, locality) for m in self.matchers)
|
||||
|
||||
|
||||
TierFilter.ALL = TierFilter(matchers=(TierMatcher(),))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqContext:
|
||||
req_id: str
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
load_tier_filter: TierFilter = TierFilter.ALL
|
||||
# Per-request scratch space keyed by value type, so a tier can parse
|
||||
# kv_transfer_params once (in on_new_request) and read the result back
|
||||
# on later calls for the same request.
|
||||
@@ -110,17 +154,10 @@ class PrepareStoreOutput:
|
||||
evicted_keys: list[OffloadKey]
|
||||
|
||||
|
||||
class Locality(Enum):
|
||||
"""Locality of a tier's storage relative to the publishing instance."""
|
||||
|
||||
LOCAL = "LOCAL"
|
||||
REMOTE = "REMOTE"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingEvent:
|
||||
keys: list[OffloadKey]
|
||||
medium: str
|
||||
medium: Medium
|
||||
# True if blocks are removed, False if stored
|
||||
removed: bool
|
||||
locality: Locality | None = None
|
||||
|
||||
@@ -6,13 +6,13 @@ from typing import Literal
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadKey,
|
||||
@@ -52,7 +52,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
store_threshold: int = 1,
|
||||
max_tracker_size: int = 64_000,
|
||||
):
|
||||
self.medium: str = MEDIUM_CPU
|
||||
self.medium: Medium = Medium.CPU
|
||||
self._num_blocks: int = num_blocks
|
||||
self._num_allocated_blocks: int = 0
|
||||
self._free_list: list[int] = []
|
||||
|
||||
@@ -7,12 +7,14 @@ Abstract interfaces and data types for the secondary tiering layer.
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadKey,
|
||||
@@ -27,6 +29,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
|
||||
# Type alias for job IDs used in async transfer tracking
|
||||
JobId = int
|
||||
|
||||
@@ -108,6 +111,8 @@ class SecondaryTierManager(ABC):
|
||||
async jobs; get_finished_jobs() polls for completion.
|
||||
"""
|
||||
|
||||
medium: ClassVar[Medium | None] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
@@ -124,6 +129,7 @@ class SecondaryTierManager(ABC):
|
||||
self._offloading_spec = offloading_spec
|
||||
self._primary_kv_view: memoryview = primary_kv_view
|
||||
self.tier_type = tier_type
|
||||
self.locality: Locality | None = None
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
|
||||
@@ -11,12 +11,13 @@ TieringOffloadingManager without requiring actual storage or network backends.
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
@@ -42,6 +43,8 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
- Completes transfers immediately (synchronous)
|
||||
"""
|
||||
|
||||
medium: ClassVar[Medium] = Medium.CPU
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
|
||||
@@ -30,11 +30,11 @@ except ImportError:
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
@@ -100,7 +100,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
content.
|
||||
"""
|
||||
|
||||
medium: ClassVar[str] = MEDIUM_FS
|
||||
medium: ClassVar[Medium] = Medium.STORAGE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -324,6 +324,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier in self.secondary_tiers:
|
||||
if tier is exclude_tier:
|
||||
continue
|
||||
if not req_context.load_tier_filter.allows(tier.medium, tier.locality):
|
||||
continue
|
||||
result = tier.lookup(key, req_context)
|
||||
if result is LookupResult.HIT:
|
||||
promoted = self._initiate_promotion(tier, key, req_context)
|
||||
|
||||
@@ -7,13 +7,13 @@ import time
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_OBJ
|
||||
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
|
||||
from vllm.distributed.nixl_utils import nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
@@ -98,7 +98,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``.
|
||||
"""
|
||||
|
||||
medium: ClassVar[str] = MEDIUM_OBJ
|
||||
medium: ClassVar[Medium] = Medium.STORAGE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user