[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Chang Guo
2026-07-17 10:31:52 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5
parent 26c909ed74
commit 426d48bfa1
11 changed files with 326 additions and 10 deletions
+5 -1
View File
@@ -83,9 +83,11 @@ Each entry in `secondary_tiers` is a dict with a required `type` field plus tier
The filesystem and object-store tiers can publish hash-only `BlockStored` KV events for blocks they successfully store, tagged with a stable per-tier `medium` (`FS` for the filesystem tier, `OBJ` for the object-store tier). Set `enable_kv_events: true` in the tier's entry to opt in; events are published only when KV cache events are also enabled globally via `--kv-events-config`.
Set the optional `locality` tier field to `LOCAL` or `REMOTE` to describe the tier's storage location relative to the publishing vLLM instance. `LOCAL` marks storage local to that instance, while `REMOTE` marks storage that is not local to it. When the setting is omitted, locality is unspecified. vLLM does not infer it from the tier type, so an OBJ tier is not implicitly `REMOTE`. A KV event includes `locality` only when the tier explicitly configures it. This metadata describes the tier property without implying that a consumer can already route requests to its blocks.
### Filesystem (FS)
The filesystem tier (`type: "fs"`) writes blocks to a directory on local storage.
The filesystem tier (`type: "fs"`) writes blocks to a filesystem directory.
| Key | Required | Default | Notes |
| --- | --- | --- | --- |
@@ -94,6 +96,7 @@ The filesystem tier (`type: "fs"`) writes blocks to a directory on local storage
| `n_read_threads` | no | `16` | Read-priority I/O threads (load path). |
| `n_write_threads` | no | `16` | Write-priority I/O threads (store path). |
| `enable_kv_events` | no | `false` | Publish `BlockStored` KV events (medium `FS`) for successfully stored blocks. Requires KV cache events to be enabled globally. |
| `locality` | no | unspecified | `LOCAL` or `REMOTE` relative to the publishing vLLM instance. Included in the tier's KV events only when explicitly configured. |
Each thread group prefers its own queue but pulls from the other when its primary queue is empty, so a write-heavy or read-heavy burst won't leave the off-priority queue waiting. Size the totals to your storage's effective concurrency.
@@ -134,6 +137,7 @@ The object-store tier (`type: "obj"`) offloads blocks to an S3-compatible object
| `prefix` | no | `""` | Key prefix prepended to all object keys. |
| `io_threads` | no | `4` | Number of NIXL OBJ backend I/O threads. |
| `enable_kv_events` | no | `false` | Publish `BlockStored` KV events (medium `OBJ`) for successfully stored blocks. Requires KV cache events to be enabled globally. |
| `locality` | no | unspecified | `LOCAL` or `REMOTE` relative to the publishing vLLM instance. Included in the tier's KV events only when explicitly configured; OBJ does not imply `REMOTE`. |
`store_config` fields:
@@ -42,12 +42,16 @@ class BlockStored(KVCacheEvent):
"""
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
kv_cache_spec_sliding_window: int | None = None
locality: str | None = None
class BlockRemoved(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
medium: str | None
group_idx: int | None = None
locality: str | None = None
class AllBlocksCleared(KVCacheEvent):
+99
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import msgspec
import pytest
from vllm.distributed.kv_events import BlockRemoved, BlockStored
@@ -9,9 +12,44 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
_FAKE_HASH: bytes = b"\xab" * 32
class _LegacyBlockStored(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
gc=False, # type: ignore[call-arg]
tag="BlockStored", # type: ignore[call-arg]
):
"""BlockStored wire schema before locality was added."""
block_hashes: list[bytes]
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
kv_cache_spec_sliding_window: int | None = None
class _LegacyBlockRemoved(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
gc=False, # type: ignore[call-arg]
tag="BlockRemoved", # type: ignore[call-arg]
):
"""BlockRemoved wire schema before locality was added."""
block_hashes: list[bytes]
medium: str | None
group_idx: int | None = None
def _make_block_stored(
group_idx: int | None = None,
kv_cache_spec_sliding_window: int | None = None,
locality: str | None = None,
) -> BlockStored:
return BlockStored(
block_hashes=[_FAKE_HASH],
@@ -23,16 +61,19 @@ def _make_block_stored(
lora_name=None,
group_idx=group_idx,
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
locality=locality,
)
def _make_block_removed(
group_idx: int | None = None,
locality: str | None = None,
) -> BlockRemoved:
return BlockRemoved(
block_hashes=[_FAKE_HASH],
medium="GPU",
group_idx=group_idx,
locality=locality,
)
@@ -84,3 +125,61 @@ def test_block_stored_hash_differs_by_sliding_window():
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
assert hash(event_a) != hash(event_b)
@pytest.mark.parametrize(
("event_a", "event_b"),
[
(
_make_block_stored(locality="LOCAL"),
_make_block_stored(locality="REMOTE"),
),
(
_make_block_removed(locality="LOCAL"),
_make_block_removed(locality="REMOTE"),
),
],
)
def test_event_hash_differs_by_locality(
event_a: BlockStored | BlockRemoved,
event_b: BlockStored | BlockRemoved,
):
assert hash(event_a) != hash(event_b)
def test_block_stored_locality_is_wire_compatible():
legacy = _LegacyBlockStored(
block_hashes=[_FAKE_HASH],
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
)
legacy_payload = msgspec.msgpack.encode(legacy)
assert (
msgspec.msgpack.encode(
_make_block_stored(
group_idx=2,
kv_cache_spec_sliding_window=128,
)
)
== legacy_payload
)
assert msgspec.msgpack.decode(legacy_payload, type=BlockStored).locality is None
new_payload = msgspec.msgpack.encode(_make_block_stored(locality="LOCAL"))
assert msgspec.msgpack.decode(new_payload)["locality"] == "LOCAL"
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockStored).medium == "GPU"
def test_block_removed_locality_is_wire_compatible():
legacy = _LegacyBlockRemoved(block_hashes=[_FAKE_HASH], medium="GPU")
legacy_payload = msgspec.msgpack.encode(legacy)
assert msgspec.msgpack.encode(_make_block_removed()) == legacy_payload
assert msgspec.msgpack.decode(legacy_payload, type=BlockRemoved).locality is None
new_payload = msgspec.msgpack.encode(_make_block_removed(locality="REMOTE"))
assert msgspec.msgpack.decode(new_payload)["locality"] == "REMOTE"
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockRemoved).medium == "GPU"
@@ -7,7 +7,7 @@ import torch
from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_events import MEDIUM_CPU, MEDIUM_FS, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
build_offloading_config,
)
@@ -26,6 +26,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheSpecKind,
)
from vllm.v1.kv_offload.base import (
Locality,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
@@ -103,12 +104,81 @@ def _record_chunks(
return keys
def _stored_event(keys: list[OffloadKey]) -> OffloadingEvent:
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=False)
def _stored_event(
keys: list[OffloadKey],
locality: Locality | None = None,
medium: str = _CPU_MEDIUM,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=False,
locality=locality,
)
def _removed_event(keys: list[OffloadKey]) -> OffloadingEvent:
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=True)
def _removed_event(
keys: list[OffloadKey],
locality: Locality | None = None,
medium: str = _CPU_MEDIUM,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=True,
locality=locality,
)
def test_take_events_forwards_locality_to_rich_store():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockStored)
assert events[0].token_ids == [1, 2, 3, 4]
assert events[0].block_size == 4
assert events[0].locality == "LOCAL"
def test_take_events_forwards_locality_to_placeholder_store():
tracker = _tracker(self_describing_kv_events=False)
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockStored)
assert events[0].block_size == 0
assert events[0].locality == "REMOTE"
def test_take_events_forwards_locality_to_remove():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockRemoved)
assert events[0].locality == "LOCAL"
def test_take_events_publishes_routable_block_stored():
@@ -20,6 +20,7 @@ import torch
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingEvent,
OffloadingKVEventsConfig,
@@ -35,6 +36,7 @@ from vllm.v1.kv_offload.config import (
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
)
@@ -186,6 +188,7 @@ def fs_tier_with_events(tmp_path):
n_read_threads=4,
n_write_threads=4,
enable_kv_events=True,
locality="LOCAL",
)
yield tier
tier.shutdown()
@@ -251,6 +254,40 @@ def test_invalid_path_raises_at_construction():
)
@pytest.mark.parametrize("locality", ["local", ""])
def test_invalid_locality_raises_at_construction(tmp_path, locality):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
with pytest.raises(ValueError, match="Locality"):
FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
locality=locality,
)
def test_factory_forwards_locality_to_fs_tier(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tier = SecondaryTierFactory.create_secondary_tier(
{
"type": "fs",
"root_dir": str(tmp_path),
"n_read_threads": 1,
"n_write_threads": 1,
"locality": "LOCAL",
},
memoryview(tensor.numpy()),
_MOCK_OFFLOADING_SPEC,
)
try:
assert isinstance(tier, FileSystemTierManager)
assert tier.locality is Locality.LOCAL
finally:
tier.shutdown()
def test_failed_load_missing_file(fs_tier):
"""Test that loading a block whose file does not exist results in a failed job."""
tier, _ = fs_tier
@@ -442,11 +479,38 @@ def test_successful_store_emits_stored_event(fs_tier_with_events):
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "FS"
assert events[0].locality is Locality.LOCAL
assert not events[0].removed
# take_events drains the buffer.
assert list(tier.take_events()) == []
@pytest.mark.parametrize(
("locality", "expected"),
[(None, None), ("REMOTE", Locality.REMOTE)],
)
def test_store_event_uses_configured_locality(tmp_path, locality, expected):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
locality_config = {} if locality is None else {"locality": locality}
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
enable_kv_events=True,
**locality_config,
)
try:
tier.submit_store(make_job(1, [key(1)], [0]))
assert all(r.success for r in drain(tier))
events = list(tier.take_events())
assert len(events) == 1
assert events[0].locality is expected
finally:
tier.shutdown()
def test_load_job_emits_no_event(fs_tier_with_events):
tier = fs_tier_with_events
tier.submit_store(make_job(1, [key(1)], [0]))
@@ -15,9 +15,11 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingKVEventsConfig,
OffloadKey,
@@ -265,6 +267,12 @@ def lookup_and_wait(
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("locality", ["local", ""])
def test_invalid_locality_raises_at_construction(locality):
with pytest.raises(ValueError, match="Locality"):
_make_tier(locality=locality)
class TestMockObjTierBasic:
def setup_method(self):
self.tier, self.agent = _make_tier(num_blocks=4)
@@ -453,6 +461,7 @@ class TestObjTierKVEvents:
self.tier, self.agent = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
enable_kv_events=True,
locality="REMOTE",
)
def test_successful_store_emits_stored_event(self):
@@ -466,10 +475,32 @@ class TestObjTierKVEvents:
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "OBJ"
assert events[0].locality is Locality.REMOTE
assert not events[0].removed
# take_events drains the buffer.
assert list(self.tier.take_events()) == []
@pytest.mark.parametrize(
("locality", "expected"),
[(None, None), ("LOCAL", Locality.LOCAL)],
)
def test_store_event_uses_configured_locality(self, locality, expected):
locality_config = {} if locality is None else {"locality": locality}
tier, _ = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
enable_kv_events=True,
**locality_config,
)
try:
tier.submit_store(make_job(1, [key(1)], [0]))
assert all(r.success for r in drain(tier))
events = list(tier.take_events())
assert len(events) == 1
assert events[0].locality is expected
finally:
tier.shutdown()
def test_mixed_job_results_emit_event_only_for_successful_job(self):
"""With a failed and a successful store job resolving in the same
poll, exactly one event is emitted and its keys belong to the
+6
View File
@@ -74,6 +74,8 @@ class BlockStored(KVCacheEvent):
# filter groups as they are learned. Remove events only need group_idx+hash.
kv_cache_spec_kind: str | None = None
kv_cache_spec_sliding_window: int | None = None
locality: str | None = None
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
def __hash__(self) -> int:
return hash(
@@ -88,6 +90,7 @@ class BlockStored(KVCacheEvent):
self.group_idx,
self.kv_cache_spec_kind,
self.kv_cache_spec_sliding_window,
self.locality,
)
)
@@ -96,6 +99,8 @@ class BlockRemoved(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
medium: str | None
group_idx: int | None = None
locality: str | None = None
"""LOCAL or REMOTE relative to the publisher; None means unspecified."""
def __hash__(self) -> int:
return hash(
@@ -103,6 +108,7 @@ class BlockRemoved(KVCacheEvent):
tuple(self.block_hashes),
self.medium,
self.group_idx,
self.locality,
)
)
@@ -198,7 +198,12 @@ class OffloadingEventsTracker:
kv_cache_spec=group_config.kv_event_group_spec,
)
def _placeholder_stored(self, key: OffloadKey, medium: str) -> BlockStored:
def _placeholder_stored(
self,
key: OffloadKey,
medium: str,
locality: str | None,
) -> BlockStored:
return BlockStored(
block_hashes=[
maybe_convert_block_hash(BlockHash(get_offload_block_hash(key)))
@@ -210,12 +215,14 @@ class OffloadingEventsTracker:
medium=medium,
lora_name=None,
group_idx=get_offload_group_idx(key),
locality=locality,
)
def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Metadata is read, NOT popped: the entry must survive until the
# eviction event so BlockRemoved can fan out to the same hashes.
# Events are self-contained (own parent), so key order is free.
locality = event.locality.value if event.locality is not None else None
for key in event.keys:
meta = self._pending_event_metadata.get(key)
if meta is None:
@@ -227,7 +234,7 @@ class OffloadingEventsTracker:
"placeholder payload. Expected for non-full-attention "
"groups; otherwise indicates a missing populate path."
)
yield self._placeholder_stored(key, event.medium)
yield self._placeholder_stored(key, event.medium, locality)
continue
yield BlockStored(
@@ -252,10 +259,12 @@ class OffloadingEventsTracker:
kv_cache_spec_sliding_window=(
meta.kv_cache_spec.kv_cache_spec_sliding_window
),
locality=locality,
)
def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]:
# Keep group_idx unambiguous if a manager batch spans groups.
locality = event.locality.value if event.locality is not None else None
by_group: dict[int, list] = {}
for key in event.keys:
meta = self._pending_event_metadata.pop(key, None)
@@ -283,4 +292,5 @@ class OffloadingEventsTracker:
block_hashes=hashes,
medium=event.medium,
group_idx=group_idx,
locality=locality,
)
+8
View File
@@ -97,12 +97,20 @@ class PrepareStoreOutput:
evicted_keys: list[OffloadKey]
class Locality(Enum):
"""Locality of a tier's storage relative to the publishing instance."""
LOCAL = "LOCAL"
REMOTE = "REMOTE"
@dataclass
class OffloadingEvent:
keys: list[OffloadKey]
medium: str
# True if blocks are removed, False if stored
removed: bool
locality: Locality | None = None
"""
+11 -1
View File
@@ -33,6 +33,7 @@ from typing_extensions import override
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingEvent,
OffloadKey,
@@ -110,6 +111,7 @@ class FileSystemTierManager(SecondaryTierManager):
n_read_threads: int = 16,
n_write_threads: int = 16,
enable_kv_events: bool = False,
locality: str | None = None,
):
"""
Args:
@@ -123,8 +125,11 @@ class FileSystemTierManager(SecondaryTierManager):
enable_kv_events: Emit BlockStored KV events for blocks
successfully stored to this tier. Effective only when KV
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
if enable_kv_events:
@@ -223,7 +228,12 @@ class FileSystemTierManager(SecondaryTierManager):
keys = self._store_job_keys.pop(job_id, None)
if success and keys:
self.events.append(
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
OffloadingEvent(
keys=keys,
medium=self.medium,
removed=False,
locality=self.locality,
)
)
results.append(JobResult(job_id=job_id, success=success))
return results
+11 -1
View File
@@ -12,6 +12,7 @@ from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
from vllm.distributed.nixl_utils import nixl_agent_config
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingEvent,
OffloadKey,
@@ -108,6 +109,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
prefix: str = "",
io_threads: int = 4,
enable_kv_events: bool = False,
locality: str | None = None,
):
"""
Args:
@@ -120,8 +122,11 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
enable_kv_events: Emit BlockStored KV events for blocks
successfully stored to this tier. Effective only when KV
cache events are enabled globally (kv_events_config).
locality: Whether this tier's storage is LOCAL or REMOTE relative
to the publishing vLLM instance.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
self.locality = Locality(locality) if locality is not None else None
self.events: list[OffloadingEvent] | None = None
if enable_kv_events:
@@ -323,7 +328,12 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
keys = self._store_job_keys.pop(result.job_id, None)
if result.success and keys:
self.events.append(
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
OffloadingEvent(
keys=keys,
medium=self.medium,
removed=False,
locality=self.locality,
)
)
return results