[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Chang Guo
2026-07-17 10:31:52 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5
parent 26c909ed74
commit 426d48bfa1
11 changed files with 326 additions and 10 deletions
+99
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import msgspec
import pytest
from vllm.distributed.kv_events import BlockRemoved, BlockStored
@@ -9,9 +12,44 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
_FAKE_HASH: bytes = b"\xab" * 32
class _LegacyBlockStored(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
gc=False, # type: ignore[call-arg]
tag="BlockStored", # type: ignore[call-arg]
):
"""BlockStored wire schema before locality was added."""
block_hashes: list[bytes]
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
kv_cache_spec_sliding_window: int | None = None
class _LegacyBlockRemoved(
msgspec.Struct,
omit_defaults=True, # type: ignore[call-arg]
gc=False, # type: ignore[call-arg]
tag="BlockRemoved", # type: ignore[call-arg]
):
"""BlockRemoved wire schema before locality was added."""
block_hashes: list[bytes]
medium: str | None
group_idx: int | None = None
def _make_block_stored(
group_idx: int | None = None,
kv_cache_spec_sliding_window: int | None = None,
locality: str | None = None,
) -> BlockStored:
return BlockStored(
block_hashes=[_FAKE_HASH],
@@ -23,16 +61,19 @@ def _make_block_stored(
lora_name=None,
group_idx=group_idx,
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
locality=locality,
)
def _make_block_removed(
group_idx: int | None = None,
locality: str | None = None,
) -> BlockRemoved:
return BlockRemoved(
block_hashes=[_FAKE_HASH],
medium="GPU",
group_idx=group_idx,
locality=locality,
)
@@ -84,3 +125,61 @@ def test_block_stored_hash_differs_by_sliding_window():
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
assert hash(event_a) != hash(event_b)
@pytest.mark.parametrize(
("event_a", "event_b"),
[
(
_make_block_stored(locality="LOCAL"),
_make_block_stored(locality="REMOTE"),
),
(
_make_block_removed(locality="LOCAL"),
_make_block_removed(locality="REMOTE"),
),
],
)
def test_event_hash_differs_by_locality(
event_a: BlockStored | BlockRemoved,
event_b: BlockStored | BlockRemoved,
):
assert hash(event_a) != hash(event_b)
def test_block_stored_locality_is_wire_compatible():
legacy = _LegacyBlockStored(
block_hashes=[_FAKE_HASH],
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
)
legacy_payload = msgspec.msgpack.encode(legacy)
assert (
msgspec.msgpack.encode(
_make_block_stored(
group_idx=2,
kv_cache_spec_sliding_window=128,
)
)
== legacy_payload
)
assert msgspec.msgpack.decode(legacy_payload, type=BlockStored).locality is None
new_payload = msgspec.msgpack.encode(_make_block_stored(locality="LOCAL"))
assert msgspec.msgpack.decode(new_payload)["locality"] == "LOCAL"
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockStored).medium == "GPU"
def test_block_removed_locality_is_wire_compatible():
legacy = _LegacyBlockRemoved(block_hashes=[_FAKE_HASH], medium="GPU")
legacy_payload = msgspec.msgpack.encode(legacy)
assert msgspec.msgpack.encode(_make_block_removed()) == legacy_payload
assert msgspec.msgpack.decode(legacy_payload, type=BlockRemoved).locality is None
new_payload = msgspec.msgpack.encode(_make_block_removed(locality="REMOTE"))
assert msgspec.msgpack.decode(new_payload)["locality"] == "REMOTE"
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockRemoved).medium == "GPU"
@@ -7,7 +7,7 @@ import torch
from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
from vllm.distributed.kv_events import MEDIUM_CPU, MEDIUM_FS, BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
build_offloading_config,
)
@@ -26,6 +26,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheSpecKind,
)
from vllm.v1.kv_offload.base import (
Locality,
OffloadingEvent,
OffloadingKVEventsConfig,
OffloadKey,
@@ -103,12 +104,81 @@ def _record_chunks(
return keys
def _stored_event(keys: list[OffloadKey]) -> OffloadingEvent:
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=False)
def _stored_event(
keys: list[OffloadKey],
locality: Locality | None = None,
medium: str = _CPU_MEDIUM,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=False,
locality=locality,
)
def _removed_event(keys: list[OffloadKey]) -> OffloadingEvent:
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=True)
def _removed_event(
keys: list[OffloadKey],
locality: Locality | None = None,
medium: str = _CPU_MEDIUM,
) -> OffloadingEvent:
return OffloadingEvent(
keys=keys,
medium=medium,
removed=True,
locality=locality,
)
def test_take_events_forwards_locality_to_rich_store():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockStored)
assert events[0].token_ids == [1, 2, 3, 4]
assert events[0].block_size == 4
assert events[0].locality == "LOCAL"
def test_take_events_forwards_locality_to_placeholder_store():
tracker = _tracker(self_describing_kv_events=False)
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockStored)
assert events[0].block_size == 0
assert events[0].locality == "REMOTE"
def test_take_events_forwards_locality_to_remove():
tracker = _tracker()
req = _request(block_hashes=[_hash(0)], token_count=4)
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
events = list(
tracker.take_events(
[_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
)
)
assert len(events) == 1
assert isinstance(events[0], BlockRemoved)
assert events[0].locality == "LOCAL"
def test_take_events_publishes_routable_block_stored():
@@ -20,6 +20,7 @@ import torch
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingEvent,
OffloadingKVEventsConfig,
@@ -35,6 +36,7 @@ from vllm.v1.kv_offload.config import (
OffloadingParallelConfig,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
)
@@ -186,6 +188,7 @@ def fs_tier_with_events(tmp_path):
n_read_threads=4,
n_write_threads=4,
enable_kv_events=True,
locality="LOCAL",
)
yield tier
tier.shutdown()
@@ -251,6 +254,40 @@ def test_invalid_path_raises_at_construction():
)
@pytest.mark.parametrize("locality", ["local", ""])
def test_invalid_locality_raises_at_construction(tmp_path, locality):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
with pytest.raises(ValueError, match="Locality"):
FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
locality=locality,
)
def test_factory_forwards_locality_to_fs_tier(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tier = SecondaryTierFactory.create_secondary_tier(
{
"type": "fs",
"root_dir": str(tmp_path),
"n_read_threads": 1,
"n_write_threads": 1,
"locality": "LOCAL",
},
memoryview(tensor.numpy()),
_MOCK_OFFLOADING_SPEC,
)
try:
assert isinstance(tier, FileSystemTierManager)
assert tier.locality is Locality.LOCAL
finally:
tier.shutdown()
def test_failed_load_missing_file(fs_tier):
"""Test that loading a block whose file does not exist results in a failed job."""
tier, _ = fs_tier
@@ -442,11 +479,38 @@ def test_successful_store_emits_stored_event(fs_tier_with_events):
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "FS"
assert events[0].locality is Locality.LOCAL
assert not events[0].removed
# take_events drains the buffer.
assert list(tier.take_events()) == []
@pytest.mark.parametrize(
("locality", "expected"),
[(None, None), ("REMOTE", Locality.REMOTE)],
)
def test_store_event_uses_configured_locality(tmp_path, locality, expected):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
locality_config = {} if locality is None else {"locality": locality}
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
enable_kv_events=True,
**locality_config,
)
try:
tier.submit_store(make_job(1, [key(1)], [0]))
assert all(r.success for r in drain(tier))
events = list(tier.take_events())
assert len(events) == 1
assert events[0].locality is expected
finally:
tier.shutdown()
def test_load_job_emits_no_event(fs_tier_with_events):
tier = fs_tier_with_events
tier.submit_store(make_job(1, [key(1)], [0]))
@@ -15,9 +15,11 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import (
Locality,
LookupResult,
OffloadingKVEventsConfig,
OffloadKey,
@@ -265,6 +267,12 @@ def lookup_and_wait(
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("locality", ["local", ""])
def test_invalid_locality_raises_at_construction(locality):
with pytest.raises(ValueError, match="Locality"):
_make_tier(locality=locality)
class TestMockObjTierBasic:
def setup_method(self):
self.tier, self.agent = _make_tier(num_blocks=4)
@@ -453,6 +461,7 @@ class TestObjTierKVEvents:
self.tier, self.agent = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
enable_kv_events=True,
locality="REMOTE",
)
def test_successful_store_emits_stored_event(self):
@@ -466,10 +475,32 @@ class TestObjTierKVEvents:
assert events[0].keys == keys
# Literal medium pins the wire contract, not just the constant choice.
assert events[0].medium == "OBJ"
assert events[0].locality is Locality.REMOTE
assert not events[0].removed
# take_events drains the buffer.
assert list(self.tier.take_events()) == []
@pytest.mark.parametrize(
("locality", "expected"),
[(None, None), ("LOCAL", Locality.LOCAL)],
)
def test_store_event_uses_configured_locality(self, locality, expected):
locality_config = {} if locality is None else {"locality": locality}
tier, _ = _make_tier(
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
enable_kv_events=True,
**locality_config,
)
try:
tier.submit_store(make_job(1, [key(1)], [0]))
assert all(r.success for r in drain(tier))
events = list(tier.take_events())
assert len(events) == 1
assert events[0].locality is expected
finally:
tier.shutdown()
def test_mixed_job_results_emit_event_only_for_successful_job(self):
"""With a failed and a successful store job resolving in the same
poll, exactly one event is emitted and its keys belong to the