[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
parent
26c909ed74
commit
426d48bfa1
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
@@ -9,9 +12,44 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
_FAKE_HASH: bytes = b"\xab" * 32
|
||||
|
||||
|
||||
class _LegacyBlockStored(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag="BlockStored", # type: ignore[call-arg]
|
||||
):
|
||||
"""BlockStored wire schema before locality was added."""
|
||||
|
||||
block_hashes: list[bytes]
|
||||
parent_block_hash: bytes | None
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
lora_id: int | None
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
group_idx: int | None = None
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
|
||||
|
||||
class _LegacyBlockRemoved(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag="BlockRemoved", # type: ignore[call-arg]
|
||||
):
|
||||
"""BlockRemoved wire schema before locality was added."""
|
||||
|
||||
block_hashes: list[bytes]
|
||||
medium: str | None
|
||||
group_idx: int | None = None
|
||||
|
||||
|
||||
def _make_block_stored(
|
||||
group_idx: int | None = None,
|
||||
kv_cache_spec_sliding_window: int | None = None,
|
||||
locality: str | None = None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
@@ -23,16 +61,19 @@ def _make_block_stored(
|
||||
lora_name=None,
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
def _make_block_removed(
|
||||
group_idx: int | None = None,
|
||||
locality: str | None = None,
|
||||
) -> BlockRemoved:
|
||||
return BlockRemoved(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
medium="GPU",
|
||||
group_idx=group_idx,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
@@ -84,3 +125,61 @@ def test_block_stored_hash_differs_by_sliding_window():
|
||||
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
|
||||
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event_a", "event_b"),
|
||||
[
|
||||
(
|
||||
_make_block_stored(locality="LOCAL"),
|
||||
_make_block_stored(locality="REMOTE"),
|
||||
),
|
||||
(
|
||||
_make_block_removed(locality="LOCAL"),
|
||||
_make_block_removed(locality="REMOTE"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_event_hash_differs_by_locality(
|
||||
event_a: BlockStored | BlockRemoved,
|
||||
event_b: BlockStored | BlockRemoved,
|
||||
):
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
|
||||
def test_block_stored_locality_is_wire_compatible():
|
||||
legacy = _LegacyBlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
)
|
||||
legacy_payload = msgspec.msgpack.encode(legacy)
|
||||
assert (
|
||||
msgspec.msgpack.encode(
|
||||
_make_block_stored(
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
)
|
||||
)
|
||||
== legacy_payload
|
||||
)
|
||||
assert msgspec.msgpack.decode(legacy_payload, type=BlockStored).locality is None
|
||||
new_payload = msgspec.msgpack.encode(_make_block_stored(locality="LOCAL"))
|
||||
assert msgspec.msgpack.decode(new_payload)["locality"] == "LOCAL"
|
||||
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockStored).medium == "GPU"
|
||||
|
||||
|
||||
def test_block_removed_locality_is_wire_compatible():
|
||||
legacy = _LegacyBlockRemoved(block_hashes=[_FAKE_HASH], medium="GPU")
|
||||
legacy_payload = msgspec.msgpack.encode(legacy)
|
||||
assert msgspec.msgpack.encode(_make_block_removed()) == legacy_payload
|
||||
assert msgspec.msgpack.decode(legacy_payload, type=BlockRemoved).locality is None
|
||||
new_payload = msgspec.msgpack.encode(_make_block_removed(locality="REMOTE"))
|
||||
assert msgspec.msgpack.decode(new_payload)["locality"] == "REMOTE"
|
||||
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockRemoved).medium == "GPU"
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, MEDIUM_FS, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpecKind,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
@@ -103,12 +104,81 @@ def _record_chunks(
|
||||
return keys
|
||||
|
||||
|
||||
def _stored_event(keys: list[OffloadKey]) -> OffloadingEvent:
|
||||
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=False)
|
||||
def _stored_event(
|
||||
keys: list[OffloadKey],
|
||||
locality: Locality | None = None,
|
||||
medium: str = _CPU_MEDIUM,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
medium=medium,
|
||||
removed=False,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
def _removed_event(keys: list[OffloadKey]) -> OffloadingEvent:
|
||||
return OffloadingEvent(keys=keys, medium=_CPU_MEDIUM, removed=True)
|
||||
def _removed_event(
|
||||
keys: list[OffloadKey],
|
||||
locality: Locality | None = None,
|
||||
medium: str = _CPU_MEDIUM,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
medium=medium,
|
||||
removed=True,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_rich_store():
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_stored_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
|
||||
)
|
||||
)
|
||||
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].token_ids == [1, 2, 3, 4]
|
||||
assert events[0].block_size == 4
|
||||
assert events[0].locality == "LOCAL"
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_placeholder_store():
|
||||
tracker = _tracker(self_describing_kv_events=False)
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_stored_event([key], locality=Locality.REMOTE, medium=MEDIUM_FS)]
|
||||
)
|
||||
)
|
||||
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].block_size == 0
|
||||
assert events[0].locality == "REMOTE"
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_remove():
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
key = _record_chunks(tracker, req, _group_config(), num_chunks=1)[0]
|
||||
|
||||
events = list(
|
||||
tracker.take_events(
|
||||
[_removed_event([key], locality=Locality.LOCAL, medium=MEDIUM_FS)]
|
||||
)
|
||||
)
|
||||
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockRemoved)
|
||||
assert events[0].locality == "LOCAL"
|
||||
|
||||
|
||||
def test_take_events_publishes_routable_block_stored():
|
||||
|
||||
@@ -20,6 +20,7 @@ import torch
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingKVEventsConfig,
|
||||
@@ -35,6 +36,7 @@ from vllm.v1.kv_offload.config import (
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||
FileSystemTierManager,
|
||||
)
|
||||
@@ -186,6 +188,7 @@ def fs_tier_with_events(tmp_path):
|
||||
n_read_threads=4,
|
||||
n_write_threads=4,
|
||||
enable_kv_events=True,
|
||||
locality="LOCAL",
|
||||
)
|
||||
yield tier
|
||||
tier.shutdown()
|
||||
@@ -251,6 +254,40 @@ def test_invalid_path_raises_at_construction():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("locality", ["local", ""])
|
||||
def test_invalid_locality_raises_at_construction(tmp_path, locality):
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
|
||||
with pytest.raises(ValueError, match="Locality"):
|
||||
FileSystemTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=memoryview(tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
def test_factory_forwards_locality_to_fs_tier(tmp_path):
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
tier = SecondaryTierFactory.create_secondary_tier(
|
||||
{
|
||||
"type": "fs",
|
||||
"root_dir": str(tmp_path),
|
||||
"n_read_threads": 1,
|
||||
"n_write_threads": 1,
|
||||
"locality": "LOCAL",
|
||||
},
|
||||
memoryview(tensor.numpy()),
|
||||
_MOCK_OFFLOADING_SPEC,
|
||||
)
|
||||
try:
|
||||
assert isinstance(tier, FileSystemTierManager)
|
||||
assert tier.locality is Locality.LOCAL
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
def test_failed_load_missing_file(fs_tier):
|
||||
"""Test that loading a block whose file does not exist results in a failed job."""
|
||||
tier, _ = fs_tier
|
||||
@@ -442,11 +479,38 @@ def test_successful_store_emits_stored_event(fs_tier_with_events):
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "FS"
|
||||
assert events[0].locality is Locality.LOCAL
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
assert list(tier.take_events()) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("locality", "expected"),
|
||||
[(None, None), ("REMOTE", Locality.REMOTE)],
|
||||
)
|
||||
def test_store_event_uses_configured_locality(tmp_path, locality, expected):
|
||||
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
|
||||
locality_config = {} if locality is None else {"locality": locality}
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
|
||||
primary_kv_view=memoryview(tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
enable_kv_events=True,
|
||||
**locality_config,
|
||||
)
|
||||
try:
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
events = list(tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].locality is expected
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
def test_load_job_emits_no_event(fs_tier_with_events):
|
||||
tier = fs_tier_with_events
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
|
||||
@@ -15,9 +15,11 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
Locality,
|
||||
LookupResult,
|
||||
OffloadingKVEventsConfig,
|
||||
OffloadKey,
|
||||
@@ -265,6 +267,12 @@ def lookup_and_wait(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("locality", ["local", ""])
|
||||
def test_invalid_locality_raises_at_construction(locality):
|
||||
with pytest.raises(ValueError, match="Locality"):
|
||||
_make_tier(locality=locality)
|
||||
|
||||
|
||||
class TestMockObjTierBasic:
|
||||
def setup_method(self):
|
||||
self.tier, self.agent = _make_tier(num_blocks=4)
|
||||
@@ -453,6 +461,7 @@ class TestObjTierKVEvents:
|
||||
self.tier, self.agent = _make_tier(
|
||||
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
|
||||
enable_kv_events=True,
|
||||
locality="REMOTE",
|
||||
)
|
||||
|
||||
def test_successful_store_emits_stored_event(self):
|
||||
@@ -466,10 +475,32 @@ class TestObjTierKVEvents:
|
||||
assert events[0].keys == keys
|
||||
# Literal medium pins the wire contract, not just the constant choice.
|
||||
assert events[0].medium == "OBJ"
|
||||
assert events[0].locality is Locality.REMOTE
|
||||
assert not events[0].removed
|
||||
# take_events drains the buffer.
|
||||
assert list(self.tier.take_events()) == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("locality", "expected"),
|
||||
[(None, None), ("LOCAL", Locality.LOCAL)],
|
||||
)
|
||||
def test_store_event_uses_configured_locality(self, locality, expected):
|
||||
locality_config = {} if locality is None else {"locality": locality}
|
||||
tier, _ = _make_tier(
|
||||
offloading_spec=_make_events_spec(enable_kv_cache_events=True),
|
||||
enable_kv_events=True,
|
||||
**locality_config,
|
||||
)
|
||||
try:
|
||||
tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
events = list(tier.take_events())
|
||||
assert len(events) == 1
|
||||
assert events[0].locality is expected
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
def test_mixed_job_results_emit_event_only_for_successful_job(self):
|
||||
"""With a failed and a successful store job resolving in the same
|
||||
poll, exactly one event is emitted and its keys belong to the
|
||||
|
||||
Reference in New Issue
Block a user