[KV Offload] Add optional tier locality to FS/OBJ KV events (#48281)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
parent
26c909ed74
commit
426d48bfa1
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
@@ -9,9 +12,44 @@ from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
_FAKE_HASH: bytes = b"\xab" * 32
|
||||
|
||||
|
||||
class _LegacyBlockStored(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag="BlockStored", # type: ignore[call-arg]
|
||||
):
|
||||
"""BlockStored wire schema before locality was added."""
|
||||
|
||||
block_hashes: list[bytes]
|
||||
parent_block_hash: bytes | None
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
lora_id: int | None
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
group_idx: int | None = None
|
||||
kv_cache_spec_kind: str | None = None
|
||||
kv_cache_spec_sliding_window: int | None = None
|
||||
|
||||
|
||||
class _LegacyBlockRemoved(
|
||||
msgspec.Struct,
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag="BlockRemoved", # type: ignore[call-arg]
|
||||
):
|
||||
"""BlockRemoved wire schema before locality was added."""
|
||||
|
||||
block_hashes: list[bytes]
|
||||
medium: str | None
|
||||
group_idx: int | None = None
|
||||
|
||||
|
||||
def _make_block_stored(
|
||||
group_idx: int | None = None,
|
||||
kv_cache_spec_sliding_window: int | None = None,
|
||||
locality: str | None = None,
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
@@ -23,16 +61,19 @@ def _make_block_stored(
|
||||
lora_name=None,
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
def _make_block_removed(
|
||||
group_idx: int | None = None,
|
||||
locality: str | None = None,
|
||||
) -> BlockRemoved:
|
||||
return BlockRemoved(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
medium="GPU",
|
||||
group_idx=group_idx,
|
||||
locality=locality,
|
||||
)
|
||||
|
||||
|
||||
@@ -84,3 +125,61 @@ def test_block_stored_hash_differs_by_sliding_window():
|
||||
event_a = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=128)
|
||||
event_b = _make_block_stored(group_idx=1, kv_cache_spec_sliding_window=256)
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event_a", "event_b"),
|
||||
[
|
||||
(
|
||||
_make_block_stored(locality="LOCAL"),
|
||||
_make_block_stored(locality="REMOTE"),
|
||||
),
|
||||
(
|
||||
_make_block_removed(locality="LOCAL"),
|
||||
_make_block_removed(locality="REMOTE"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_event_hash_differs_by_locality(
|
||||
event_a: BlockStored | BlockRemoved,
|
||||
event_b: BlockStored | BlockRemoved,
|
||||
):
|
||||
assert hash(event_a) != hash(event_b)
|
||||
|
||||
|
||||
def test_block_stored_locality_is_wire_compatible():
|
||||
legacy = _LegacyBlockStored(
|
||||
block_hashes=[_FAKE_HASH],
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
)
|
||||
legacy_payload = msgspec.msgpack.encode(legacy)
|
||||
assert (
|
||||
msgspec.msgpack.encode(
|
||||
_make_block_stored(
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
)
|
||||
)
|
||||
== legacy_payload
|
||||
)
|
||||
assert msgspec.msgpack.decode(legacy_payload, type=BlockStored).locality is None
|
||||
new_payload = msgspec.msgpack.encode(_make_block_stored(locality="LOCAL"))
|
||||
assert msgspec.msgpack.decode(new_payload)["locality"] == "LOCAL"
|
||||
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockStored).medium == "GPU"
|
||||
|
||||
|
||||
def test_block_removed_locality_is_wire_compatible():
|
||||
legacy = _LegacyBlockRemoved(block_hashes=[_FAKE_HASH], medium="GPU")
|
||||
legacy_payload = msgspec.msgpack.encode(legacy)
|
||||
assert msgspec.msgpack.encode(_make_block_removed()) == legacy_payload
|
||||
assert msgspec.msgpack.decode(legacy_payload, type=BlockRemoved).locality is None
|
||||
new_payload = msgspec.msgpack.encode(_make_block_removed(locality="REMOTE"))
|
||||
assert msgspec.msgpack.decode(new_payload)["locality"] == "REMOTE"
|
||||
assert msgspec.msgpack.decode(new_payload, type=_LegacyBlockRemoved).medium == "GPU"
|
||||
|
||||
Reference in New Issue
Block a user