[KV Connector] Support heterogeneous TP sharing in Mooncake Store Connector (#53129)

Signed-off-by: Guanyi Chen <939416532@qq.com>
Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
Signed-off-by: Zheng Zhou <zhouzh93@mail2.sysu.edu.cn>
Signed-off-by: zane <zhouzh93@mail2.sysu.edu.cn>
Co-authored-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
周峥
2026-09-01 10:18:06 -07:00
committed by GitHub
co-authored by Guanyi Chen
parent 2fe5cef35e
commit c866ba9d11
8 changed files with 1771 additions and 366 deletions
@@ -48,6 +48,31 @@ def _make_vllm_config():
)
@pytest.mark.parametrize(
"extra_config",
[
{"store_tp_size": 4},
{"enable_store_tp_lcm": True, "prefill_tp_sizes": [6, 4]},
{"prefill_tp_sizes": [4, 2]},
{"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 0]},
None,
],
)
def test_store_tp_does_not_override_backend_kv_cache_layout(extra_config):
vllm_config = create_vllm_config(
kv_connector="MooncakeStoreConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
assert (
mooncake_store_connector.MooncakeStoreConnector.get_required_kvcache_layout(
vllm_config
)
is None
)
def _make_kv_cache_config() -> KVCacheConfig:
"""Single-group full-attention KVCacheConfig — enough for the connector
constructor's validate() pass."""
@@ -90,15 +115,18 @@ def test_scheduler_requires_align_mode_for_mamba():
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
def _make_block_stored() -> BlockStored:
def _make_block_stored(
block_hash: bytes = b"hash", group_idx: int | None = None
) -> BlockStored:
return BlockStored(
block_hashes=[b"hash"],
block_hashes=[block_hash],
parent_block_hash=None,
token_ids=[1, 2, 3],
block_size=16,
lora_id=None,
medium="cpu",
lora_name=None,
group_idx=group_idx,
)
@@ -169,7 +197,7 @@ def test_worker_methods_delegate_to_store_worker():
assert invalid_block_ids == {3, 4}
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -184,7 +212,7 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_events.return_value = []
mock_worker_cls.return_value.enable_kv_events = False
assert connector.get_kv_connector_kv_cache_events() is None
@@ -248,6 +276,9 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
)
mock_worker_cls.return_value.get_kv_events.return_value = [event]
mock_worker_cls.return_value.enable_kv_events = True
mock_worker_cls.return_value.kv_send_thread = MagicMock()
mock_worker_cls.return_value.group_tp_replication_factors = (1,)
kv_events = connector.get_kv_connector_kv_cache_events()
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
@@ -255,6 +286,119 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
assert kv_events.get_all_events() == [event]
def test_get_kv_connector_kv_cache_events_keeps_empty_worker_contribution():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
worker = mock_worker_cls.return_value
worker.enable_kv_events = True
worker.kv_send_thread = MagicMock()
worker.group_tp_replication_factors = (2,)
worker.get_kv_events.return_value = []
kv_events = connector.get_kv_connector_kv_cache_events()
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
assert kv_events.get_number_of_workers() == 1
assert kv_events.get_all_events() == []
def _aggregate_store_events(
worker_events: list[list[BlockStored]],
replication_factors: tuple[int, ...],
) -> list[BlockStored]:
combined = mooncake_store_connector.MooncakeStoreKVEvents(
num_workers=1,
group_tp_replication_factors=replication_factors,
)
combined.add_events(worker_events[0])
for events in worker_events[1:]:
combined.add_events(events)
combined.increment_workers()
combined.aggregate()
return combined.get_all_events()
def test_gqa_store_events_follow_tp_replica_striping():
block_a = _make_block_stored(b"a", group_idx=0)
block_b = _make_block_stored(b"b", group_idx=0)
events = _aggregate_store_events(
[[block_a], [block_b], [block_a], [block_b]],
replication_factors=(2,),
)
assert events == [block_a, block_b]
def test_mqa_store_events_require_one_replica():
blocks = [_make_block_stored(bytes([index]), group_idx=0) for index in range(4)]
events = _aggregate_store_events(
[[block] for block in blocks],
replication_factors=(4,),
)
assert events == blocks
def test_store_events_require_every_non_replicated_worker():
complete = _make_block_stored(b"complete", group_idx=0)
incomplete = _make_block_stored(b"incomplete", group_idx=0)
events = _aggregate_store_events(
[
[complete, incomplete],
[complete],
[complete],
[complete],
],
replication_factors=(1,),
)
assert events == [complete]
def test_store_events_reject_incomplete_replica_coverage():
complete = _make_block_stored(b"complete", group_idx=0)
incomplete = _make_block_stored(b"incomplete", group_idx=0)
events = _aggregate_store_events(
[[complete, incomplete], [complete], [], []],
replication_factors=(2,),
)
assert events == [complete]
def test_store_events_use_each_groups_replication_factor():
full_attention = _make_block_stored(b"full", group_idx=0)
replicated_attention = _make_block_stored(b"replicated", group_idx=1)
events = _aggregate_store_events(
[
[full_attention, replicated_attention],
[full_attention],
[full_attention, replicated_attention],
[full_attention],
],
replication_factors=(1, 2),
)
assert events == [full_attention, replicated_attention]
def test_update_connector_output_and_take_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -284,6 +428,28 @@ def test_update_connector_output_and_take_events():
assert connector._kv_cache_events is None
def test_store_events_persist_across_polls():
connector = object.__new__(mooncake_store_connector.MooncakeStoreConnector)
connector.connector_scheduler = MagicMock()
connector._kv_cache_events = None
complete = _make_block_stored(b"complete", group_idx=0)
pending = _make_block_stored(b"pending", group_idx=0)
def poll(worker_events: list[BlockStored]) -> list[BlockStored]:
events = mooncake_store_connector.MooncakeStoreKVEvents(
num_workers=4,
group_tp_replication_factors=(2,),
)
events.add_events(worker_events)
connector.update_connector_output(KVConnectorOutput(kv_cache_events=events))
return list(connector.take_events())
assert poll([complete, pending]) == []
assert poll([complete]) == [complete]
assert poll([pending, pending]) == [pending]
assert connector._kv_cache_events is None
# ============================================================
# reset_cache() — RL hard-reset path via typed LookupKey protocol
# ============================================================
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Mooncake Store payload layouts."""
import ctypes
import random
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
LBHNCStoreLayout,
LBNHCStoreLayout,
RankLocalStoreLayout,
)
from vllm.utils.math_utils import cdiv
BLOCK_SIZE = 128
def _rank_local_layout(num_regions: int, num_block_lens: int) -> RankLocalStoreLayout:
metadata = KeyMetadata("test-model", 1, 0, 0, 0)
layout = RankLocalStoreLayout(metadata, BLOCK_SIZE, BLOCK_SIZE)
layout.set_kv_caches_base_addr(
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
)
layout.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
return layout
def _rank_local_reference(
layout: RankLocalStoreLayout, start: int, end: int, block_ids: list[int]
) -> tuple[list[int], list[int], int]:
block_id = block_ids[start // layout.block_size]
length = len(layout.block_len)
addrs = [
base_addr + block_id * layout.block_len[index % length]
for index, base_addr in enumerate(layout.kv_caches_base_addr)
]
sizes = [
layout.block_len[index % length] * cdiv(end - start, layout.block_size)
for index in range(len(layout.kv_caches_base_addr))
]
return addrs, sizes, block_id
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
def test_rank_local_descriptors_match_reference(num_regions: int, num_block_lens: int):
layout = _rank_local_layout(num_regions, num_block_lens)
rng = random.Random(0)
block_ids = [rng.randrange(0, 1 << 20) for _ in range(300)]
chunks = []
block = 0
while block < len(block_ids) - 4:
span = rng.choice([1, 1, 1, 2, 4])
chunks.append((block * BLOCK_SIZE, (block + span) * BLOCK_SIZE))
block += span + rng.choice([0, 1])
addrs, sizes, selected_blocks = layout.prepare_values(
chunks, block_ids, [0] * len(chunks)
)
for chunk, chunk_addrs, chunk_sizes, block_id in zip(
chunks, addrs, sizes, selected_blocks, strict=True
):
assert (chunk_addrs, chunk_sizes, block_id) == _rank_local_reference(
layout, *chunk, block_ids
)
assert all(type(addr) is int for addr in chunk_addrs)
assert type(block_id) is int
def test_rank_local_descriptors_handle_empty_and_invalid_chunks():
layout = _rank_local_layout(4, 4)
assert layout.prepare_values([], [1, 2, 3], []) == ([], [], [])
with pytest.raises(AssertionError):
layout.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1], [0])
@pytest.mark.parametrize(
("layout_cls", "store_format"),
[
(LBHNCStoreLayout, "tp_shared_lbhnc"),
(LBNHCStoreLayout, "tp_shared_lbnhc"),
],
)
def test_tp_shared_layout_owns_store_namespace(layout_cls, store_format):
assert (
layout_cls.shared_namespace(4, 2)
== f"@store_tp:4@store_pp:2@store_format:{store_format}"
)
def test_rank_local_database_api_rejects_tp_shared_layout():
metadata = KeyMetadata("test-model", 0, 0, 0, 0)
layout = LBHNCStoreLayout(
metadata,
block_size=16,
hash_block_size=16,
local_tp_size=2,
store_tp_size=4,
tp_rank=0,
num_kv_heads=8,
)
database = ChunkedTokenDatabase(metadata, 16, store_layout=layout)
with pytest.raises(RuntimeError, match="rank-local"):
database.prepare_value_for_block(0)
@pytest.mark.parametrize(
("layout_cls", "producer_strides", "consumer_strides"),
[
(LBHNCStoreLayout, (128, 64, 4, 1), (256, 64, 4, 1)),
(LBNHCStoreLayout, (128, 4, 8, 1), (256, 4, 16, 1)),
],
)
def test_tp_shared_layout_round_trip_across_tp_sizes(
layout_cls, producer_strides, consumer_strides
):
block_size = 16
shape = (1, 2, block_size, 4)
stored: dict[int, bytes] = {}
producer_tensors = []
for tp_rank in range(4):
tensor = torch.empty_strided(shape, producer_strides, dtype=torch.float16)
tensor.copy_(
torch.arange(128, dtype=torch.float16).view(shape) + tp_rank * 1000
)
producer_tensors.append(tensor)
metadata = KeyMetadata("test-model", tp_rank, 0, 0, 0)
layout = layout_cls(
metadata,
block_size,
block_size,
local_tp_size=4,
store_tp_size=4,
tp_rank=tp_rank,
num_kv_heads=8,
)
layout.register_kv_caches([tensor], 1)
addrs, sizes, _ = layout.prepare_values([(0, block_size)], [0], [tp_rank])
stored[tp_rank] = b"".join(
ctypes.string_at(addr, size)
for addr, size in zip(addrs[0], sizes[0], strict=True)
)
for tp_rank in range(2):
tensor = torch.empty_strided(
(1, 4, block_size, 4), consumer_strides, dtype=torch.float16
)
tensor.zero_()
metadata = KeyMetadata("test-model", tp_rank, 0, 0, 0)
layout = layout_cls(
metadata,
block_size,
block_size,
local_tp_size=2,
store_tp_size=4,
tp_rank=tp_rank,
num_kv_heads=8,
)
layout.register_kv_caches([tensor], 1)
shard_ids = layout.local_shard_ids
addrs, sizes, _ = layout.prepare_values(
[(0, block_size)] * len(shard_ids), [0], shard_ids
)
for shard_id, shard_addrs, shard_sizes in zip(
shard_ids, addrs, sizes, strict=True
):
offset = 0
for addr, size in zip(shard_addrs, shard_sizes, strict=True):
ctypes.memmove(addr, stored[shard_id][offset : offset + size], size)
offset += size
expected = torch.cat(producer_tensors[tp_rank * 2 : tp_rank * 2 + 2], dim=1)
torch.testing.assert_close(tensor, expected)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ChunkedTokenDatabase.prepare_values."""
import random
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
)
from vllm.utils.math_utils import cdiv
BLOCK_SIZE = 128
def _reference_prepare_value(
db: ChunkedTokenDatabase, start: int, end: int, block_ids: list[int]
) -> tuple[list[int], list[int], int]:
"""Compute a token range with the original scalar implementation."""
addr_list = []
size_list = []
block_id = block_ids[start // db.block_size]
length = len(db.block_len)
for index, base_addr in enumerate(db.kv_caches_base_addr):
addr = base_addr + block_id * db.block_len[index % length]
assert (end - start) % db.block_size == 0
size = db.block_len[index % length] * cdiv(end - start, db.block_size)
addr_list.append(addr)
size_list.append(size)
return addr_list, size_list, block_id
def _make_db(num_regions: int, num_block_lens: int) -> ChunkedTokenDatabase:
md = KeyMetadata(model_name="t", tp_rank=1, pcp_rank=0, dcp_rank=0, pp_rank=0)
db = ChunkedTokenDatabase(md, BLOCK_SIZE)
db.set_kv_caches_base_addr(
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
)
# Exercise repeated block lengths when there are more cache regions.
db.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
return db
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
def test_prepare_values_matches_reference(num_regions: int, num_block_lens: int):
db = _make_db(num_regions, num_block_lens)
rng = random.Random(0)
n_blocks = 300
block_ids = [rng.randrange(0, 1 << 20) for _ in range(n_blocks)]
chunks = []
b = 0
while b < n_blocks - 4:
span = rng.choice([1, 1, 1, 2, 4])
chunks.append((b * BLOCK_SIZE, (b + span) * BLOCK_SIZE))
b += span + rng.choice([0, 1])
addrs, sizes, bids = db.prepare_values(chunks, block_ids)
assert len(addrs) == len(sizes) == len(bids) == len(chunks)
for (start, end), addr, size, bid in zip(chunks, addrs, sizes, bids):
ref_addr, ref_size, ref_bid = _reference_prepare_value(
db, start, end, block_ids
)
assert addr == ref_addr
assert size == ref_size
assert bid == ref_bid
# Native bindings require Python ints rather than numpy scalars.
assert all(type(a) is int for a in addr)
assert type(bid) is int
def test_prepare_value_single_matches_reference():
db = _make_db(8, 8)
block_ids = list(range(64))
got = db.prepare_value(5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids)
assert got == _reference_prepare_value(
db, 5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids
)
def test_prepare_values_empty():
db = _make_db(4, 4)
assert db.prepare_values([], [1, 2, 3]) == ([], [], [])
def test_prepare_values_rejects_unaligned_chunk():
db = _make_db(4, 4)
with pytest.raises(AssertionError):
db.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1])
@@ -17,6 +17,7 @@ import pytest
import torch
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.distributed.kv_events import KVEventAggregator
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
rdma_utils,
)
@@ -30,9 +31,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BlobBlockHashes,
ChunkedTokenDatabase,
KeyMetadata,
LBHNCStoreLayout,
LBNHCStoreLayout,
LoadSpec,
MooncakeLookupResult,
PoolKey,
RankLocalStoreLayout,
ReqMeta,
TailKeyBoundary,
)
@@ -44,8 +48,19 @@ from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheLayout,
RSWASpec,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
_TP_SHARED_NAMESPACE = "@store_tp:4@store_pp:1@store_format:tp_shared_lbhnc"
_TP_SHARED_LBNHC_NAMESPACE = "@store_tp:4@store_pp:1@store_format:tp_shared_lbnhc"
def _tp_shared_prefix(tp_rank: int) -> str:
return (
f"test-model{_TP_SHARED_NAMESPACE}@tp_rank:{tp_rank}"
"@pcp0@dcp0@pp_rank:0@group:0"
)
class _RecordingBlockHashes:
@@ -78,6 +93,46 @@ def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
)
def _make_tp_shared_db(
tp_rank: int = 0,
) -> tuple[ChunkedTokenDatabase, torch.Tensor]:
num_blocks = 2
local_heads = 4
block_size = 16
content_size = 4
strides = (
local_heads * block_size * content_size,
block_size * content_size,
content_size,
1,
)
tensor = torch.empty_strided(
(num_blocks, local_heads, block_size, content_size),
strides,
dtype=torch.float16,
)
metadata = KeyMetadata(
"test-model",
tp_rank,
0,
0,
0,
store_namespace=_TP_SHARED_NAMESPACE,
)
layout = LBHNCStoreLayout(
metadata,
block_size,
block_size,
local_tp_size=2,
store_tp_size=4,
tp_rank=tp_rank,
num_kv_heads=8,
)
db = ChunkedTokenDatabase(metadata, block_size, store_layout=layout)
layout.register_kv_caches([tensor], num_blocks)
return db, tensor
def _make_store_sending_thread(
store: MagicMock,
*,
@@ -243,11 +298,15 @@ def _make_vllm_config(
rank: int = 0,
decode_context_parallel_size: int = 1,
kv_role: str = "kv_both",
pipeline_parallel_size: int = 1,
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
) -> SimpleNamespace:
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
return SimpleNamespace(
model_config=_FakeModelConfig(),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
pipeline_parallel_size=pipeline_parallel_size,
rank=rank,
data_parallel_index=0,
decode_context_parallel_size=decode_context_parallel_size,
@@ -256,7 +315,7 @@ def _make_vllm_config(
kv_transfer_config=_FakeKVTransferConfig(
kv_role=kv_role, extra_config=extra_config
),
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
cache_config=cache_config,
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
)
@@ -334,6 +393,138 @@ def test_pool_key_to_string_without_prefix_is_unchanged():
)
def test_pool_key_store_tp_namespace():
key = PoolKey(
KeyMetadata(
"test-model",
2,
0,
0,
0,
store_namespace=_TP_SHARED_NAMESPACE,
),
"deadbeef",
)
assert key.to_string() == f"{_tp_shared_prefix(2)}@deadbeef"
def test_tp_shared_sending_writes_each_store_shard():
store = MagicMock()
store.batch_is_exist.return_value = [0, 0, 0, 0]
store.batch_put_from_multi_buffers.return_value = [1, 1, 1, 1]
db, _ = _make_tp_shared_db()
thread = _make_store_sending_thread(
store,
token_databases=[db],
replicate_config=SimpleNamespace(),
)
req = _make_store_req("req", [b"h0", b"h1"])
_run_store_req(thread, req)
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
f"{_tp_shared_prefix(0)}@6830",
f"{_tp_shared_prefix(1)}@6830",
f"{_tp_shared_prefix(0)}@6831",
f"{_tp_shared_prefix(1)}@6831",
]
assert len(addrs) == len(sizes) == 4
def test_tp_shared_kv_events_aggregate_once_per_logical_block():
aggregator = KVEventAggregator(num_workers=2)
for tp_rank in range(2):
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
db, _ = _make_tp_shared_db(tp_rank)
thread = _make_store_sending_thread(
store,
token_databases=[db],
tp_rank=tp_rank,
replicate_config=SimpleNamespace(),
)
thread.enable_kv_event = True
_run_store_req(thread, _make_event_store_req(16))
events = thread.get_kv_events()
assert len(events) == 1
aggregator.add_events(events)
assert len(aggregator.get_common_events()) == 1
def test_tp_shared_kv_event_waits_for_all_missing_shards():
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, -1]
db, _ = _make_tp_shared_db()
thread = _make_store_sending_thread(
store,
token_databases=[db],
replicate_config=SimpleNamespace(),
)
thread.enable_kv_event = True
_run_store_req(thread, _make_event_store_req(16))
assert thread.get_kv_events() == []
def test_tp_shared_kv_event_counts_existing_shards_as_satisfied():
store = MagicMock()
store.batch_is_exist.return_value = [1, 0]
store.batch_put_from_multi_buffers.return_value = [256]
db, _ = _make_tp_shared_db()
thread = _make_store_sending_thread(
store,
token_databases=[db],
replicate_config=SimpleNamespace(),
)
thread.enable_kv_event = True
_run_store_req(thread, _make_event_store_req(16))
assert len(thread.get_kv_events()) == 1
def test_tp_shared_receiving_reads_each_local_store_shard():
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [1, 1, 1, 1]
db, _ = _make_tp_shared_db()
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["layer0"], spec)],
scheduler_block_size=16,
hash_block_size=16,
)
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
store=store,
token_databases=[db],
block_size=16,
tp_rank=0,
ready_event=threading.Event(),
coord=coord,
)
thread.request_queue.task_done = MagicMock()
thread._handle_request(_make_load_req("req", [b"h0", b"h1"], token_len=32))
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
assert keys == [
f"{_tp_shared_prefix(0)}@6830",
f"{_tp_shared_prefix(1)}@6830",
f"{_tp_shared_prefix(0)}@6831",
f"{_tp_shared_prefix(1)}@6831",
]
assert len(addrs) == len(sizes) == 4
def test_pool_key_cache_prefix_namespaces_and_disambiguates():
"""A non-empty cache_prefix is prepended, and two instances with
different prefixes never collide on identical block hashes."""
@@ -1155,8 +1346,7 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
)
db0.set_kv_caches_base_addr([0x1000])
db0.set_block_len([256])
db0.prepare_values = MagicMock(wraps=db0.prepare_values)
db0.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
db0.store_layout.prepare_values = MagicMock(wraps=db0.store_layout.prepare_values)
db1 = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
@@ -1164,8 +1354,7 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
)
db1.set_kv_caches_base_addr([0x2000])
db1.set_block_len([512])
db1.prepare_values = MagicMock(wraps=db1.prepare_values)
db1.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
db1.store_layout.prepare_values = MagicMock(wraps=db1.store_layout.prepare_values)
thread = _make_store_sending_thread(
store,
@@ -1183,10 +1372,12 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
),
)
db0.prepare_value.assert_not_called()
db1.prepare_value.assert_not_called()
db0.prepare_values.assert_called_once_with([(0, 16), (32, 48)], [1, 2, 3])
db1.prepare_values.assert_called_once_with([(16, 32), (32, 48)], [3, 2, 1])
db0.store_layout.prepare_values.assert_called_once_with(
[(0, 16), (32, 48)], [1, 2, 3], [0, 0]
)
db1.store_layout.prepare_values.assert_called_once_with(
[(16, 32), (32, 48)], [3, 2, 1], [0, 0]
)
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
assert [key.rsplit("@", 1)[-1] for key in keys] == [
@@ -1836,8 +2027,10 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
assert second_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
block_len = thread.token_databases[0].block_len[0]
layout = thread.token_databases[0].store_layout
assert isinstance(layout, RankLocalStoreLayout)
base_addr = layout.kv_caches_base_addr[0]
block_len = layout.block_len[0]
assert first_addrs == [[base_addr], [base_addr + block_len]]
assert second_addrs == [[base_addr + 2 * block_len]]
expected_size = block_len
@@ -2004,6 +2197,294 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
assert w.disk_offload_buffer_budget_bytes is None
def test_worker_enables_lbhnc_store_tp_layout(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch, tp_size=2)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
w = worker.MooncakeStoreWorker(
_make_vllm_config(extra_config={"store_tp_size": 4}),
_make_kv_cache_config(),
)
assert w.store_tp_size == 4
assert w.token_dbs[0].metadata.store_namespace == _TP_SHARED_NAMESPACE
assert isinstance(w.token_dbs[0].store_layout, LBHNCStoreLayout)
assert w.token_dbs[0].store_layout.local_shard_ids == (0, 1)
assert len(w._lookup_key_prefixes[0]) == 4
def test_worker_enables_lbnhc_store_tp_layout(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch, tp_size=2)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
with patch.object(worker.logger, "warning_once") as warning_once:
w = worker.MooncakeStoreWorker(
_make_vllm_config(
extra_config={"store_tp_size": 4},
kv_cache_layout=KVCacheLayout.LBNHC,
),
_make_kv_cache_config(),
)
warning_once.assert_called_once()
assert w.store_tp_size == 4
assert w.token_dbs[0].metadata.store_namespace == _TP_SHARED_LBNHC_NAMESPACE
assert isinstance(w.token_dbs[0].store_layout, LBNHCStoreLayout)
assert w.token_dbs[0].store_layout.local_shard_ids == (0, 1)
assert len(w._lookup_key_prefixes[0]) == 4
def test_tp_sharded_layout_rejects_full_attention_subclasses():
spec = RSWASpec(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=None,
rswa_window=32,
)
store_worker = object.__new__(worker.MooncakeStoreWorker)
store_worker.pcp_size = 1
store_worker.dcp_size = 1
store_worker._kv_cache_groups = [SimpleNamespace(kv_cache_spec=spec)]
assert not store_worker._supports_tp_sharded_store_layout(LBHNCStoreLayout, {})
@pytest.mark.parametrize(
("extra_config", "expected"),
[
({"store_tp_size": 4}, 4),
(
{
"enable_store_tp_lcm": True,
"prefill_tp_sizes": [3, 4, 6],
},
12,
),
(
{
"enable_store_tp_lcm": False,
"prefill_tp_sizes": [4, 2],
},
None,
),
({"enable_store_tp_lcm": True, "prefill_tp_sizes": []}, None),
({"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 0]}, None),
({"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 2.0]}, None),
({"store_tp_size": 0}, None),
({"store_tp_size": -1}, None),
({"store_tp_size": True}, None),
({"store_tp_size": 4.0}, None),
],
)
def test_resolve_store_tp_size(extra_config: dict[str, object], expected: int | None):
assert worker.resolve_store_tp_size(extra_config) == expected
def test_lcm_store_tp_gives_prefill_and_decode_common_namespace(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
lcm_config = {
"enable_store_tp_lcm": True,
"prefill_tp_sizes": [4, 2],
}
workers = []
topologies = [
(4, 2, "kv_both", (2,)),
(2, 1, "kv_both", (2, 3)),
(2, 1, "kv_consumer", (2, 3)),
]
for tp_size, tp_rank, kv_role, expected_shards in topologies:
_patch_worker_runtime(
monkeypatch,
tp_size=tp_size,
tp_rank=tp_rank,
)
extra_config = dict(lcm_config)
if kv_role == "kv_consumer":
extra_config["save_decode_cache"] = True
store_worker = worker.MooncakeStoreWorker(
_make_vllm_config(extra_config=extra_config, kv_role=kv_role),
_make_kv_cache_config(),
)
assert store_worker.store_tp_size == 4
assert isinstance(store_worker.token_dbs[0].store_layout, LBHNCStoreLayout)
assert store_worker.token_dbs[0].store_layout.local_shard_ids == expected_shards
workers.append(store_worker)
shard_keys = [
store_worker.token_dbs[0].store_layout.key_for(2, BlockHash(b"h"))
for store_worker in workers
]
assert len(set(shard_keys)) == 1
assert workers[-1].can_put
@pytest.mark.parametrize(
("tp_size", "prefill_tp_sizes"),
[
(3, [4, 2]),
(2, [4, 3]),
(2, []),
(2, [4, 0]),
],
)
def test_lcm_store_tp_falls_back_when_topology_is_incompatible(
tmp_path,
monkeypatch,
tp_size: int,
prefill_tp_sizes: list[int],
):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch, tp_size=tp_size)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
store_worker = worker.MooncakeStoreWorker(
_make_vllm_config(
extra_config={
"enable_store_tp_lcm": True,
"prefill_tp_sizes": prefill_tp_sizes,
}
),
_make_kv_cache_config(),
)
assert store_worker.store_tp_size is None
assert type(store_worker.token_dbs[0]) is ChunkedTokenDatabase
assert "@store_format:rank_local_tp" in store_worker.token_dbs[0].key_for(
BlockHash(b"h")
)
@pytest.mark.parametrize("store_tp_size", [1, 3, 6])
def test_worker_keeps_rank_local_layout_for_unsupported_store_tp(
tmp_path,
monkeypatch,
store_tp_size: int,
):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch, tp_size=2)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
w = worker.MooncakeStoreWorker(
_make_vllm_config(
extra_config={"store_tp_size": store_tp_size},
kv_cache_layout=KVCacheLayout.LBNHC,
),
_make_kv_cache_config(),
)
assert w.store_tp_size is None
assert type(w.token_dbs[0]) is ChunkedTokenDatabase
key = w.token_dbs[0].key_for(BlockHash(b"h"))
assert "@store_tp:" not in key
assert "@store_pp:1" in key
assert "@store_format:rank_local_tp2_layout_LBNHC" in key
def test_store_tp_opt_in_isolates_different_pp_sizes(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch, tp_size=2)
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
keys = []
for pp_size in (1, 2):
w = worker.MooncakeStoreWorker(
_make_vllm_config(
extra_config={"store_tp_size": 4},
pipeline_parallel_size=pp_size,
),
_make_kv_cache_config(),
)
layout = w.token_dbs[0].store_layout
keys.append(layout.key_for(layout.local_shard_ids[0], BlockHash(b"h")))
assert "@store_pp:1" in keys[0]
assert "@store_pp:2" in keys[1]
assert keys[0] != keys[1]
def test_save_decode_cache_keeps_transfer_path_enabled(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -2185,6 +2666,58 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
)
def test_mqa_p4_to_d2_uses_shared_rank_zero_namespace(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"master_server_address": "10.0.0.7:50051",
},
),
)
block_hash = BlockHash(b"hash")
put_keys: set[str] = set()
get_keys: set[str] = set()
for tp_size, target in ((4, put_keys), (2, get_keys)):
for tp_rank in range(tp_size):
_patch_worker_runtime(monkeypatch, tp_rank=tp_rank, tp_size=tp_size)
w = worker.MooncakeStoreWorker(
_make_vllm_config(
rank=tp_rank,
extra_config={"store_tp_size": 4},
),
_make_kv_cache_config(),
)
assert w.store_tp_size is None
assert w.token_dbs[0].metadata.tp_rank == 0
key = w.token_dbs[0].key_for(block_hash)
if tp_size == 4:
put_step = w._group_tp_replication_factors[0]
if tp_rank % put_step == 0:
target.add(key)
else:
target.add(key)
assert (
put_keys
== get_keys
== {
(
"test-model@store_pp:1@store_format:tp_shared_mqa"
"@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@68617368"
)
}
)
def test_requester_worker_group_semantics_falls_back_without_group_ids(
tmp_path,
monkeypatch,
@@ -2396,9 +2929,11 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, *_args: ([256] * len(keys))
)
def successful_put(keys, addrs, sizes, *_args):
return [256] * len(keys)
store.batch_put_from_multi_buffers.side_effect = successful_put
full_spec = FullAttentionSpec(
block_size=32, num_kv_heads=8, head_size=64, dtype=None
@@ -2754,6 +3289,7 @@ def _make_bare_worker(
worker.recv_request_queue = queue.Queue()
worker.finished_store_req = set()
worker.tp_size = 1
worker.store_tp_size = None
worker.num_kv_head = 1
worker.pp_size = 1
# Minimal single-full-attention-group config so the coordinator-based
@@ -2812,6 +3348,40 @@ def test_lookup_key_prefixes_cover_dcp_rank_namespaces():
)
def test_lookup_key_prefixes_cover_store_tp_shards():
worker = _make_bare_worker()
worker.store_tp_size = 4
metadata = KeyMetadata(
"test-model",
0,
0,
0,
0,
group_id=0,
store_namespace=_TP_SHARED_NAMESPACE,
)
worker.token_dbs = [
ChunkedTokenDatabase(
metadata,
block_size=16,
store_layout=LBHNCStoreLayout(
metadata,
block_size=16,
hash_block_size=16,
local_tp_size=2,
store_tp_size=4,
tp_rank=0,
num_kv_heads=8,
),
)
]
worker._init_lookup_key_prefixes()
assert worker._lookup_key_prefixes[0] == (
*(_tp_shared_prefix(tp_rank) for tp_rank in range(4)),
)
def test_lookup_key_prefixes_cover_pcp_rank_namespaces():
worker = _make_bare_worker()
worker.tp_size = 4