[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-05-19 01:07:46 -07:00
committed by GitHub
parent f34623bf3c
commit 056bc2e166
10 changed files with 1834 additions and 445 deletions
@@ -9,12 +9,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
connector,
worker,
connector as mooncake_store_connector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeStoreConnectorMetadata,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
from vllm.v1.outputs import KVConnectorOutput
from .utils import create_vllm_config
@@ -27,6 +32,17 @@ def _make_vllm_config():
)
def _make_kv_cache_config() -> KVCacheConfig:
"""Single-group full-attention KVCacheConfig — enough for the connector
constructor's validate() pass."""
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
return KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[KVCacheTensor(size=8192, shared_by=["layer0"])],
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
)
def _make_block_stored() -> BlockStored:
return BlockStored(
block_hashes=[b"hash"],
@@ -41,6 +57,7 @@ def _make_block_stored() -> BlockStored:
def test_scheduler_role_initializes_store_scheduler_only():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -53,75 +70,19 @@ def test_scheduler_role_initializes_store_scheduler_only():
"connector.MooncakeStoreWorker"
) as mock_worker,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
mock_scheduler.assert_called_once_with(vllm_config)
mock_scheduler.assert_called_once_with(vllm_config, kv_cache_config)
mock_worker.assert_not_called()
assert conn.connector_scheduler is mock_scheduler.return_value
assert conn.connector_worker is None
def test_worker_role_initializes_store_worker_on_rank0():
vllm_config = _make_vllm_config()
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreScheduler"
) as mock_scheduler,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
mock_scheduler.assert_not_called()
mock_worker.assert_called_once_with(vllm_config)
assert conn.connector_scheduler is None
assert conn.connector_worker is mock_worker.return_value
def test_worker_role_initializes_on_nonzero_rank():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.rank = 1
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker,
):
connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
mock_worker.assert_called_once_with(vllm_config)
def test_lookup_rpc_path_uses_data_parallel_index_in_dense_dp():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.data_parallel_rank = 0
vllm_config.parallel_config.data_parallel_index = 3
path = worker.get_zmq_rpc_path_lookup(vllm_config)
assert path.endswith("_dp_rank3")
def test_lookup_rpc_path_uses_local_rank_when_local_engines_only():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.data_parallel_index = 7
vllm_config.parallel_config.data_parallel_rank_local = 1
vllm_config.parallel_config.data_parallel_hybrid_lb = True
path = worker.get_zmq_rpc_path_lookup(vllm_config)
assert path.endswith("_dp_rank1")
assert connector.connector_scheduler is mock_scheduler.return_value
assert connector.connector_worker is None
def test_worker_methods_delegate_to_store_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
kv_caches = {"layer0": MagicMock()}
metadata = MooncakeStoreConnectorMetadata(set(), set())
finished_req_ids = {"req-1"}
@@ -133,22 +94,25 @@ def test_worker_methods_delegate_to_store_worker():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
worker_inst = mock_worker_cls.return_value
worker_inst.get_finished.return_value = ({"req-1"}, {"req-2"})
conn.bind_connector_metadata(metadata)
worker = mock_worker_cls.return_value
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
connector.bind_connector_metadata(metadata)
conn.register_kv_caches(kv_caches)
result = conn.get_finished(finished_req_ids)
connector.register_kv_caches(kv_caches)
result = connector.get_finished(finished_req_ids)
worker_inst.register_kv_caches.assert_called_once_with(kv_caches)
worker_inst.get_finished.assert_called_once_with(finished_req_ids, metadata)
worker.register_kv_caches.assert_called_once_with(kv_caches)
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
assert result == ({"req-1"}, {"req-2"})
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -157,14 +121,17 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_events.return_value = []
assert conn.get_kv_connector_kv_cache_events() is None
assert connector.get_kv_connector_kv_cache_events() is None
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
event = _make_block_stored()
with (
@@ -174,12 +141,14 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_events.return_value = [event]
kv_events = conn.get_kv_connector_kv_cache_events()
kv_events = connector.get_kv_connector_kv_cache_events()
assert isinstance(kv_events, connector.MooncakeStoreKVEvents)
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
assert kv_events.get_number_of_workers() == 1
assert kv_events.get_all_events() == [event]
@@ -187,6 +156,7 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
def test_prefer_cross_layer_blocks_from_config():
# Default: disabled
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
patch(
@@ -194,8 +164,10 @@ def test_prefer_cross_layer_blocks_from_config():
"connector.MooncakeStoreScheduler"
),
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
assert conn.prefer_cross_layer_blocks is False
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert connector.prefer_cross_layer_blocks is False
# Enabled via config
vllm_config_enabled = create_vllm_config(
@@ -210,14 +182,15 @@ def test_prefer_cross_layer_blocks_from_config():
"connector.MooncakeStoreScheduler"
),
):
conn_enabled = connector.MooncakeStoreConnector(
vllm_config_enabled, KVConnectorRole.SCHEDULER
connector_enabled = mooncake_store_connector.MooncakeStoreConnector(
vllm_config_enabled, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert conn_enabled.prefer_cross_layer_blocks is True
assert connector_enabled.prefer_cross_layer_blocks is True
def test_register_cross_layers_kv_cache_delegates_to_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -226,18 +199,21 @@ def test_register_cross_layers_kv_cache_delegates_to_worker():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
fake_tensor = MagicMock()
fake_backend = MagicMock()
conn.register_cross_layers_kv_cache(fake_tensor, fake_backend)
connector.register_cross_layers_kv_cache(fake_tensor, fake_backend)
worker_inst = mock_worker_cls.return_value
worker_inst.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
worker = mock_worker_cls.return_value
worker.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
def test_update_connector_output_and_take_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
event = _make_block_stored()
with (
@@ -247,12 +223,14 @@ def test_update_connector_output_and_take_events():
"connector.MooncakeStoreScheduler"
),
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
kv_events = connector.MooncakeStoreKVEvents(num_workers=1)
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
kv_events.add_events([event])
conn.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
assert conn._kv_cache_events is kv_events
assert list(conn.take_events()) == [event]
assert conn._kv_cache_events is None
assert connector._kv_cache_events is kv_events
assert list(connector.take_events()) == [event]
assert connector._kv_cache_events is None
@@ -0,0 +1,302 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from math import lcm
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
)
from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
def _make_coord(groups, hash_block_size, use_eagle=False):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
the test fixtures."""
block_sizes = [g.kv_cache_spec.block_size for g in groups]
scheduler_block_size = lcm(*block_sizes)
return MooncakeStoreCoordinator(
groups,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
use_eagle=use_eagle,
)
# ----- ExternalCachedBlockPool -----
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
cmap = ExternalCachedBlockPool()
h = BlockHash(b"\xaa" * 4)
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
assert len(res) == 2
assert res[0] is not cmap.null_block
assert res[1] is not cmap.null_block
def test_external_cached_block_pool_hit_all_groups():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
assert len(res) == 2
assert res[0] is not cmap.null_block
assert res[1] is not cmap.null_block
def test_external_cached_block_pool_miss_one_group():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h))})
assert cmap.get_cached_block(h, [0, 1]) is None
def test_external_cached_block_pool_unknown_hash():
h_known = BlockHash(b"\x01" * 4)
h_unknown = BlockHash(b"\x02" * 4)
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
assert cmap.get_cached_block(h_unknown, [0]) is None
# ----- Helpers -----
def _full(block_size=16, sliding_window=None):
return FullAttentionSpec(
block_size=block_size,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=sliding_window,
)
def _swa(block_size=16, sliding_window=32):
return SlidingWindowSpec(
block_size=block_size,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=sliding_window,
)
def _hashes(n: int) -> list[BlockHash]:
return [BlockHash(bytes([i + 1]) * 4) for i in range(n)]
# ----- Single-group coordinator -----
def test_coordinator_single_full_attention_all_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
assert masks[0] == [True, True, True, True]
def test_coordinator_single_full_attention_partial_prefix():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 32
assert masks[0] == [True, True]
def test_coordinator_single_full_attention_no_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool(set())
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 0
assert masks[0] == []
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
"""SWA tautological-pool: hit_length spans full prefix, mask is
tail-window only."""
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4) # 4 chunks * 16 tokens
cmap = ExternalCachedBlockPool()
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
assert masks[0][-2:] == [True, True]
assert all(not m for m in masks[0][:-2])
# ----- Hybrid coordinator (single-group worker, multi-group coordinator) -----
def test_coordinator_hybrid_full_plus_swa_all_hit():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 64
def test_coordinator_hybrid_hole_in_full_clips_both():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
exists |= {(1, bytes(h)) for h in hs}
cmap = ExternalCachedBlockPool(exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 16
def test_coordinator_group_block_size_double_hash():
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
hashes merge before pool lookup."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _full(32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32))
exists = {(0, bytes(h)) for h in hs}
exists |= {(1, bytes(bh)) for bh in big_hashes}
cmap = ExternalCachedBlockPool(exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 64
assert hit % 32 == 0
# ----- store_mask -----
def test_store_mask_full_attention_all_true():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(64)
assert masks == ([True, True, True, True],)
def test_store_mask_zero_aligned_returns_empty_per_group():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(0)
assert masks == ([], [])
def test_store_mask_swa_only_window_around_each_lcm_boundary():
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
aligned=64 the SWA group should mark exactly the blocks ending at 32
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
participate in any future hit."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
# Full-attn: 2 chunks * 32 tokens.
assert masks[0] == [True, True]
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
assert masks[1] == [False, False, False, True, False, False, False, True]
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=16)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
assert masks[0] == [True, True]
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
# Boundary at 64: chunks 6 and 7. Others stay False.
assert masks[1] == [False, False, True, True, False, False, True, True]
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
(aligned_len=512). Validates that the tile-once strategy produces the
expected per-segment tail-window pattern, repeated."""
full_mla = _full(block_size=256)
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
swa_4_sw16 = _swa(block_size=4, sliding_window=16)
swa_8_sw64 = _swa(block_size=8, sliding_window=64)
groups = [
KVCacheGroupSpec(["L0"], full_mla),
KVCacheGroupSpec(["L1"], swa_64_sw128),
KVCacheGroupSpec(["L2"], swa_64_sw512),
KVCacheGroupSpec(["L3"], swa_4_sw16),
KVCacheGroupSpec(["L4"], swa_8_sw64),
]
coord = _make_coord(groups, hash_block_size=4)
assert coord.lcm_block_size == 256
masks = coord.store_mask(512)
# Full-MLA: 2 chunks of 256, both stored.
assert masks[0] == [True, True]
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
# Per-segment template = [F,F,T,T]; tiled twice.
assert masks[1] == [False, False, True, True] * 2
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
assert masks[2] == [True] * 8
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
# Last 4 of each 64-chunk segment True.
assert masks[3] == ([False] * 60 + [True] * 4) * 2
# SWA(8, sw=64): tail = ceil(63/8) = 8; C = 256/8 = 32.
# Last 8 of each 32-chunk segment True.
assert masks[4] == ([False] * 24 + [True] * 8) * 2
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
"""When every non-full-attn group already aligns to lcm_block_size, the
fast path returns all-True without invoking find_longest_cache_hit."""
full = _full(block_size=64)
swa = _swa(block_size=64, sliding_window=128)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=64)
assert coord.lcm_block_size == 64
masks = coord.store_mask(256)
# Every block in every group is True — no sub-lcm filtering possible.
assert masks == ([True] * 4, [True] * 4)
def test_store_mask_fast_path_single_attention_group():
"""Two groups sharing the same SWA spec collapse to one attention group;
no lcm filter applies, every chunk is True."""
swa = _swa(block_size=16, sliding_window=32)
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=16)
assert len(coord.attention_groups) == 1
masks = coord.store_mask(64)
assert masks == ([True] * 4, [True] * 4)
@@ -0,0 +1,342 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
(SWA + Full) attention config, using a dict-backed mock store."""
import sys
import threading
import types
from unittest.mock import MagicMock, patch
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker as mooncake_store_worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
KVCacheStoreRecvingThread,
KVCacheStoreSendingThread,
)
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
SlidingWindowSpec,
)
class _DictStore:
"""In-memory MooncakeDistributedStore stand-in."""
def __init__(self):
self._data: dict[str, bytes] = {}
def setup(self, *_args, **_kwargs):
return 0
def register_buffer(self, addr, length):
return 0
def batch_is_exist(self, keys):
return [1 if k in self._data else 0 for k in keys]
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
for k in keys:
self._data[k] = b"x"
return [0] * len(keys)
def batch_get_into_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
return [0 if k in self._data else -1 for k in keys]
def _minimal_vllm_config(cache_block_size=16):
cfg = MagicMock()
cfg.cache_config.block_size = cache_block_size
cfg.cache_config.num_gpu_blocks = 4
cfg.cache_config.hash_block_size = None
cfg.cache_config.enable_prefix_caching = True
cfg.parallel_config.prefill_context_parallel_size = 1
cfg.parallel_config.decode_context_parallel_size = 1
cfg.parallel_config.pipeline_parallel_size = 1
cfg.parallel_config.world_size = 1
cfg.parallel_config.rank = 0
cfg.parallel_config.data_parallel_rank_local = 0
cfg.parallel_config.data_parallel_size_local = 1
cfg.kv_transfer_config.kv_role = "kv_both"
cfg.kv_transfer_config.kv_connector_extra_config = {}
cfg.kv_events_config = None
cfg.model_config.model = "/tmp/m"
cfg.model_config.use_mla = False
cfg.model_config.get_num_layers.return_value = 2
cfg.model_config.get_total_num_kv_heads.return_value = 8
cfg.model_config.max_model_len = 4096
cfg.scheduler_config.max_num_batched_tokens = 8192
# Without this, MagicMock's truthy use_eagle() triggers the coordinator's
# "use_eagle && nothing annotated → flag all groups" fallback.
cfg.speculative_config = None
return cfg
def _build_worker_with_dict_store(vllm_config, kv_cache_config, store):
"""Build a MooncakeStoreWorker patching all distributed dependencies."""
fake_mooncake_store = types.ModuleType("mooncake.store")
fake_mooncake_store.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined]
fake_mooncake_store.ReplicateConfig = MagicMock # type: ignore[attr-defined]
with (
patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}),
patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg,
):
sc = MCfg.load_from_env.return_value
sc.metadata_server = ""
sc.global_segment_size = 1 << 20
sc.local_buffer_size = 1 << 20
sc.protocol = "tcp"
sc.device_name = ""
sc.master_server_address = ""
sc.enable_offload = False
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_tensor_model_parallel_world_size",
return_value=1,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_pcp_group"
) as mock_pcp,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_dcp_group"
) as mock_dcp,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_ip",
return_value="127.0.0.1",
),
):
mock_pcp.return_value.world_size = 1
mock_dcp.return_value.world_size = 1
worker = mooncake_store_worker.MooncakeStoreWorker(
vllm_config, kv_cache_config=kv_cache_config
)
return worker
def test_e2e_swa_plus_full_save_then_lookup_hits():
"""
E2E: build a SWA+Full hybrid worker, save all blocks via the sending
thread (synchronously), then verify lookup returns the full hit length.
Also verify that evicting SWA's early blocks (outside its window) still
allows a full hit because the window covers the tail.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
swa = SlidingWindowSpec(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=32,
)
cfg = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(size=8192, shared_by=["L0"]),
KVCacheTensor(size=8192, shared_by=["L1"]),
],
kv_cache_groups=[
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], swa),
],
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
store = _DictStore()
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
worker.tp_size = 1
worker.pp_size = 1
worker.put_step = 1
worker.num_kv_head = 8
# Register kv_caches using mocked thread classes so register_kv_caches
# doesn't try to start real background threads (which set ready_event).
kv_caches = {
"L0": torch.zeros(2, 4, 8, 8, 64),
"L1": torch.zeros(2, 4, 8, 8, 64),
}
def _fake_thread_init(*args, **kwargs):
"""Mock thread that sets all threading.Event args so waits don't block."""
for v in list(args) + list(kwargs.values()):
if isinstance(v, threading.Event):
v.set()
m = MagicMock()
m.start = lambda: None
return m
with (
patch.object(
mooncake_store_worker,
"KVCacheStoreSendingThread",
side_effect=_fake_thread_init,
),
patch.object(
mooncake_store_worker,
"KVCacheStoreRecvingThread",
side_effect=_fake_thread_init,
),
):
worker.register_kv_caches(kv_caches)
# Now build a real sending thread (no .start()) over the worker's token_dbs
# and the dict-backed store, so _handle_request runs synchronously.
ready = threading.Event()
send_thread = KVCacheStoreSendingThread(
store=store,
token_databases=worker.token_dbs,
block_size=worker.block_size,
coord=worker.coord,
tp_rank=worker.tp_rank,
put_step=worker.put_step,
kv_role=worker.kv_role,
ready_event=ready,
enable_kv_event=False,
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
save_req = ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
can_save=True,
)
send_thread.add_stored_request("r0")
# Put the request in the queue so task_done() doesn't underflow.
send_thread.request_queue.put(save_req)
req = send_thread.request_queue.get()
send_thread._handle_request(req)
# Point worker.store at the dict store (the worker constructor captured
# the MagicMock; replace with the real dict store for lookup).
worker.store = store
# Both groups stored all 4 blocks -> full hit.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
swa_keys_outside_window = [
k
for k in list(store._data.keys())
if "@group:1" in k and (("@" + hs[0].hex()) in k or ("@" + hs[1].hex()) in k)
]
for k in swa_keys_outside_window:
del store._data[k]
# SWA window=32 -> only last 2 blocks must be present in SWA group.
# Full has all 4. Coordinator should still return 64.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
def test_recv_skips_swa_blocks_before_window():
"""Producer stored every block for both groups; consumer must only fetch
SWA blocks within the sliding window, not the head."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
swa = SlidingWindowSpec(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=32,
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], swa),
]
md0 = KeyMetadata("m", 0, 0, 0, 0, group_id=0)
md1 = KeyMetadata("m", 0, 0, 0, 0, group_id=1)
db_full = ChunkedTokenDatabase(md0, block_size=16, hash_block_size=16)
db_swa = ChunkedTokenDatabase(md1, block_size=16, hash_block_size=16)
db_full.set_kv_caches_base_addr([0])
db_full.set_block_len([1024])
db_swa.set_kv_caches_base_addr([1 << 20])
db_swa.set_block_len([1024])
requested_keys: list[str] = []
class _CapturingStore:
def batch_get_into_multi_buffers(self, keys, addrs, sizes):
requested_keys.extend(keys)
return [0] * len(keys)
ready = threading.Event()
coord = MooncakeStoreCoordinator(
groups, scheduler_block_size=16, hash_block_size=16
)
recv = KVCacheStoreRecvingThread(
store=_CapturingStore(),
token_databases=[db_full, db_swa],
block_size=16,
tp_rank=0,
ready_event=ready,
coord=coord,
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
req = ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
load_spec=LoadSpec(
vllm_cached_tokens=0, kvpool_cached_tokens=64, can_load=True, token_len=64
),
)
recv.request_queue.put(req)
recv._handle_request(recv.request_queue.get())
full_keys = [k for k in requested_keys if "@group:0" in k]
swa_keys = [k for k in requested_keys if "@group:1" in k]
# Full attention: load every block (4).
assert len(full_keys) == 4
# SWA: only the 2 tail-window blocks (hashes hs[2], hs[3]).
assert len(swa_keys) == 2
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
assert swa_hashes == {hs[2].hex(), hs[3].hex()}
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
must merge every 4 fine hashes into one chunk hash via
BlockHashListWithBlockSize."""
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
db.set_kv_caches_base_addr([0])
db.set_block_len([512])
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
assert len(out) == 2
assert out[0][0] == 0 and out[0][1] == 16
assert out[1][0] == 16 and out[1][1] == 32
# Each chunk's hash is the concatenation of 4 fine hashes.
expected0 = b"".join(fine_hashes[0:4]).hex()
expected1 = b"".join(fine_hashes[4:8]).hex()
assert out[0][2].chunk_hash == expected0
assert out[1][2].chunk_hash == expected1
@@ -0,0 +1,111 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
def _make_bare_scheduler() -> MooncakeStoreScheduler:
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.original_block_size = 16
scheduler._block_size = 16
scheduler._discard_partial_chunks = True
scheduler.load_specs = {}
scheduler._preempted_req_ids = set()
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
scheduler._request_trackers = {}
return scheduler
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[([2],)],
num_computed_tokens=[44],
),
num_scheduled_tokens={"req-0": 4},
scheduled_spec_decode_tokens=(
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
),
)
def _add_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
token_ids: list[int],
block_hashes: list[bytes],
prefill_end_tokens: int,
) -> None:
request = SimpleNamespace(
all_token_ids=token_ids,
block_hashes=block_hashes,
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=44,
allocated_block_ids=([0, 1],),
num_saved_tokens=32,
token_ids=token_ids[:44],
prefill_end_tokens=prefill_end_tokens,
)
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
# yet, so the tracker's token_len does not advance and num_tokens_to_save
# stays below chunk_boundary — the save is naturally skipped.
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(44)),
block_hashes=[b"h0", b"h1"],
prefill_end_tokens=48,
)
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=[101, 102, 103])
)
assert meta.requests == []
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 44
assert tracker.num_saved_tokens == 32
assert tracker.allocated_block_ids == ([0, 1, 2],)
def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=48,
)
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=None)
)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 48
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 48
assert tracker.num_saved_tokens == 48
@@ -19,7 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker as mooncake_store_worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
@@ -27,20 +30,37 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
)
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
return mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["layer0"], spec)],
scheduler_block_size=16,
hash_block_size=16,
)
def _make_store_sending_thread(
store: MagicMock,
*,
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
token_databases: list[ChunkedTokenDatabase] | None = None,
block_size: int = 16,
replicate_config: object | None = None,
) -> worker.KVCacheStoreSendingThread:
token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
)
token_database.set_kv_caches_base_addr([0x1000])
token_database.set_block_len([256])
thread = worker.KVCacheStoreSendingThread(
) -> mooncake_store_worker.KVCacheStoreSendingThread:
if coord is None:
coord = _default_send_coord()
if token_databases is None:
db = ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([256])
token_databases = [db]
thread = mooncake_store_worker.KVCacheStoreSendingThread(
store=store,
token_database=token_database,
block_size=16,
token_databases=token_databases,
block_size=block_size,
coord=coord,
tp_rank=0,
put_step=1,
kv_role="kv_producer",
@@ -55,18 +75,27 @@ def _make_store_recving_thread(
store: MagicMock,
*,
disk_offload_buffer_budget_bytes: int | None = None,
) -> worker.KVCacheStoreRecvingThread:
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
)
token_database.set_kv_caches_base_addr([0x1000])
token_database.set_block_len([256])
thread = worker.KVCacheStoreRecvingThread(
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["layer0"], spec)],
scheduler_block_size=16,
hash_block_size=16,
)
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
store=store,
token_database=token_database,
token_databases=[token_database],
block_size=16,
tp_rank=0,
ready_event=threading.Event(),
coord=coord,
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
)
thread.request_queue.task_done = MagicMock()
@@ -83,7 +112,7 @@ def _make_load_req(
return ReqMeta(
req_id=req_id,
token_len_chunk=token_len,
block_ids=list(range(len(block_hashes))),
block_ids=(list(range(len(block_hashes))),),
block_hashes=block_hashes,
load_spec=LoadSpec(
vllm_cached_tokens=vllm_cached_tokens,
@@ -98,7 +127,7 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
return ReqMeta(
req_id=req_id,
token_len_chunk=32,
block_ids=[0, 1],
block_ids=([0, 1],),
block_hashes=block_hashes,
can_save=True,
original_block_size=16,
@@ -149,10 +178,31 @@ def _make_vllm_config(
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
rank=0,
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
),
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
)
def _make_kv_cache_config(*, block_size: int = 16) -> object:
"""Minimal single-group KVCacheConfig for topology tests."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
)
return KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
)
@@ -445,9 +495,9 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
assert store.batch_get_into_multi_buffers.call_count == 1
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert sizes == [[256], [256], [256]]
store.batch_get_replica_desc.assert_not_called()
@@ -467,9 +517,9 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
token_len=48,
)
expected_keys = [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
store.batch_get_replica_desc.return_value = {
expected_keys[0]: [_ReplicaDesc("memory")],
@@ -521,10 +571,10 @@ def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
assert first_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
]
assert second_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
]
@@ -556,14 +606,14 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2]
second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2]
assert first_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
]
assert second_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
base_addr = thread.token_database.kv_caches_base_addr[0]
block_len = thread.token_database.block_len[0]
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
block_len = thread.token_databases[0].block_len[0]
assert first_addrs == [[base_addr], [base_addr + block_len]]
assert second_addrs == [[base_addr + 2 * block_len]]
expected_size = block_len
@@ -610,9 +660,9 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
assert store.batch_get_into_multi_buffers.call_count == 1
assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
@@ -654,7 +704,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
},
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert not hasattr(w, "_isolate_offload_resources")
assert store.setup.call_args.args == (
@@ -690,7 +740,7 @@ def test_requester_worker_init_prefers_local_hostname_override(
},
),
)
worker.MooncakeStoreWorker(_make_vllm_config())
worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert store.setup.call_args.args[0] == "worker-a:50053"
@@ -718,7 +768,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
},
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert w.disk_offload_buffer_budget_bytes is None
@@ -748,7 +798,8 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
extra_config={
"preferred_segment": "10.0.0.7:50053",
}
)
),
_make_kv_cache_config(),
)
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
@@ -760,6 +811,158 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
# ---------------------------------------------------------------------------
def test_store_sending_thread_clamps_token_len_to_lcm():
"""Partial chunks past the last lcm boundary aren't stored — cache hits
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
# Default coord: single full-attn block_size=16, lcm=16.
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
thread = _make_store_sending_thread(store)
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=33,
block_ids=([0, 1, 2],),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
original_block_size=16,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
assert len(keys) == 2
def test_store_sending_thread_skips_when_token_len_below_lcm():
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
so neither the existence check nor the put should be issued."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
store = MagicMock()
# lcm=64 via single full-attn block_size=64.
spec = FullAttentionSpec(block_size=64, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L"], spec)],
scheduler_block_size=64,
hash_block_size=64,
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=64,
hash_block_size=64,
)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([1024])
thread = _make_store_sending_thread(
store, coord=coord, token_databases=[db], block_size=64
)
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=32,
block_ids=([0, 1],),
block_hashes=[b"a0", b"a1"],
can_save=True,
original_block_size=64,
)
)
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["r0"] == 0
def test_store_sending_thread_only_stores_swa_blocks_in_window():
"""For SWA groups, only blocks within ``sliding_window`` of an
lcm-aligned boundary are stored — a block outside every such window
can never participate in any future hit (mirrors recv-side masking).
Setup: full-attn (block_size=32) + SWA (block_size=8, sliding_window=8)
→ lcm=32. With token_len=64 there are two LCM boundaries (32, 64);
the SWA group should store exactly one block per boundary: the block
ending at 32 and the block ending at 64.
"""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
[256] * len(keys)
)
full_spec = FullAttentionSpec(
block_size=32, num_kv_heads=8, head_size=64, dtype=None
)
swa_spec = SlidingWindowSpec(
block_size=8,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=8,
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
scheduler_block_size=32,
hash_block_size=8,
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=32,
hash_block_size=8,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([512])
db_swa = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=8,
)
db_swa.set_kv_caches_base_addr([0x2000])
db_swa.set_block_len([128])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_swa],
block_size=32,
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
original_block_size=32,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
full_keys = [k for k in keys if "@group:0" in k]
swa_keys = [k for k in keys if "@group:1" in k]
# Full-attn: 2 blocks (chunks ending at 32 and 64).
assert len(full_keys) == 2
# SWA: only the two blocks ending at lcm boundaries (32 and 64), i.e.
# blocks covering tokens [24,32) and [56,64) — hashes hs[3] and hs[7].
assert len(swa_keys) == 2
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
def _auto_set_ready_event(*args, **kwargs):
"""Side effect for mocked thread constructors that auto-sets ready_event."""
for arg in args:
@@ -771,37 +974,108 @@ def _auto_set_ready_event(*args, **kwargs):
return MagicMock()
def _register_with_mocked_threads(
worker: mooncake_store_worker.MooncakeStoreWorker,
kv_caches: dict[str, torch.Tensor],
) -> None:
"""Call register_kv_caches with the I/O transfer threads mocked out."""
prefix = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker."
with (
patch(prefix + "KVCacheStoreSendingThread", side_effect=_auto_set_ready_event),
patch(prefix + "KVCacheStoreRecvingThread", side_effect=_auto_set_ready_event),
):
worker.register_kv_caches(kv_caches)
def _make_bare_worker(
*,
num_gpu_blocks: int = 10,
block_size: int = 16,
kv_role: str = "kv_both",
) -> worker.MooncakeStoreWorker:
) -> mooncake_store_worker.MooncakeStoreWorker:
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
Sets only the attributes that register_kv_caches() reads so we can
test the stride-based layout detection without a real
MooncakeDistributedStore.
"""
w = object.__new__(worker.MooncakeStoreWorker)
w.cache_config = MagicMock()
w.cache_config.num_gpu_blocks = num_gpu_blocks
w.store = MagicMock()
w.store.register_buffer.return_value = 0
w.use_mla = False
w.token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=block_size
worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
worker.cache_config = MagicMock()
worker.cache_config.num_gpu_blocks = num_gpu_blocks
worker.store = MagicMock()
worker.store.register_buffer.return_value = 0
worker.use_mla = False
worker.kv_role = kv_role
worker.block_size = block_size
worker.tp_rank = 0
worker.put_step = 1
worker.enable_kv_events = False
worker.kv_send_thread = None
worker.kv_recv_thread = None
worker.tp_size = 1
worker.num_kv_head = 1
worker.pp_size = 1
# Minimal single-full-attention-group config so the coordinator-based
# lookup path works (the connector no longer carries a legacy single-group
# path; everything flows through the coordinator).
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
)
w.kv_role = kv_role
w.block_size = block_size
w.tp_rank = 0
w.put_step = 1
w.enable_kv_events = False
w.disk_offload_buffer_budget_bytes = None
w.kv_send_thread = None
w.kv_recv_thread = None
w.store_replicate_config = SimpleNamespace()
return w
worker.disk_offload_buffer_budget_bytes = None
worker.store_replicate_config = SimpleNamespace()
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
)
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
worker._kv_cache_groups = [group]
worker.pcp_size = 1
worker.dcp_size = 1
worker.hash_block_size = block_size
worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0)
# Pre-build a single-group token_dbs so lookup-only tests don't have to
# call register_kv_caches.
worker.token_dbs = [
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=block_size,
hash_block_size=block_size,
)
]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=block_size,
hash_block_size=block_size,
)
return worker
def test_lookup_partial_prefix_returns_first_hit_length():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1, 0]
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
tail. Coordinator-driven lookup returns full prefix even though the
pre-window blocks are absent."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
swa = SlidingWindowSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
)
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=worker.hash_block_size,
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
# ---------------------------------------------------------------------------
@@ -812,34 +1086,17 @@ def _make_bare_worker(
def test_register_kv_caches_blocks_first_single_segment():
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
num_blocks = 10
page_size_elements = 64 # elements per block
w = _make_bare_worker(num_gpu_blocks=num_blocks)
page_size_elements = 64
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Shape: (num_blocks, page_size_elements) — blocks outermost, no outer_dims
tensor = torch.zeros(num_blocks, page_size_elements, dtype=torch.float16)
_register_with_mocked_threads(worker, {"layer0": tensor})
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreSendingThread",
side_effect=_auto_set_ready_event,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreRecvingThread",
side_effect=_auto_set_ready_event,
),
):
w.register_kv_caches({"layer0": tensor})
assert len(w.kv_caches_base_addr) == 1
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
assert len(w.block_len) == 1
assert w.block_len[0] == expected_block_len
w.store.register_buffer.assert_called_once_with(
db = worker.token_dbs[0]
assert db.kv_caches_base_addr == [tensor.untyped_storage().data_ptr()]
assert db.block_len == [tensor.untyped_storage().nbytes() // num_blocks]
worker.store.register_buffer.assert_called_once_with(
tensor.untyped_storage().data_ptr(),
tensor.untyped_storage().nbytes(),
)
@@ -851,8 +1108,7 @@ def test_register_kv_caches_kv_first_two_segments():
block_size_tokens = 16
num_kv_heads = 4
head_size = 8
w = _make_bare_worker(num_gpu_blocks=num_blocks)
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Shape: (2, num_blocks, block_size, num_kv_heads, head_size) — K/V outermost
tensor = torch.zeros(
@@ -863,32 +1119,13 @@ def test_register_kv_caches_kv_first_two_segments():
head_size,
dtype=torch.float16,
)
_register_with_mocked_threads(worker, {"layer0": tensor})
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreSendingThread",
side_effect=_auto_set_ready_event,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreRecvingThread",
side_effect=_auto_set_ready_event,
),
):
w.register_kv_caches({"layer0": tensor})
# K/V-first: dim 0 has stride > page_size, so 2 segments
assert len(w.kv_caches_base_addr) == 2
assert len(w.block_len) == 2
el = tensor.element_size()
seg_stride = tensor.stride(0) * el # stride of the K/V dim in bytes
db = worker.token_dbs[0]
seg_stride = tensor.stride(0) * tensor.element_size()
base = tensor.untyped_storage().data_ptr()
assert w.kv_caches_base_addr[0] == base
assert w.kv_caches_base_addr[1] == base + seg_stride
assert w.block_len[0] == seg_stride // num_blocks
assert w.block_len[1] == seg_stride // num_blocks
assert db.kv_caches_base_addr == [base, base + seg_stride]
assert db.block_len == [seg_stride // num_blocks] * 2
def test_register_kv_caches_cross_layer_single_segment():
@@ -897,7 +1134,7 @@ def test_register_kv_caches_cross_layer_single_segment():
num_layers = 4
per_layer_page_elements = 64 # elements per layer per block
w = _make_bare_worker(num_gpu_blocks=num_blocks)
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Cross-layer blocks-first tensor: all layers packed into a single
# contiguous block. Shape (num_blocks, num_layers * per_layer_page)
@@ -918,10 +1155,11 @@ def test_register_kv_caches_cross_layer_single_segment():
),
):
# Use the cross-layer wrapper key, same as register_cross_layers_kv_caches
w.register_kv_caches({"__cross_layer__": tensor})
worker.register_kv_caches({"__cross_layer__": tensor})
assert len(w.kv_caches_base_addr) == 1
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
db = worker.token_dbs[0]
assert len(db.kv_caches_base_addr) == 1
assert db.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
# block_len should be per_layer_page_size * num_layers
@@ -929,11 +1167,11 @@ def test_register_kv_caches_cross_layer_single_segment():
expected_block_len
== num_layers * per_layer_page_elements * tensor.element_size()
)
assert len(w.block_len) == 1
assert w.block_len[0] == expected_block_len
assert len(db.block_len) == 1
assert db.block_len[0] == expected_block_len
# Also verify via register_cross_layers_kv_caches wrapper
w2 = _make_bare_worker(num_gpu_blocks=num_blocks)
worker2 = _make_bare_worker(num_gpu_blocks=num_blocks)
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
@@ -946,10 +1184,11 @@ def test_register_kv_caches_cross_layer_single_segment():
side_effect=_auto_set_ready_event,
),
):
w2.register_cross_layers_kv_caches(tensor)
worker2.register_cross_layers_kv_caches(tensor)
assert w2.kv_caches_base_addr == w.kv_caches_base_addr
assert w2.block_len == w.block_len
db2 = worker2.token_dbs[0]
assert db2.kv_caches_base_addr == db.kv_caches_base_addr
assert db2.block_len == db.block_len
# ---------------------------------------------------------------------------
@@ -1070,7 +1309,8 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
)
w = worker.MooncakeStoreWorker(
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"})
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}),
_make_kv_cache_config(),
)
# setup() receives global_segment_size=0 and the configured local buffer.
@@ -1114,7 +1354,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
# setup() receives global_segment_size=4 GiB (rank contributes a segment).
assert store.setup.call_args.args == (