[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-05-19 01:07:46 -07:00
committed by GitHub
parent f34623bf3c
commit 056bc2e166
10 changed files with 1834 additions and 445 deletions
@@ -9,12 +9,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
connector,
worker,
connector as mooncake_store_connector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeStoreConnectorMetadata,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
from vllm.v1.outputs import KVConnectorOutput
from .utils import create_vllm_config
@@ -27,6 +32,17 @@ def _make_vllm_config():
)
def _make_kv_cache_config() -> KVCacheConfig:
"""Single-group full-attention KVCacheConfig — enough for the connector
constructor's validate() pass."""
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
return KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[KVCacheTensor(size=8192, shared_by=["layer0"])],
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
)
def _make_block_stored() -> BlockStored:
return BlockStored(
block_hashes=[b"hash"],
@@ -41,6 +57,7 @@ def _make_block_stored() -> BlockStored:
def test_scheduler_role_initializes_store_scheduler_only():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -53,75 +70,19 @@ def test_scheduler_role_initializes_store_scheduler_only():
"connector.MooncakeStoreWorker"
) as mock_worker,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
mock_scheduler.assert_called_once_with(vllm_config)
mock_scheduler.assert_called_once_with(vllm_config, kv_cache_config)
mock_worker.assert_not_called()
assert conn.connector_scheduler is mock_scheduler.return_value
assert conn.connector_worker is None
def test_worker_role_initializes_store_worker_on_rank0():
vllm_config = _make_vllm_config()
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreScheduler"
) as mock_scheduler,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
mock_scheduler.assert_not_called()
mock_worker.assert_called_once_with(vllm_config)
assert conn.connector_scheduler is None
assert conn.connector_worker is mock_worker.return_value
def test_worker_role_initializes_on_nonzero_rank():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.rank = 1
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker,
):
connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
mock_worker.assert_called_once_with(vllm_config)
def test_lookup_rpc_path_uses_data_parallel_index_in_dense_dp():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.data_parallel_rank = 0
vllm_config.parallel_config.data_parallel_index = 3
path = worker.get_zmq_rpc_path_lookup(vllm_config)
assert path.endswith("_dp_rank3")
def test_lookup_rpc_path_uses_local_rank_when_local_engines_only():
vllm_config = _make_vllm_config()
vllm_config.parallel_config.data_parallel_index = 7
vllm_config.parallel_config.data_parallel_rank_local = 1
vllm_config.parallel_config.data_parallel_hybrid_lb = True
path = worker.get_zmq_rpc_path_lookup(vllm_config)
assert path.endswith("_dp_rank1")
assert connector.connector_scheduler is mock_scheduler.return_value
assert connector.connector_worker is None
def test_worker_methods_delegate_to_store_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
kv_caches = {"layer0": MagicMock()}
metadata = MooncakeStoreConnectorMetadata(set(), set())
finished_req_ids = {"req-1"}
@@ -133,22 +94,25 @@ def test_worker_methods_delegate_to_store_worker():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
worker_inst = mock_worker_cls.return_value
worker_inst.get_finished.return_value = ({"req-1"}, {"req-2"})
conn.bind_connector_metadata(metadata)
worker = mock_worker_cls.return_value
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
connector.bind_connector_metadata(metadata)
conn.register_kv_caches(kv_caches)
result = conn.get_finished(finished_req_ids)
connector.register_kv_caches(kv_caches)
result = connector.get_finished(finished_req_ids)
worker_inst.register_kv_caches.assert_called_once_with(kv_caches)
worker_inst.get_finished.assert_called_once_with(finished_req_ids, metadata)
worker.register_kv_caches.assert_called_once_with(kv_caches)
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
assert result == ({"req-1"}, {"req-2"})
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -157,14 +121,17 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_events.return_value = []
assert conn.get_kv_connector_kv_cache_events() is None
assert connector.get_kv_connector_kv_cache_events() is None
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
event = _make_block_stored()
with (
@@ -174,12 +141,14 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
mock_worker_cls.return_value.get_kv_events.return_value = [event]
kv_events = conn.get_kv_connector_kv_cache_events()
kv_events = connector.get_kv_connector_kv_cache_events()
assert isinstance(kv_events, connector.MooncakeStoreKVEvents)
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
assert kv_events.get_number_of_workers() == 1
assert kv_events.get_all_events() == [event]
@@ -187,6 +156,7 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
def test_prefer_cross_layer_blocks_from_config():
# Default: disabled
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
patch(
@@ -194,8 +164,10 @@ def test_prefer_cross_layer_blocks_from_config():
"connector.MooncakeStoreScheduler"
),
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
assert conn.prefer_cross_layer_blocks is False
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert connector.prefer_cross_layer_blocks is False
# Enabled via config
vllm_config_enabled = create_vllm_config(
@@ -210,14 +182,15 @@ def test_prefer_cross_layer_blocks_from_config():
"connector.MooncakeStoreScheduler"
),
):
conn_enabled = connector.MooncakeStoreConnector(
vllm_config_enabled, KVConnectorRole.SCHEDULER
connector_enabled = mooncake_store_connector.MooncakeStoreConnector(
vllm_config_enabled, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert conn_enabled.prefer_cross_layer_blocks is True
assert connector_enabled.prefer_cross_layer_blocks is True
def test_register_cross_layers_kv_cache_delegates_to_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
@@ -226,18 +199,21 @@ def test_register_cross_layers_kv_cache_delegates_to_worker():
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
fake_tensor = MagicMock()
fake_backend = MagicMock()
conn.register_cross_layers_kv_cache(fake_tensor, fake_backend)
connector.register_cross_layers_kv_cache(fake_tensor, fake_backend)
worker_inst = mock_worker_cls.return_value
worker_inst.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
worker = mock_worker_cls.return_value
worker.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
def test_update_connector_output_and_take_events():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
event = _make_block_stored()
with (
@@ -247,12 +223,14 @@ def test_update_connector_output_and_take_events():
"connector.MooncakeStoreScheduler"
),
):
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
kv_events = connector.MooncakeStoreKVEvents(num_workers=1)
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
kv_events.add_events([event])
conn.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
assert conn._kv_cache_events is kv_events
assert list(conn.take_events()) == [event]
assert conn._kv_cache_events is None
assert connector._kv_cache_events is kv_events
assert list(connector.take_events()) == [event]
assert connector._kv_cache_events is None
@@ -0,0 +1,302 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from math import lcm
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
)
from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
def _make_coord(groups, hash_block_size, use_eagle=False):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
the test fixtures."""
block_sizes = [g.kv_cache_spec.block_size for g in groups]
scheduler_block_size = lcm(*block_sizes)
return MooncakeStoreCoordinator(
groups,
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
use_eagle=use_eagle,
)
# ----- ExternalCachedBlockPool -----
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
cmap = ExternalCachedBlockPool()
h = BlockHash(b"\xaa" * 4)
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
assert len(res) == 2
assert res[0] is not cmap.null_block
assert res[1] is not cmap.null_block
def test_external_cached_block_pool_hit_all_groups():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
assert len(res) == 2
assert res[0] is not cmap.null_block
assert res[1] is not cmap.null_block
def test_external_cached_block_pool_miss_one_group():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h))})
assert cmap.get_cached_block(h, [0, 1]) is None
def test_external_cached_block_pool_unknown_hash():
h_known = BlockHash(b"\x01" * 4)
h_unknown = BlockHash(b"\x02" * 4)
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
assert cmap.get_cached_block(h_unknown, [0]) is None
# ----- Helpers -----
def _full(block_size=16, sliding_window=None):
return FullAttentionSpec(
block_size=block_size,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=sliding_window,
)
def _swa(block_size=16, sliding_window=32):
return SlidingWindowSpec(
block_size=block_size,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=sliding_window,
)
def _hashes(n: int) -> list[BlockHash]:
return [BlockHash(bytes([i + 1]) * 4) for i in range(n)]
# ----- Single-group coordinator -----
def test_coordinator_single_full_attention_all_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
assert masks[0] == [True, True, True, True]
def test_coordinator_single_full_attention_partial_prefix():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 32
assert masks[0] == [True, True]
def test_coordinator_single_full_attention_no_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool(set())
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 0
assert masks[0] == []
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
"""SWA tautological-pool: hit_length spans full prefix, mask is
tail-window only."""
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4) # 4 chunks * 16 tokens
cmap = ExternalCachedBlockPool()
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
assert masks[0][-2:] == [True, True]
assert all(not m for m in masks[0][:-2])
# ----- Hybrid coordinator (single-group worker, multi-group coordinator) -----
def test_coordinator_hybrid_full_plus_swa_all_hit():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 64
def test_coordinator_hybrid_hole_in_full_clips_both():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
exists |= {(1, bytes(h)) for h in hs}
cmap = ExternalCachedBlockPool(exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 16
def test_coordinator_group_block_size_double_hash():
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
hashes merge before pool lookup."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _full(32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32))
exists = {(0, bytes(h)) for h in hs}
exists |= {(1, bytes(bh)) for bh in big_hashes}
cmap = ExternalCachedBlockPool(exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 64
assert hit % 32 == 0
# ----- store_mask -----
def test_store_mask_full_attention_all_true():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(64)
assert masks == ([True, True, True, True],)
def test_store_mask_zero_aligned_returns_empty_per_group():
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(0)
assert masks == ([], [])
def test_store_mask_swa_only_window_around_each_lcm_boundary():
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
aligned=64 the SWA group should mark exactly the blocks ending at 32
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
participate in any future hit."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
# Full-attn: 2 chunks * 32 tokens.
assert masks[0] == [True, True]
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
assert masks[1] == [False, False, False, True, False, False, False, True]
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=16)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
assert masks[0] == [True, True]
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
# Boundary at 64: chunks 6 and 7. Others stay False.
assert masks[1] == [False, False, True, True, False, False, True, True]
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
(aligned_len=512). Validates that the tile-once strategy produces the
expected per-segment tail-window pattern, repeated."""
full_mla = _full(block_size=256)
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
swa_4_sw16 = _swa(block_size=4, sliding_window=16)
swa_8_sw64 = _swa(block_size=8, sliding_window=64)
groups = [
KVCacheGroupSpec(["L0"], full_mla),
KVCacheGroupSpec(["L1"], swa_64_sw128),
KVCacheGroupSpec(["L2"], swa_64_sw512),
KVCacheGroupSpec(["L3"], swa_4_sw16),
KVCacheGroupSpec(["L4"], swa_8_sw64),
]
coord = _make_coord(groups, hash_block_size=4)
assert coord.lcm_block_size == 256
masks = coord.store_mask(512)
# Full-MLA: 2 chunks of 256, both stored.
assert masks[0] == [True, True]
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
# Per-segment template = [F,F,T,T]; tiled twice.
assert masks[1] == [False, False, True, True] * 2
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
assert masks[2] == [True] * 8
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
# Last 4 of each 64-chunk segment True.
assert masks[3] == ([False] * 60 + [True] * 4) * 2
# SWA(8, sw=64): tail = ceil(63/8) = 8; C = 256/8 = 32.
# Last 8 of each 32-chunk segment True.
assert masks[4] == ([False] * 24 + [True] * 8) * 2
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
"""When every non-full-attn group already aligns to lcm_block_size, the
fast path returns all-True without invoking find_longest_cache_hit."""
full = _full(block_size=64)
swa = _swa(block_size=64, sliding_window=128)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=64)
assert coord.lcm_block_size == 64
masks = coord.store_mask(256)
# Every block in every group is True — no sub-lcm filtering possible.
assert masks == ([True] * 4, [True] * 4)
def test_store_mask_fast_path_single_attention_group():
"""Two groups sharing the same SWA spec collapse to one attention group;
no lcm filter applies, every chunk is True."""
swa = _swa(block_size=16, sliding_window=32)
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=16)
assert len(coord.attention_groups) == 1
masks = coord.store_mask(64)
assert masks == ([True] * 4, [True] * 4)
@@ -0,0 +1,342 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
(SWA + Full) attention config, using a dict-backed mock store."""
import sys
import threading
import types
from unittest.mock import MagicMock, patch
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker as mooncake_store_worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
KVCacheStoreRecvingThread,
KVCacheStoreSendingThread,
)
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
SlidingWindowSpec,
)
class _DictStore:
"""In-memory MooncakeDistributedStore stand-in."""
def __init__(self):
self._data: dict[str, bytes] = {}
def setup(self, *_args, **_kwargs):
return 0
def register_buffer(self, addr, length):
return 0
def batch_is_exist(self, keys):
return [1 if k in self._data else 0 for k in keys]
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
for k in keys:
self._data[k] = b"x"
return [0] * len(keys)
def batch_get_into_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
return [0 if k in self._data else -1 for k in keys]
def _minimal_vllm_config(cache_block_size=16):
cfg = MagicMock()
cfg.cache_config.block_size = cache_block_size
cfg.cache_config.num_gpu_blocks = 4
cfg.cache_config.hash_block_size = None
cfg.cache_config.enable_prefix_caching = True
cfg.parallel_config.prefill_context_parallel_size = 1
cfg.parallel_config.decode_context_parallel_size = 1
cfg.parallel_config.pipeline_parallel_size = 1
cfg.parallel_config.world_size = 1
cfg.parallel_config.rank = 0
cfg.parallel_config.data_parallel_rank_local = 0
cfg.parallel_config.data_parallel_size_local = 1
cfg.kv_transfer_config.kv_role = "kv_both"
cfg.kv_transfer_config.kv_connector_extra_config = {}
cfg.kv_events_config = None
cfg.model_config.model = "/tmp/m"
cfg.model_config.use_mla = False
cfg.model_config.get_num_layers.return_value = 2
cfg.model_config.get_total_num_kv_heads.return_value = 8
cfg.model_config.max_model_len = 4096
cfg.scheduler_config.max_num_batched_tokens = 8192
# Without this, MagicMock's truthy use_eagle() triggers the coordinator's
# "use_eagle && nothing annotated → flag all groups" fallback.
cfg.speculative_config = None
return cfg
def _build_worker_with_dict_store(vllm_config, kv_cache_config, store):
"""Build a MooncakeStoreWorker patching all distributed dependencies."""
fake_mooncake_store = types.ModuleType("mooncake.store")
fake_mooncake_store.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined]
fake_mooncake_store.ReplicateConfig = MagicMock # type: ignore[attr-defined]
with (
patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}),
patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg,
):
sc = MCfg.load_from_env.return_value
sc.metadata_server = ""
sc.global_segment_size = 1 << 20
sc.local_buffer_size = 1 << 20
sc.protocol = "tcp"
sc.device_name = ""
sc.master_server_address = ""
sc.enable_offload = False
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_tensor_model_parallel_world_size",
return_value=1,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_pcp_group"
) as mock_pcp,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_dcp_group"
) as mock_dcp,
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".store.worker.get_ip",
return_value="127.0.0.1",
),
):
mock_pcp.return_value.world_size = 1
mock_dcp.return_value.world_size = 1
worker = mooncake_store_worker.MooncakeStoreWorker(
vllm_config, kv_cache_config=kv_cache_config
)
return worker
def test_e2e_swa_plus_full_save_then_lookup_hits():
"""
E2E: build a SWA+Full hybrid worker, save all blocks via the sending
thread (synchronously), then verify lookup returns the full hit length.
Also verify that evicting SWA's early blocks (outside its window) still
allows a full hit because the window covers the tail.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
swa = SlidingWindowSpec(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=32,
)
cfg = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(size=8192, shared_by=["L0"]),
KVCacheTensor(size=8192, shared_by=["L1"]),
],
kv_cache_groups=[
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], swa),
],
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
store = _DictStore()
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
worker.tp_size = 1
worker.pp_size = 1
worker.put_step = 1
worker.num_kv_head = 8
# Register kv_caches using mocked thread classes so register_kv_caches
# doesn't try to start real background threads (which set ready_event).
kv_caches = {
"L0": torch.zeros(2, 4, 8, 8, 64),
"L1": torch.zeros(2, 4, 8, 8, 64),
}
def _fake_thread_init(*args, **kwargs):
"""Mock thread that sets all threading.Event args so waits don't block."""
for v in list(args) + list(kwargs.values()):
if isinstance(v, threading.Event):
v.set()
m = MagicMock()
m.start = lambda: None
return m
with (
patch.object(
mooncake_store_worker,
"KVCacheStoreSendingThread",
side_effect=_fake_thread_init,
),
patch.object(
mooncake_store_worker,
"KVCacheStoreRecvingThread",
side_effect=_fake_thread_init,
),
):
worker.register_kv_caches(kv_caches)
# Now build a real sending thread (no .start()) over the worker's token_dbs
# and the dict-backed store, so _handle_request runs synchronously.
ready = threading.Event()
send_thread = KVCacheStoreSendingThread(
store=store,
token_databases=worker.token_dbs,
block_size=worker.block_size,
coord=worker.coord,
tp_rank=worker.tp_rank,
put_step=worker.put_step,
kv_role=worker.kv_role,
ready_event=ready,
enable_kv_event=False,
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
save_req = ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
can_save=True,
)
send_thread.add_stored_request("r0")
# Put the request in the queue so task_done() doesn't underflow.
send_thread.request_queue.put(save_req)
req = send_thread.request_queue.get()
send_thread._handle_request(req)
# Point worker.store at the dict store (the worker constructor captured
# the MagicMock; replace with the real dict store for lookup).
worker.store = store
# Both groups stored all 4 blocks -> full hit.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
swa_keys_outside_window = [
k
for k in list(store._data.keys())
if "@group:1" in k and (("@" + hs[0].hex()) in k or ("@" + hs[1].hex()) in k)
]
for k in swa_keys_outside_window:
del store._data[k]
# SWA window=32 -> only last 2 blocks must be present in SWA group.
# Full has all 4. Coordinator should still return 64.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
def test_recv_skips_swa_blocks_before_window():
"""Producer stored every block for both groups; consumer must only fetch
SWA blocks within the sliding window, not the head."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
swa = SlidingWindowSpec(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=32,
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], swa),
]
md0 = KeyMetadata("m", 0, 0, 0, 0, group_id=0)
md1 = KeyMetadata("m", 0, 0, 0, 0, group_id=1)
db_full = ChunkedTokenDatabase(md0, block_size=16, hash_block_size=16)
db_swa = ChunkedTokenDatabase(md1, block_size=16, hash_block_size=16)
db_full.set_kv_caches_base_addr([0])
db_full.set_block_len([1024])
db_swa.set_kv_caches_base_addr([1 << 20])
db_swa.set_block_len([1024])
requested_keys: list[str] = []
class _CapturingStore:
def batch_get_into_multi_buffers(self, keys, addrs, sizes):
requested_keys.extend(keys)
return [0] * len(keys)
ready = threading.Event()
coord = MooncakeStoreCoordinator(
groups, scheduler_block_size=16, hash_block_size=16
)
recv = KVCacheStoreRecvingThread(
store=_CapturingStore(),
token_databases=[db_full, db_swa],
block_size=16,
tp_rank=0,
ready_event=ready,
coord=coord,
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
req = ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
load_spec=LoadSpec(
vllm_cached_tokens=0, kvpool_cached_tokens=64, can_load=True, token_len=64
),
)
recv.request_queue.put(req)
recv._handle_request(recv.request_queue.get())
full_keys = [k for k in requested_keys if "@group:0" in k]
swa_keys = [k for k in requested_keys if "@group:1" in k]
# Full attention: load every block (4).
assert len(full_keys) == 4
# SWA: only the 2 tail-window blocks (hashes hs[2], hs[3]).
assert len(swa_keys) == 2
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
assert swa_hashes == {hs[2].hex(), hs[3].hex()}
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
must merge every 4 fine hashes into one chunk hash via
BlockHashListWithBlockSize."""
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
db.set_kv_caches_base_addr([0])
db.set_block_len([512])
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
assert len(out) == 2
assert out[0][0] == 0 and out[0][1] == 16
assert out[1][0] == 16 and out[1][1] == 32
# Each chunk's hash is the concatenation of 4 fine hashes.
expected0 = b"".join(fine_hashes[0:4]).hex()
expected1 = b"".join(fine_hashes[4:8]).hex()
assert out[0][2].chunk_hash == expected0
assert out[1][2].chunk_hash == expected1
@@ -0,0 +1,111 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
def _make_bare_scheduler() -> MooncakeStoreScheduler:
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.original_block_size = 16
scheduler._block_size = 16
scheduler._discard_partial_chunks = True
scheduler.load_specs = {}
scheduler._preempted_req_ids = set()
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
scheduler._request_trackers = {}
return scheduler
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[([2],)],
num_computed_tokens=[44],
),
num_scheduled_tokens={"req-0": 4},
scheduled_spec_decode_tokens=(
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
),
)
def _add_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
token_ids: list[int],
block_hashes: list[bytes],
prefill_end_tokens: int,
) -> None:
request = SimpleNamespace(
all_token_ids=token_ids,
block_hashes=block_hashes,
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=44,
allocated_block_ids=([0, 1],),
num_saved_tokens=32,
token_ids=token_ids[:44],
prefill_end_tokens=prefill_end_tokens,
)
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
# yet, so the tracker's token_len does not advance and num_tokens_to_save
# stays below chunk_boundary — the save is naturally skipped.
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(44)),
block_hashes=[b"h0", b"h1"],
prefill_end_tokens=48,
)
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=[101, 102, 103])
)
assert meta.requests == []
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 44
assert tracker.num_saved_tokens == 32
assert tracker.allocated_block_ids == ([0, 1, 2],)
def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=48,
)
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=None)
)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 48
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 48
assert tracker.num_saved_tokens == 48
@@ -19,7 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
worker as mooncake_store_worker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
@@ -27,20 +30,37 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
)
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
return mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["layer0"], spec)],
scheduler_block_size=16,
hash_block_size=16,
)
def _make_store_sending_thread(
store: MagicMock,
*,
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
token_databases: list[ChunkedTokenDatabase] | None = None,
block_size: int = 16,
replicate_config: object | None = None,
) -> worker.KVCacheStoreSendingThread:
token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
)
token_database.set_kv_caches_base_addr([0x1000])
token_database.set_block_len([256])
thread = worker.KVCacheStoreSendingThread(
) -> mooncake_store_worker.KVCacheStoreSendingThread:
if coord is None:
coord = _default_send_coord()
if token_databases is None:
db = ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([256])
token_databases = [db]
thread = mooncake_store_worker.KVCacheStoreSendingThread(
store=store,
token_database=token_database,
block_size=16,
token_databases=token_databases,
block_size=block_size,
coord=coord,
tp_rank=0,
put_step=1,
kv_role="kv_producer",
@@ -55,18 +75,27 @@ def _make_store_recving_thread(
store: MagicMock,
*,
disk_offload_buffer_budget_bytes: int | None = None,
) -> worker.KVCacheStoreRecvingThread:
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
)
token_database.set_kv_caches_base_addr([0x1000])
token_database.set_block_len([256])
thread = worker.KVCacheStoreRecvingThread(
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["layer0"], spec)],
scheduler_block_size=16,
hash_block_size=16,
)
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
store=store,
token_database=token_database,
token_databases=[token_database],
block_size=16,
tp_rank=0,
ready_event=threading.Event(),
coord=coord,
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
)
thread.request_queue.task_done = MagicMock()
@@ -83,7 +112,7 @@ def _make_load_req(
return ReqMeta(
req_id=req_id,
token_len_chunk=token_len,
block_ids=list(range(len(block_hashes))),
block_ids=(list(range(len(block_hashes))),),
block_hashes=block_hashes,
load_spec=LoadSpec(
vllm_cached_tokens=vllm_cached_tokens,
@@ -98,7 +127,7 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
return ReqMeta(
req_id=req_id,
token_len_chunk=32,
block_ids=[0, 1],
block_ids=([0, 1],),
block_hashes=block_hashes,
can_save=True,
original_block_size=16,
@@ -149,10 +178,31 @@ def _make_vllm_config(
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
rank=0,
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
),
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
)
def _make_kv_cache_config(*, block_size: int = 16) -> object:
"""Minimal single-group KVCacheConfig for topology tests."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
)
return KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
)
@@ -445,9 +495,9 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
assert store.batch_get_into_multi_buffers.call_count == 1
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert sizes == [[256], [256], [256]]
store.batch_get_replica_desc.assert_not_called()
@@ -467,9 +517,9 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
token_len=48,
)
expected_keys = [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
store.batch_get_replica_desc.return_value = {
expected_keys[0]: [_ReplicaDesc("memory")],
@@ -521,10 +571,10 @@ def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
assert first_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
]
assert second_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
]
@@ -556,14 +606,14 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2]
second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2]
assert first_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
]
assert second_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
base_addr = thread.token_database.kv_caches_base_addr[0]
block_len = thread.token_database.block_len[0]
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
block_len = thread.token_databases[0].block_len[0]
assert first_addrs == [[base_addr], [base_addr + block_len]]
assert second_addrs == [[base_addr + 2 * block_len]]
expected_size = block_len
@@ -610,9 +660,9 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
assert store.batch_get_into_multi_buffers.call_count == 1
assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
@@ -654,7 +704,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
},
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert not hasattr(w, "_isolate_offload_resources")
assert store.setup.call_args.args == (
@@ -690,7 +740,7 @@ def test_requester_worker_init_prefers_local_hostname_override(
},
),
)
worker.MooncakeStoreWorker(_make_vllm_config())
worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert store.setup.call_args.args[0] == "worker-a:50053"
@@ -718,7 +768,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
},
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
assert w.disk_offload_buffer_budget_bytes is None
@@ -748,7 +798,8 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
extra_config={
"preferred_segment": "10.0.0.7:50053",
}
)
),
_make_kv_cache_config(),
)
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
@@ -760,6 +811,158 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
# ---------------------------------------------------------------------------
def test_store_sending_thread_clamps_token_len_to_lcm():
"""Partial chunks past the last lcm boundary aren't stored — cache hits
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
# Default coord: single full-attn block_size=16, lcm=16.
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
thread = _make_store_sending_thread(store)
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=33,
block_ids=([0, 1, 2],),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
original_block_size=16,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
assert len(keys) == 2
def test_store_sending_thread_skips_when_token_len_below_lcm():
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
so neither the existence check nor the put should be issued."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
store = MagicMock()
# lcm=64 via single full-attn block_size=64.
spec = FullAttentionSpec(block_size=64, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L"], spec)],
scheduler_block_size=64,
hash_block_size=64,
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=64,
hash_block_size=64,
)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([1024])
thread = _make_store_sending_thread(
store, coord=coord, token_databases=[db], block_size=64
)
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=32,
block_ids=([0, 1],),
block_hashes=[b"a0", b"a1"],
can_save=True,
original_block_size=64,
)
)
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["r0"] == 0
def test_store_sending_thread_only_stores_swa_blocks_in_window():
"""For SWA groups, only blocks within ``sliding_window`` of an
lcm-aligned boundary are stored — a block outside every such window
can never participate in any future hit (mirrors recv-side masking).
Setup: full-attn (block_size=32) + SWA (block_size=8, sliding_window=8)
→ lcm=32. With token_len=64 there are two LCM boundaries (32, 64);
the SWA group should store exactly one block per boundary: the block
ending at 32 and the block ending at 64.
"""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
[256] * len(keys)
)
full_spec = FullAttentionSpec(
block_size=32, num_kv_heads=8, head_size=64, dtype=None
)
swa_spec = SlidingWindowSpec(
block_size=8,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=8,
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
scheduler_block_size=32,
hash_block_size=8,
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=32,
hash_block_size=8,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([512])
db_swa = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=8,
)
db_swa.set_kv_caches_base_addr([0x2000])
db_swa.set_block_len([128])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_swa],
block_size=32,
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
original_block_size=32,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
full_keys = [k for k in keys if "@group:0" in k]
swa_keys = [k for k in keys if "@group:1" in k]
# Full-attn: 2 blocks (chunks ending at 32 and 64).
assert len(full_keys) == 2
# SWA: only the two blocks ending at lcm boundaries (32 and 64), i.e.
# blocks covering tokens [24,32) and [56,64) — hashes hs[3] and hs[7].
assert len(swa_keys) == 2
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
def _auto_set_ready_event(*args, **kwargs):
"""Side effect for mocked thread constructors that auto-sets ready_event."""
for arg in args:
@@ -771,37 +974,108 @@ def _auto_set_ready_event(*args, **kwargs):
return MagicMock()
def _register_with_mocked_threads(
worker: mooncake_store_worker.MooncakeStoreWorker,
kv_caches: dict[str, torch.Tensor],
) -> None:
"""Call register_kv_caches with the I/O transfer threads mocked out."""
prefix = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker."
with (
patch(prefix + "KVCacheStoreSendingThread", side_effect=_auto_set_ready_event),
patch(prefix + "KVCacheStoreRecvingThread", side_effect=_auto_set_ready_event),
):
worker.register_kv_caches(kv_caches)
def _make_bare_worker(
*,
num_gpu_blocks: int = 10,
block_size: int = 16,
kv_role: str = "kv_both",
) -> worker.MooncakeStoreWorker:
) -> mooncake_store_worker.MooncakeStoreWorker:
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
Sets only the attributes that register_kv_caches() reads so we can
test the stride-based layout detection without a real
MooncakeDistributedStore.
"""
w = object.__new__(worker.MooncakeStoreWorker)
w.cache_config = MagicMock()
w.cache_config.num_gpu_blocks = num_gpu_blocks
w.store = MagicMock()
w.store.register_buffer.return_value = 0
w.use_mla = False
w.token_database = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0), block_size=block_size
worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
worker.cache_config = MagicMock()
worker.cache_config.num_gpu_blocks = num_gpu_blocks
worker.store = MagicMock()
worker.store.register_buffer.return_value = 0
worker.use_mla = False
worker.kv_role = kv_role
worker.block_size = block_size
worker.tp_rank = 0
worker.put_step = 1
worker.enable_kv_events = False
worker.kv_send_thread = None
worker.kv_recv_thread = None
worker.tp_size = 1
worker.num_kv_head = 1
worker.pp_size = 1
# Minimal single-full-attention-group config so the coordinator-based
# lookup path works (the connector no longer carries a legacy single-group
# path; everything flows through the coordinator).
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
)
w.kv_role = kv_role
w.block_size = block_size
w.tp_rank = 0
w.put_step = 1
w.enable_kv_events = False
w.disk_offload_buffer_budget_bytes = None
w.kv_send_thread = None
w.kv_recv_thread = None
w.store_replicate_config = SimpleNamespace()
return w
worker.disk_offload_buffer_budget_bytes = None
worker.store_replicate_config = SimpleNamespace()
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
)
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
worker._kv_cache_groups = [group]
worker.pcp_size = 1
worker.dcp_size = 1
worker.hash_block_size = block_size
worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0)
# Pre-build a single-group token_dbs so lookup-only tests don't have to
# call register_kv_caches.
worker.token_dbs = [
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=block_size,
hash_block_size=block_size,
)
]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=block_size,
hash_block_size=block_size,
)
return worker
def test_lookup_partial_prefix_returns_first_hit_length():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1, 0]
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
tail. Coordinator-driven lookup returns full prefix even though the
pre-window blocks are absent."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
swa = SlidingWindowSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
)
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=worker.hash_block_size,
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
# ---------------------------------------------------------------------------
@@ -812,34 +1086,17 @@ def _make_bare_worker(
def test_register_kv_caches_blocks_first_single_segment():
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
num_blocks = 10
page_size_elements = 64 # elements per block
w = _make_bare_worker(num_gpu_blocks=num_blocks)
page_size_elements = 64
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Shape: (num_blocks, page_size_elements) — blocks outermost, no outer_dims
tensor = torch.zeros(num_blocks, page_size_elements, dtype=torch.float16)
_register_with_mocked_threads(worker, {"layer0": tensor})
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreSendingThread",
side_effect=_auto_set_ready_event,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreRecvingThread",
side_effect=_auto_set_ready_event,
),
):
w.register_kv_caches({"layer0": tensor})
assert len(w.kv_caches_base_addr) == 1
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
assert len(w.block_len) == 1
assert w.block_len[0] == expected_block_len
w.store.register_buffer.assert_called_once_with(
db = worker.token_dbs[0]
assert db.kv_caches_base_addr == [tensor.untyped_storage().data_ptr()]
assert db.block_len == [tensor.untyped_storage().nbytes() // num_blocks]
worker.store.register_buffer.assert_called_once_with(
tensor.untyped_storage().data_ptr(),
tensor.untyped_storage().nbytes(),
)
@@ -851,8 +1108,7 @@ def test_register_kv_caches_kv_first_two_segments():
block_size_tokens = 16
num_kv_heads = 4
head_size = 8
w = _make_bare_worker(num_gpu_blocks=num_blocks)
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Shape: (2, num_blocks, block_size, num_kv_heads, head_size) — K/V outermost
tensor = torch.zeros(
@@ -863,32 +1119,13 @@ def test_register_kv_caches_kv_first_two_segments():
head_size,
dtype=torch.float16,
)
_register_with_mocked_threads(worker, {"layer0": tensor})
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreSendingThread",
side_effect=_auto_set_ready_event,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"worker.KVCacheStoreRecvingThread",
side_effect=_auto_set_ready_event,
),
):
w.register_kv_caches({"layer0": tensor})
# K/V-first: dim 0 has stride > page_size, so 2 segments
assert len(w.kv_caches_base_addr) == 2
assert len(w.block_len) == 2
el = tensor.element_size()
seg_stride = tensor.stride(0) * el # stride of the K/V dim in bytes
db = worker.token_dbs[0]
seg_stride = tensor.stride(0) * tensor.element_size()
base = tensor.untyped_storage().data_ptr()
assert w.kv_caches_base_addr[0] == base
assert w.kv_caches_base_addr[1] == base + seg_stride
assert w.block_len[0] == seg_stride // num_blocks
assert w.block_len[1] == seg_stride // num_blocks
assert db.kv_caches_base_addr == [base, base + seg_stride]
assert db.block_len == [seg_stride // num_blocks] * 2
def test_register_kv_caches_cross_layer_single_segment():
@@ -897,7 +1134,7 @@ def test_register_kv_caches_cross_layer_single_segment():
num_layers = 4
per_layer_page_elements = 64 # elements per layer per block
w = _make_bare_worker(num_gpu_blocks=num_blocks)
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
# Cross-layer blocks-first tensor: all layers packed into a single
# contiguous block. Shape (num_blocks, num_layers * per_layer_page)
@@ -918,10 +1155,11 @@ def test_register_kv_caches_cross_layer_single_segment():
),
):
# Use the cross-layer wrapper key, same as register_cross_layers_kv_caches
w.register_kv_caches({"__cross_layer__": tensor})
worker.register_kv_caches({"__cross_layer__": tensor})
assert len(w.kv_caches_base_addr) == 1
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
db = worker.token_dbs[0]
assert len(db.kv_caches_base_addr) == 1
assert db.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
# block_len should be per_layer_page_size * num_layers
@@ -929,11 +1167,11 @@ def test_register_kv_caches_cross_layer_single_segment():
expected_block_len
== num_layers * per_layer_page_elements * tensor.element_size()
)
assert len(w.block_len) == 1
assert w.block_len[0] == expected_block_len
assert len(db.block_len) == 1
assert db.block_len[0] == expected_block_len
# Also verify via register_cross_layers_kv_caches wrapper
w2 = _make_bare_worker(num_gpu_blocks=num_blocks)
worker2 = _make_bare_worker(num_gpu_blocks=num_blocks)
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
@@ -946,10 +1184,11 @@ def test_register_kv_caches_cross_layer_single_segment():
side_effect=_auto_set_ready_event,
),
):
w2.register_cross_layers_kv_caches(tensor)
worker2.register_cross_layers_kv_caches(tensor)
assert w2.kv_caches_base_addr == w.kv_caches_base_addr
assert w2.block_len == w.block_len
db2 = worker2.token_dbs[0]
assert db2.kv_caches_base_addr == db.kv_caches_base_addr
assert db2.block_len == db.block_len
# ---------------------------------------------------------------------------
@@ -1070,7 +1309,8 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
)
w = worker.MooncakeStoreWorker(
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"})
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}),
_make_kv_cache_config(),
)
# setup() receives global_segment_size=0 and the configured local buffer.
@@ -1114,7 +1354,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
),
)
w = worker.MooncakeStoreWorker(_make_vllm_config())
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
# setup() receives global_segment_size=4 GiB (rank contributes a segment).
assert store.setup.call_args.args == (
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
@@ -76,7 +77,7 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
class MooncakeStoreConnector(KVConnectorBase_V1):
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@property
@@ -87,6 +88,36 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
== "true"
)
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None:
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
unsupported: list[str] = []
cache_block_size = vllm_config.cache_config.block_size
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
spec = g.kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
# Enforce Mamba align mode
if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size:
unsupported.append(
f"group {g_idx}: MambaSpec with block_size="
f"{spec.block_size} != cache_config.block_size="
f"{cache_block_size} (mamba_cache_mode != 'align')"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
dcp = vllm_config.parallel_config.decode_context_parallel_size
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
unsupported.append(
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
)
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
)
def __init__(
self,
vllm_config: VllmConfig,
@@ -99,6 +130,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self.kv_role = vllm_config.kv_transfer_config.kv_role
self._kv_cache_events: MooncakeStoreKVEvents | None = None
@@ -106,9 +140,11 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self.connector_worker: MooncakeStoreWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = MooncakeStoreScheduler(vllm_config)
self.connector_scheduler = MooncakeStoreScheduler(
vllm_config, kv_cache_config
)
else:
self.connector_worker = MooncakeStoreWorker(vllm_config)
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
# ============================================================
# Scheduler-side methods
@@ -146,6 +182,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished_all_groups(request, (block_ids,))
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -184,6 +227,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
self, kv_cache: torch.Tensor, attn_backend: type
):
assert self.connector_worker is not None
assert (
self._kv_cache_config is not None
and len(self._kv_cache_config.kv_cache_groups) == 1
), "Cross-layer KV cache does not supported with hybrid models"
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from typing import cast
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashList,
BlockHashListWithBlockSize,
KVCacheBlock,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
spec_manager_map,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheSpec,
UniformTypeKVCacheSpecs,
)
# Dummy placeholder hash for store_mask's template computation.
_DUMMY_BLOCK_HASH = BlockHash(b"\x00" * 32)
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
def get_cached_block(
self,
block_hash: BlockHash,
group_ids: list[int],
) -> list[KVCacheBlock] | None:
# Mirrors BlockPool.get_cached_block: hit only when every group_id
# (groups sharing a spec) has the hash cached.
if self._exists is None:
return [self._present_block] * len(group_ids)
h = bytes(block_hash)
if all((g, h) in self._exists for g in group_ids):
return [self._present_block] * len(group_ids)
return None
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
def __init__(
self,
kv_cache_groups: list[KVCacheGroupSpec],
scheduler_block_size: int,
hash_block_size: int,
use_eagle: bool = False,
) -> None:
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
f"hash_block_size ({hash_block_size})"
)
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.use_eagle = use_eagle
self._verify_and_split_kv_cache_groups()
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[
tuple[KVCacheSpec, list[int], type[SingleTypeKVCacheManager]]
] = []
for i, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = spec_manager_map[type(spec)]
for existing_spec, group_ids, existing_cls in attention_groups:
if existing_spec == spec:
assert manager_cls is existing_cls
group_ids.append(i)
break
else:
attention_groups.append((spec, [i], manager_cls))
# Full attention first (matches upstream convergence ordering).
self.attention_groups = sorted(
attention_groups,
key=lambda x: not isinstance(x[0], FullAttentionSpec),
)
self.eagle_attn_group_indices: set[int] = {
i
for i, (_, group_ids, _) in enumerate(self.attention_groups)
if any(self.kv_cache_groups[gid].is_eagle_group for gid in group_ids)
}
if self.use_eagle and not self.eagle_attn_group_indices:
self.eagle_attn_group_indices = set(range(len(self.attention_groups)))
def find_longest_cache_hit(
self,
block_hashes: list[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
) -> tuple[tuple[list[bool], ...], int]:
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
recv-side callers skip False slots."""
blocks_per_group, hit_length = self._find_hit_blocks(
block_hashes, max_length, cached_block_pool
)
masks = tuple(
[blk is not cached_block_pool.null_block for blk in blocks]
for blocks in blocks_per_group
)
return masks, hit_length
def load_mask(
self,
block_hashes: list[BlockHash],
token_len: int,
) -> tuple[list[bool], ...]:
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
spec would populate chunk ``i`` locally at length ``token_len``
(e.g. SWA / Mamba tail-only).
"""
masks, _ = self.find_longest_cache_hit(
block_hashes, token_len, ExternalCachedBlockPool()
)
return masks
def store_mask(self, aligned_token_len: int) -> tuple[list[bool], ...]:
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
group ``g`` would be populated by some future cache hit at length
``L = N * lcm_block_size <= aligned_token_len``.
"""
assert aligned_token_len % self.lcm_block_size == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"lcm_block_size ({self.lcm_block_size})"
)
if aligned_token_len == 0:
return tuple([] for _ in self.kv_cache_groups)
num_chunks_per_group = [
aligned_token_len // g.kv_cache_spec.block_size
for g in self.kv_cache_groups
]
# Fast path: single group or full attn groups or uniform block_sizes
if all(
isinstance(spec, FullAttentionSpec)
or spec.block_size == self.lcm_block_size
for spec, _, _ in self.attention_groups
):
return tuple([True] * n for n in num_chunks_per_group)
n_segments = aligned_token_len // self.lcm_block_size
dummy_hashes: list[BlockHash] = [_DUMMY_BLOCK_HASH] * (
self.lcm_block_size // self.hash_block_size
)
template_masks, _ = self.find_longest_cache_hit(
dummy_hashes,
max_length=self.lcm_block_size,
cached_block_pool=ExternalCachedBlockPool(),
)
return tuple(
list(template_masks[g]) * n_segments
for g in range(len(self.kv_cache_groups))
)
def block_hashes_for_spec(
self, block_hashes: list[BlockHash], spec: KVCacheSpec
) -> BlockHashList:
if spec.block_size == self.hash_block_size:
return block_hashes
return BlockHashListWithBlockSize(
block_hashes, self.hash_block_size, spec.block_size
)
def _find_hit_blocks(
self,
block_hashes: list[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
dispatches via spec_manager_map (we don't allocate managers).
"""
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls = self.attention_groups[0]
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
block_hashes=hashes,
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
use_eagle=(0 in self.eagle_attn_group_indices),
alignment_tokens=spec.block_size,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0][0], FullAttentionSpec
)
eagle_verified: set[int] = set()
while True:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
cached = hit_blocks_by_group[group_ids[0]]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = (
curr_hit_length // spec.block_size * spec.block_size
)
continue
use_eagle = (
idx in self.eagle_attn_group_indices and idx not in eagle_verified
)
_max_length = curr_hit_length
if use_eagle:
_max_length = min(curr_hit_length + spec.block_size, max_length)
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
block_hashes=hashes,
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
use_eagle=use_eagle,
alignment_tokens=self.lcm_block_size,
)
_new_hit_length = len(hit_blocks[0]) * spec.block_size
if use_eagle:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
eagle_verified.clear()
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
if is_simple_hybrid:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic.
spec0, group_ids0, _ = self.attention_groups[0]
if isinstance(spec0, FullAttentionSpec):
num_blocks = hit_length // spec0.block_size
for gid in group_ids0:
full_blks = hit_blocks_by_group[gid]
assert full_blks is not None
del full_blks[num_blocks:]
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
hit_length,
)
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
@@ -15,7 +15,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
)
logger = init_logger(__name__)
@@ -29,6 +32,7 @@ class KeyMetadata:
pcp_rank: int
dcp_rank: int
pp_rank: int
group_id: int = 0
@dataclass(order=True)
@@ -46,6 +50,7 @@ class PoolKey:
self.key_metadata.pcp_rank,
self.key_metadata.dcp_rank,
self.key_metadata.pp_rank,
self.key_metadata.group_id,
self.chunk_hash,
)
)
@@ -57,6 +62,7 @@ class PoolKey:
f"@pcp{self.key_metadata.pcp_rank}"
f"@dcp{self.key_metadata.dcp_rank}"
f"@pp_rank:{self.key_metadata.pp_rank}"
f"@group:{self.key_metadata.group_id}"
f"@{self.chunk_hash}"
)
@@ -64,9 +70,20 @@ class PoolKey:
class ChunkedTokenDatabase:
"""Maps token positions to store keys and GPU memory addresses."""
def __init__(self, metadata: KeyMetadata, block_size: int):
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int | None = None,
):
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size or block_size
if self.block_size % self.hash_block_size != 0:
raise ValueError(
f"block_size ({self.block_size}) must be a multiple of "
f"hash_block_size ({self.hash_block_size})"
)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
@@ -77,9 +94,6 @@ class ChunkedTokenDatabase:
self.kv_caches_base_addr = kv_caches_base_addr
def set_block_len(self, block_len: list[int]):
for length in block_len:
if length % self.block_size != 0:
raise ValueError(f"block_len {length} % {self.block_size} != 0")
self.block_len = block_len
def prepare_value(
@@ -96,7 +110,8 @@ class ChunkedTokenDatabase:
length = len(self.block_len)
for index, base_addr in enumerate(self.kv_caches_base_addr):
addr = base_addr + block_id * self.block_len[index % length]
size = self.block_len[index % length] // self.block_size * (end - start)
assert (end - start) % self.block_size == 0
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
addr_list.append(addr)
size_list.append(size)
return addr_list, size_list, block_id
@@ -104,38 +119,34 @@ class ChunkedTokenDatabase:
def process_tokens(
self,
token_len: int,
block_hashes: list[BlockHash] | list[str],
block_hashes: list[BlockHash],
mask_num: int = 0,
) -> Iterable[tuple[int, int, PoolKey]]:
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
Args:
token_len: Total number of tokens.
block_hashes: Block hashes for each block.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` consecutive hashes are merged
up to the group's ``block_size`` via ``BlockHashListWithBlockSize``.
mask_num: Number of tokens to skip from the beginning.
"""
if not block_hashes:
return
if not isinstance(block_hashes[0], str):
block_hashes = [
h.hex() # type: ignore[union-attr]
for h in block_hashes
]
for chunk_id, hash_val in enumerate(block_hashes):
if self.block_size == self.hash_block_size:
chunk_hashes: Iterable[BlockHash] = block_hashes
else:
chunk_hashes = BlockHashListWithBlockSize(
block_hashes, self.hash_block_size, self.block_size
)
for chunk_id, h in enumerate(chunk_hashes):
start_idx = chunk_id * self.block_size
if start_idx >= token_len:
break
end_idx = min(start_idx + self.block_size, token_len)
if start_idx < mask_num:
continue
else:
yield (
start_idx,
end_idx,
self._make_key_by_hash(
hash_val # type: ignore[arg-type]
),
)
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
@dataclass
@@ -154,7 +165,7 @@ class RequestTracker:
req_id: str
token_len: int
allocated_block_ids: list[int]
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
# Snapshot of the prefill range length at tracker creation time.
@@ -166,15 +177,18 @@ class RequestTracker:
self,
new_block_ids: tuple[list[int], ...] | list[int],
) -> None:
if len(new_block_ids) == 0:
new_block_ids = []
elif isinstance(new_block_ids, tuple):
new_block_ids = new_block_ids[0]
elif isinstance(new_block_ids, list):
pass
else:
raise ValueError(f"Unsupported new_block_ids type {type(new_block_ids)}")
self.allocated_block_ids.extend(new_block_ids)
# Backward-compat: accept a single list (broadcast to single group).
if isinstance(new_block_ids, list):
new_block_ids = (new_block_ids,)
if len(new_block_ids) != len(self.allocated_block_ids):
raise ValueError(
f"Group count mismatch: tracker has "
f"{len(self.allocated_block_ids)} groups, update has "
f"{len(new_block_ids)}"
)
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
if new:
existing.extend(new)
@dataclass
@@ -183,7 +197,7 @@ class ReqMeta:
req_id: str
token_len_chunk: int
block_ids: list[int]
block_ids: tuple[list[int], ...]
block_hashes: list[BlockHash]
can_save: bool | None = None
@@ -22,7 +22,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
)
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -45,7 +47,11 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
def __init__(self, vllm_config: VllmConfig):
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
@@ -56,11 +62,11 @@ class MooncakeStoreScheduler:
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.original_block_size = vllm_config.cache_config.block_size
self._block_size = vllm_config.cache_config.block_size
if self.pcp_size > 1:
self._block_size *= self.pcp_size
if self.dcp_size > 1:
self._block_size *= self.dcp_size
# LCM for multi-group HMA; bs * pcp * dcp for single-group. Matches
# the engine's own scheduler block size by construction.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self._discard_partial_chunks = (
vllm_config.kv_transfer_config.get_from_extra_config(
@@ -72,7 +78,7 @@ class MooncakeStoreScheduler:
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._preempted_req_ids: set[str] = set() # preempted requests
self._unfinished_requests: dict[str, tuple[Request, list[int]]] = {}
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
def get_num_new_matched_tokens(
@@ -126,9 +132,9 @@ class MooncakeStoreScheduler:
num_external_tokens: int,
):
"""Update state after block allocation."""
local_block_ids: list[int] = []
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids()[0]
local_block_ids = blocks.get_block_ids()
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
@@ -189,11 +195,12 @@ class MooncakeStoreScheduler:
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
if not isinstance(request.block_ids[0], list):
unfolded_block_ids = request.block_ids.copy()
if isinstance(request.block_ids, tuple):
# Multi-group: preserve per-group structure.
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
else:
# TODO: support HMA
unfolded_block_ids = request.block_ids[0].copy()
# Single-group legacy: list[int] -> 1-tuple.
unfolded_block_ids = (request.block_ids.copy(),)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
@@ -237,9 +244,9 @@ class MooncakeStoreScheduler:
if req_id in self._preempted_req_ids:
# Resumed after preemption
if isinstance(new_block_ids, tuple):
block_ids_list = new_block_ids[0].copy()
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
block_ids_list = new_block_ids.copy()
new_block_ids = (new_block_ids.copy(),)
self._preempted_req_ids.discard(req_id)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
@@ -254,7 +261,7 @@ class MooncakeStoreScheduler:
request_tracker = RequestTracker(
req_id=req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=block_ids_list,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
prefill_end_tokens=len(prefill_tokens),
@@ -361,7 +368,7 @@ class MooncakeStoreScheduler:
def request_finished(
self,
request: Request,
block_ids: list[int],
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""Determine whether to delay freeing blocks for async save."""
if self.kv_role == "kv_consumer":
@@ -370,11 +377,12 @@ class MooncakeStoreScheduler:
assert tracker is not None
if tracker.num_saved_tokens <= 0:
return False, None
delay_free_blocks = len(block_ids) > 0
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
if delay_free_blocks:
logger.debug(
"Delaying free of %d blocks for request %s",
len(block_ids),
total_blocks,
request.request_id,
)
return delay_free_blocks, None
@@ -10,6 +10,7 @@ Includes the store worker, transfer threads, lookup server,
and MooncakeDistributedStore integration.
"""
import dataclasses
import json
import os
import queue
@@ -36,15 +37,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
get_mooncake_dp_engine_index,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
ChunkedTokenDatabase,
KeyMetadata,
MooncakeStoreConnectorMetadata,
PoolKey,
ReqMeta,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
logger = init_logger(__name__)
@@ -321,7 +332,7 @@ class KVTransferThread(threading.Thread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
ready_event: threading.Event,
@@ -332,7 +343,7 @@ class KVTransferThread(threading.Thread):
self.ready_event = ready_event
self.block_size = block_size
self.tp_rank = tp_rank
self.token_database = token_database
self.token_databases = token_databases
self.done_task_lock = threading.Lock()
self.request_queue: queue.Queue[Any] = queue.Queue()
self.finished_requests: set[str] = set()
@@ -385,7 +396,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
coord: MooncakeStoreCoordinator,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
put_step: int,
@@ -396,13 +408,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
):
super().__init__(
store,
token_database,
token_databases,
block_size,
tp_rank,
ready_event,
name="KVCacheStoreSendingThread",
)
self.put_step = put_step
self.coord = coord
self.kv_role = kv_role
self.stored_requests: defaultdict[str, int] = defaultdict(int)
self.enable_kv_event = enable_kv_event
@@ -449,14 +462,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
return True
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.token_len_chunk
block_ids = req_meta.block_ids
# Cache hits are always a multiple of ``lcm_block_size`` tokens
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if req_id not in self.stored_requests:
self.request_queue.task_done()
return
if token_len == 0:
self.dec_stored_request(req_id)
self.request_queue.task_done()
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk offloading "
@@ -467,23 +486,34 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.request_queue.task_done()
return
starts = []
ends = []
keys = []
# Within each lcm region only per-spec relevant chunks are loaded
# (e.g., SWA or linear attn), so mask out irrelevant chunks
store_masks = self.coord.store_mask(token_len)
starts: list[int] = []
ends: list[int] = []
keys: list[str] = []
block_hashes: list[BlockHash] = []
for index, (start, end, key) in enumerate(
self.token_database.process_tokens(token_len, req_meta.block_hashes)
):
starts.append(start)
ends.append(end)
keys.append(key.to_string())
block_hashes.append(req_meta.block_hashes[index])
group_indices: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = store_masks[g_idx]
for chunk_idx, (start, end, key) in enumerate(
db.process_tokens(token_len, req_meta.block_hashes)
):
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
starts.append(start)
ends.append(end)
keys.append(key.to_string())
block_hashes.append(req_meta.block_hashes[chunk_idx])
group_indices.append(g_idx)
# Apply put_step striding for TP
starts = starts[self.tp_rank % self.put_step :: self.put_step]
ends = ends[self.tp_rank % self.put_step :: self.put_step]
keys = keys[self.tp_rank % self.put_step :: self.put_step]
block_hashes = block_hashes[self.tp_rank % self.put_step :: self.put_step]
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
starts = starts[sl]
ends = ends[sl]
keys = keys[sl]
block_hashes = block_hashes[sl]
group_indices = group_indices[sl]
if not keys:
self.dec_stored_request(req_id)
@@ -501,38 +531,37 @@ class KVCacheStoreSendingThread(KVTransferThread):
ends = [ends[i] for i in missing_indices]
keys = [keys[i] for i in missing_indices]
block_hashes = [block_hashes[i] for i in missing_indices]
group_indices = [group_indices[i] for i in missing_indices]
logger.debug(
"Storing KV cache for %d out of %d blocks "
"(missing_count=%d) for request %s",
"Storing KV cache for %d blocks (groups=%s) for request %s",
len(keys),
token_len // self.block_size,
len(missing_indices),
set(group_indices),
req_id,
)
addrs = []
sizes = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
stored_events: list[BlockStored] = []
prev_key = None
# parent_block_hash chains live within a group, not across.
prev_key_per_group: dict[int, Any] = {}
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
for index, start in enumerate(starts):
addr, size, _ = self.token_database.prepare_value(
start, ends[index], block_ids
)
for idx, (s, e, g_idx) in enumerate(
zip(starts, ends, group_indices, strict=True)
):
db = self.token_databases[g_idx]
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
addrs.append(addr)
sizes.append(size)
if self.enable_kv_event:
token_ids = (
req_meta.token_ids[start : ends[index]]
if req_meta.token_ids is not None
else None
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None
)
stored_event = BlockStored(
block_hashes=[new_block_hashes[index]],
parent_block_hash=prev_key,
block_hashes=[new_block_hashes[idx]],
parent_block_hash=prev_key_per_group.get(g_idx),
token_ids=token_ids,
block_size=req_meta.original_block_size,
lora_id=None,
@@ -540,7 +569,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
lora_name=None,
)
stored_events.append(stored_event)
prev_key = new_block_hashes[index]
prev_key_per_group[g_idx] = new_block_hashes[idx]
if current_event is not None:
current_event.synchronize()
@@ -600,7 +629,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
def __init__(
self,
store: Any,
token_database: ChunkedTokenDatabase,
coord: MooncakeStoreCoordinator,
token_databases: list[ChunkedTokenDatabase],
block_size: int,
tp_rank: int,
ready_event: threading.Event,
@@ -608,7 +638,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
):
super().__init__(
store,
token_database,
token_databases,
block_size,
tp_rank,
ready_event,
@@ -622,6 +652,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
disk_offload_buffer_budget_bytes
)
)
self.coord = coord
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
@@ -632,32 +663,31 @@ class KVCacheStoreRecvingThread(KVTransferThread):
* self.block_size
)
addr_list = []
size_list = []
key_list = []
for start, end, key in self.token_database.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
addr, size, _ = self.token_database.prepare_value(
start, end, req_meta.block_ids
)
key_list.append(key.to_string())
addr_list.append(addr)
size_list.append(size)
# Skip chunks the consumer's per-group spec wouldn't populate
# locally (e.g. SWA pre-window) even if the producer stored them.
load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len)
addr_list: list[list[int]] = []
size_list: list[list[int]] = []
key_list: list[str] = []
for g_idx, db in enumerate(self.token_databases):
mask = load_mask_per_group[g_idx]
for start, end, key in db.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
chunk_idx = start // db.block_size
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
key_list.append(key.to_string())
addr_list.append(addr)
size_list.append(size)
# Rotate lists by tp_rank for load balancing
key_list_c = (
key_list[self.tp_rank % len(key_list) :]
+ key_list[: self.tp_rank % len(key_list)]
)
addr_list_c = (
addr_list[self.tp_rank % len(addr_list) :]
+ addr_list[: self.tp_rank % len(addr_list)]
)
size_list_c = (
size_list[self.tp_rank % len(size_list) :]
+ size_list[: self.tp_rank % len(size_list)]
)
rotation = self.tp_rank % len(key_list)
key_list_c = key_list[rotation:] + key_list[:rotation]
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
size_list_c = size_list[rotation:] + size_list[:rotation]
load_batches = [(key_list_c, addr_list_c, size_list_c)]
if self.usable_disk_offload_buffer_budget_bytes is not None:
@@ -737,7 +767,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
def __init__(self, vllm_config: VllmConfig):
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
try:
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
@@ -771,11 +805,9 @@ class MooncakeStoreWorker:
)
self.cache_config = vllm_config.cache_config
self.original_block_size = self.cache_config.block_size
self.block_size = self.cache_config.block_size
if self.pcp_size > 1:
self.block_size *= self.pcp_size
if self.dcp_size > 1:
self.block_size *= self.dcp_size
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.num_layers = model_config.get_num_layers(parallel_config)
self.use_mla = False
@@ -806,8 +838,6 @@ class MooncakeStoreWorker:
pp_rank=self.pp_rank,
)
self.token_database = ChunkedTokenDatabase(self.metadata, self.block_size)
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_env()
extra_config = (
@@ -892,6 +922,45 @@ class MooncakeStoreWorker:
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
self.finished_store_req: set[str] = set()
self._kv_cache_config = kv_cache_config
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
groups = list(kv_cache_config.kv_cache_groups)
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
g = groups[0]
groups = [
dataclasses.replace(
g,
kv_cache_spec=dataclasses.replace(
g.kv_cache_spec, block_size=self.block_size
),
)
]
self._kv_cache_groups: list[KVCacheGroupSpec] = groups
spec_cfg = getattr(vllm_config, "speculative_config", None)
use_eagle = bool(
spec_cfg.use_eagle()
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
else False
)
self.coord = MooncakeStoreCoordinator(
self._kv_cache_groups,
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
use_eagle=use_eagle,
)
# One ChunkedTokenDatabase per group; addresses populated in
# register_kv_caches once the kv-cache layout is known.
self.token_dbs: list[ChunkedTokenDatabase] = [
ChunkedTokenDatabase(
dataclasses.replace(self.metadata, group_id=g_idx),
g.kv_cache_spec.block_size,
hash_block_size=self.hash_block_size,
)
for g_idx, g in enumerate(self._kv_cache_groups)
]
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
"""Register a cross-layers KV cache tensor.
@@ -901,93 +970,86 @@ class MooncakeStoreWorker:
"""
self.register_kv_caches({"__cross_layer__": kv_cache})
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
def register_kv_caches(
self,
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
) -> None:
"""Register KV cache tensors and start transfer threads."""
# TODO(yifan): we haven't supported HMA yet.
first_kv_cache = next(iter(kv_caches.values()))
if not kv_caches:
logger.warning("No KV caches to offload.")
return
# Resolve each entry to a representative tensor for storage
# deduplication. For attention layers the value is already a tensor;
# for Mamba layers it is a list of tensors that all share the same
# underlying raw storage, so we take the first one.
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
assert isinstance(v, torch.Tensor | list)
return v if isinstance(v, torch.Tensor) else v[0]
# num_blocks from cache_config is authoritative (set after
# profiling, before KV cache allocation).
assert self.cache_config.num_gpu_blocks is not None
self.num_blocks = self.cache_config.num_gpu_blocks
# Detect the KV cache memory layout using the stride-based
# approach from simple_kv_offload/worker.py.
#
# The physical layout varies across attention backends:
# FlashAttn/ROCm : (2, num_blocks, ...) → K/V outermost
# FlashInfer/MLA : (num_blocks, ...) → blocks outermost
#
# We derive page_size_bytes = storage.nbytes() // num_blocks,
# then classify dims: any dim whose byte-stride exceeds
# page_size_bytes must be an outer segment dim (e.g. the K/V
# dim of size 2). For those backends we register each segment
# (K, V) as a separate base-address so that the per-block
# offset arithmetic in prepare_value() stays correct.
storage = first_kv_cache.untyped_storage()
el = first_kv_cache.element_size()
page_size_bytes = storage.nbytes() // self.num_blocks
outer_dims = [
d
for d in range(first_kv_cache.ndim)
if first_kv_cache.stride(d) * el > page_size_bytes
]
# Register buffers with the store (deduplicate shared storages)
# and record per-segment base addresses for every layer.
seen_ptrs: set[int] = set()
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
addrs: list[int] = []
block_lens: list[int] = []
for cache in kv_caches.values():
for value in kv_caches.values():
cache = _repr_tensor(value)
cache_storage = cache.untyped_storage()
base_addr = cache_storage.data_ptr()
if base_addr in seen_ptrs:
continue
seen_ptrs.add(base_addr)
region_len = cache_storage.nbytes()
if base_addr not in seen_ptrs:
seen_ptrs.add(base_addr)
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
# Detect layout via stride: a dim whose byte-stride exceeds
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
# outermost layout has no such dim and yields a single segment.
el = cache.element_size()
page_size_bytes = region_len // self.num_blocks
outer_dims = [
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
]
if not outer_dims:
# Blocks-first layout (FlashInfer / MLA): one segment.
self.kv_caches_base_addr.append(base_addr)
self.block_len.append(page_size_bytes)
addrs.append(base_addr)
block_lens.append(page_size_bytes)
else:
# K/V-first layout (FlashAttn / ROCm): split segments.
seg_stride = cache.stride(outer_dims[0]) * el
for idx in range(cache.shape[outer_dims[0]]):
self.kv_caches_base_addr.append(base_addr + idx * seg_stride)
self.block_len.append(seg_stride // self.num_blocks)
addrs.append(base_addr + idx * seg_stride)
block_lens.append(seg_stride // self.num_blocks)
logger.info(
"Registering KV_Caches. use_mla: %s, shape %s, "
"num_blocks: %d, block_len: %s, "
"per_key_bytes: %d, "
"num_segments: %d",
self.use_mla,
first_kv_cache.shape,
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
len(self.token_dbs),
len(addrs),
self.num_blocks,
list(set(self.block_len)),
sum(self.block_len),
len(self.kv_caches_base_addr),
)
self.token_database.set_kv_caches_base_addr(self.kv_caches_base_addr)
self.token_database.set_block_len(self.block_len)
for db in self.token_dbs:
db.set_kv_caches_base_addr(addrs)
db.set_block_len(block_lens)
# Start transfer threads
if self.kv_role in ["kv_producer", "kv_both"]:
ready_event_sending = threading.Event()
self.kv_send_thread = KVCacheStoreSendingThread(
self.store,
self.token_database,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
self.put_step,
@@ -1001,7 +1063,8 @@ class MooncakeStoreWorker:
ready_event_recving = threading.Event()
self.kv_recv_thread = KVCacheStoreRecvingThread(
self.store,
self.token_database,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
ready_event_recving,
@@ -1122,62 +1185,55 @@ class MooncakeStoreWorker:
return finished_sending
def lookup(
self,
token_len: int,
block_hashes: list[BlockHash],
) -> int:
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.
Checks across all TP ranks and PP ranks.
"""
end = 0
keys: list[str] = []
if not block_hashes or token_len <= 0:
return 0
# Build per-(group, hash) candidate keys expanded across TP/PP.
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
candidate_keys: list[str] = []
candidate_meta: list[tuple[int, bytes]] = []
tp_count = min(self.tp_size, self.num_kv_head)
for g_idx, db in enumerate(self.token_dbs):
spec_block_size = db.block_size
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
for chunk_id, h in enumerate(group_hashes):
start_idx = chunk_id * spec_block_size
if start_idx >= token_len:
break
for tp in range(tp_count):
for pp in range(self.pp_size):
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)
candidate_keys.append(PoolKey(md, h.hex()).to_string())
candidate_meta.append((g_idx, bytes(h)))
if not candidate_keys:
return 0
try:
starts: list[int] = []
for start, end, key in self.token_database.process_tokens(
token_len, block_hashes
):
keys.append(key.to_string())
starts.append(start)
# Expand keys for all TP ranks
multi_tp_keys = keys[:]
for i in range(1, min(self.tp_size, self.num_kv_head)):
for item in keys:
new_str = item.replace("@tp_rank:0", f"@tp_rank:{i}", 1)
multi_tp_keys.append(new_str)
# Expand keys for all PP ranks
pp_base_keys = multi_tp_keys.copy()
for i in range(1, self.pp_size):
for item in pp_base_keys:
new_str = item.replace("@pp_rank:0", f"@pp_rank:{i}", 1)
multi_tp_keys.append(new_str)
res = self.store.batch_is_exist(multi_tp_keys)
num_block = len(keys)
multi_tp_values = [
res[i * num_block : (i + 1) * num_block]
for i in range(min(self.tp_size, self.num_kv_head) * self.pp_size)
]
index = self._find_min_first_non_one_index(multi_tp_values)
if index != -1:
return starts[index]
res = self.store.batch_is_exist(candidate_keys)
except Exception as e:
logger.error("Remote connection failed in lookup: %s", e)
return 0
return end
@staticmethod
def _find_min_first_non_one_index(
arr: list[list[int]],
) -> int:
try:
return min(idx for row in arr for idx, val in enumerate(row) if val != 1)
except ValueError:
return -1
# A (group, hash) is "present" only when every TP*PP rank has it.
expected_per_key = max(1, tp_count * self.pp_size)
present_count: dict[tuple[int, bytes], int] = {}
for gh, exists in zip(candidate_meta, res, strict=True):
if exists == 1:
present_count[gh] = present_count.get(gh, 0) + 1
exists_set = {gh for gh, c in present_count.items() if c >= expected_per_key}
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
)
return hit_length
def get_kv_events(self) -> list[BlockStored]:
if self.enable_kv_events and self.kv_send_thread is not None:
@@ -1220,7 +1276,8 @@ class LookupKeyServer:
token_len = int.from_bytes(all_frames[0], byteorder="big")
hash_frames = all_frames[1:]
hashes_str = self.decoder.decode(hash_frames)
result = self.store_worker.lookup(token_len, hashes_str)
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
result = self.store_worker.lookup(token_len, block_hashes)
response = result.to_bytes(4, "big")
self.socket.send(response)