[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -9,12 +9,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
connector,
|
||||
worker,
|
||||
connector as mooncake_store_connector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeStoreConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_vllm_config
|
||||
@@ -27,6 +32,17 @@ def _make_vllm_config():
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config() -> KVCacheConfig:
|
||||
"""Single-group full-attention KVCacheConfig — enough for the connector
|
||||
constructor's validate() pass."""
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[KVCacheTensor(size=8192, shared_by=["layer0"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
)
|
||||
|
||||
|
||||
def _make_block_stored() -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[b"hash"],
|
||||
@@ -41,6 +57,7 @@ def _make_block_stored() -> BlockStored:
|
||||
|
||||
def test_scheduler_role_initializes_store_scheduler_only():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -53,75 +70,19 @@ def test_scheduler_role_initializes_store_scheduler_only():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_scheduler.assert_called_once_with(vllm_config)
|
||||
mock_scheduler.assert_called_once_with(vllm_config, kv_cache_config)
|
||||
mock_worker.assert_not_called()
|
||||
assert conn.connector_scheduler is mock_scheduler.return_value
|
||||
assert conn.connector_worker is None
|
||||
|
||||
|
||||
def test_worker_role_initializes_store_worker_on_rank0():
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreScheduler"
|
||||
) as mock_scheduler,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
|
||||
mock_scheduler.assert_not_called()
|
||||
mock_worker.assert_called_once_with(vllm_config)
|
||||
assert conn.connector_scheduler is None
|
||||
assert conn.connector_worker is mock_worker.return_value
|
||||
|
||||
|
||||
def test_worker_role_initializes_on_nonzero_rank():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.rank = 1
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
|
||||
mock_worker.assert_called_once_with(vllm_config)
|
||||
|
||||
|
||||
def test_lookup_rpc_path_uses_data_parallel_index_in_dense_dp():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.data_parallel_rank = 0
|
||||
vllm_config.parallel_config.data_parallel_index = 3
|
||||
|
||||
path = worker.get_zmq_rpc_path_lookup(vllm_config)
|
||||
|
||||
assert path.endswith("_dp_rank3")
|
||||
|
||||
|
||||
def test_lookup_rpc_path_uses_local_rank_when_local_engines_only():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.data_parallel_index = 7
|
||||
vllm_config.parallel_config.data_parallel_rank_local = 1
|
||||
vllm_config.parallel_config.data_parallel_hybrid_lb = True
|
||||
|
||||
path = worker.get_zmq_rpc_path_lookup(vllm_config)
|
||||
|
||||
assert path.endswith("_dp_rank1")
|
||||
assert connector.connector_scheduler is mock_scheduler.return_value
|
||||
assert connector.connector_worker is None
|
||||
|
||||
|
||||
def test_worker_methods_delegate_to_store_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
kv_caches = {"layer0": MagicMock()}
|
||||
metadata = MooncakeStoreConnectorMetadata(set(), set())
|
||||
finished_req_ids = {"req-1"}
|
||||
@@ -133,22 +94,25 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
worker_inst = mock_worker_cls.return_value
|
||||
worker_inst.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
conn.bind_connector_metadata(metadata)
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
conn.register_kv_caches(kv_caches)
|
||||
result = conn.get_finished(finished_req_ids)
|
||||
connector.register_kv_caches(kv_caches)
|
||||
result = connector.get_finished(finished_req_ids)
|
||||
|
||||
worker_inst.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker_inst.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
worker.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result == ({"req-1"}, {"req-2"})
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -157,14 +121,17 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = []
|
||||
assert conn.get_kv_connector_kv_cache_events() is None
|
||||
assert connector.get_kv_connector_kv_cache_events() is None
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
event = _make_block_stored()
|
||||
|
||||
with (
|
||||
@@ -174,12 +141,14 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = [event]
|
||||
kv_events = conn.get_kv_connector_kv_cache_events()
|
||||
kv_events = connector.get_kv_connector_kv_cache_events()
|
||||
|
||||
assert isinstance(kv_events, connector.MooncakeStoreKVEvents)
|
||||
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
|
||||
assert kv_events.get_number_of_workers() == 1
|
||||
assert kv_events.get_all_events() == [event]
|
||||
|
||||
@@ -187,6 +156,7 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
def test_prefer_cross_layer_blocks_from_config():
|
||||
# Default: disabled
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
@@ -194,8 +164,10 @@ def test_prefer_cross_layer_blocks_from_config():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
assert conn.prefer_cross_layer_blocks is False
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert connector.prefer_cross_layer_blocks is False
|
||||
|
||||
# Enabled via config
|
||||
vllm_config_enabled = create_vllm_config(
|
||||
@@ -210,14 +182,15 @@ def test_prefer_cross_layer_blocks_from_config():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn_enabled = connector.MooncakeStoreConnector(
|
||||
vllm_config_enabled, KVConnectorRole.SCHEDULER
|
||||
connector_enabled = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config_enabled, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert conn_enabled.prefer_cross_layer_blocks is True
|
||||
assert connector_enabled.prefer_cross_layer_blocks is True
|
||||
|
||||
|
||||
def test_register_cross_layers_kv_cache_delegates_to_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -226,18 +199,21 @@ def test_register_cross_layers_kv_cache_delegates_to_worker():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
fake_tensor = MagicMock()
|
||||
fake_backend = MagicMock()
|
||||
conn.register_cross_layers_kv_cache(fake_tensor, fake_backend)
|
||||
connector.register_cross_layers_kv_cache(fake_tensor, fake_backend)
|
||||
|
||||
worker_inst = mock_worker_cls.return_value
|
||||
worker_inst.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
|
||||
|
||||
|
||||
def test_update_connector_output_and_take_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
event = _make_block_stored()
|
||||
|
||||
with (
|
||||
@@ -247,12 +223,14 @@ def test_update_connector_output_and_take_events():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
|
||||
kv_events = connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events.add_events([event])
|
||||
conn.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
|
||||
assert conn._kv_cache_events is kv_events
|
||||
assert list(conn.take_events()) == [event]
|
||||
assert conn._kv_cache_events is None
|
||||
assert connector._kv_cache_events is kv_events
|
||||
assert list(connector.take_events()) == [event]
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from math import lcm
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
def _make_coord(groups, hash_block_size, use_eagle=False):
|
||||
"""Construct a coordinator using the natural LCM of group block sizes as
|
||||
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
|
||||
the test fixtures."""
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in groups]
|
||||
scheduler_block_size = lcm(*block_sizes)
|
||||
return MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
|
||||
|
||||
# ----- ExternalCachedBlockPool -----
|
||||
|
||||
|
||||
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
||||
cmap = ExternalCachedBlockPool()
|
||||
h = BlockHash(b"\xaa" * 4)
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
assert len(res) == 2
|
||||
assert res[0] is not cmap.null_block
|
||||
assert res[1] is not cmap.null_block
|
||||
|
||||
|
||||
def test_external_cached_block_pool_hit_all_groups():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
assert len(res) == 2
|
||||
assert res[0] is not cmap.null_block
|
||||
assert res[1] is not cmap.null_block
|
||||
|
||||
|
||||
def test_external_cached_block_pool_miss_one_group():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h))})
|
||||
assert cmap.get_cached_block(h, [0, 1]) is None
|
||||
|
||||
|
||||
def test_external_cached_block_pool_unknown_hash():
|
||||
h_known = BlockHash(b"\x01" * 4)
|
||||
h_unknown = BlockHash(b"\x02" * 4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
|
||||
assert cmap.get_cached_block(h_unknown, [0]) is None
|
||||
|
||||
|
||||
# ----- Helpers -----
|
||||
|
||||
|
||||
def _full(block_size=16, sliding_window=None):
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def _swa(block_size=16, sliding_window=32):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def _hashes(n: int) -> list[BlockHash]:
|
||||
return [BlockHash(bytes([i + 1]) * 4) for i in range(n)]
|
||||
|
||||
|
||||
# ----- Single-group coordinator -----
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_all_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
assert masks[0] == [True, True, True, True]
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_partial_prefix():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 32
|
||||
assert masks[0] == [True, True]
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_no_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool(set())
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 0
|
||||
assert masks[0] == []
|
||||
|
||||
|
||||
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
|
||||
"""SWA tautological-pool: hit_length spans full prefix, mask is
|
||||
tail-window only."""
|
||||
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4) # 4 chunks * 16 tokens
|
||||
cmap = ExternalCachedBlockPool()
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
|
||||
assert masks[0][-2:] == [True, True]
|
||||
assert all(not m for m in masks[0][:-2])
|
||||
|
||||
|
||||
# ----- Hybrid coordinator (single-group worker, multi-group coordinator) -----
|
||||
|
||||
|
||||
def test_coordinator_hybrid_full_plus_swa_all_hit():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 64
|
||||
|
||||
|
||||
def test_coordinator_hybrid_hole_in_full_clips_both():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
|
||||
exists |= {(1, bytes(h)) for h in hs}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 16
|
||||
|
||||
|
||||
def test_coordinator_group_block_size_double_hash():
|
||||
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
|
||||
hashes merge before pool lookup."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _full(32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32))
|
||||
exists = {(0, bytes(h)) for h in hs}
|
||||
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 64
|
||||
assert hit % 32 == 0
|
||||
|
||||
|
||||
# ----- store_mask -----
|
||||
|
||||
|
||||
def test_store_mask_full_attention_all_true():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True, True, True, True],)
|
||||
|
||||
|
||||
def test_store_mask_zero_aligned_returns_empty_per_group():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(0)
|
||||
assert masks == ([], [])
|
||||
|
||||
|
||||
def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
|
||||
aligned=64 the SWA group should mark exactly the blocks ending at 32
|
||||
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
|
||||
participate in any future hit."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
# Full-attn: 2 chunks * 32 tokens.
|
||||
assert masks[0] == [True, True]
|
||||
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
|
||||
assert masks[1] == [False, False, False, True, False, False, False, True]
|
||||
|
||||
|
||||
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
|
||||
boundary should now span two SWA tail blocks."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=16)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks[0] == [True, True]
|
||||
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
|
||||
# Boundary at 64: chunks 6 and 7. Others stay False.
|
||||
assert masks[1] == [False, False, True, True, False, False, True, True]
|
||||
|
||||
|
||||
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
|
||||
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
|
||||
(aligned_len=512). Validates that the tile-once strategy produces the
|
||||
expected per-segment tail-window pattern, repeated."""
|
||||
full_mla = _full(block_size=256)
|
||||
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
|
||||
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
|
||||
swa_4_sw16 = _swa(block_size=4, sliding_window=16)
|
||||
swa_8_sw64 = _swa(block_size=8, sliding_window=64)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full_mla),
|
||||
KVCacheGroupSpec(["L1"], swa_64_sw128),
|
||||
KVCacheGroupSpec(["L2"], swa_64_sw512),
|
||||
KVCacheGroupSpec(["L3"], swa_4_sw16),
|
||||
KVCacheGroupSpec(["L4"], swa_8_sw64),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=4)
|
||||
assert coord.lcm_block_size == 256
|
||||
masks = coord.store_mask(512)
|
||||
|
||||
# Full-MLA: 2 chunks of 256, both stored.
|
||||
assert masks[0] == [True, True]
|
||||
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
|
||||
# Per-segment template = [F,F,T,T]; tiled twice.
|
||||
assert masks[1] == [False, False, True, True] * 2
|
||||
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
|
||||
assert masks[2] == [True] * 8
|
||||
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
|
||||
# Last 4 of each 64-chunk segment True.
|
||||
assert masks[3] == ([False] * 60 + [True] * 4) * 2
|
||||
# SWA(8, sw=64): tail = ceil(63/8) = 8; C = 256/8 = 32.
|
||||
# Last 8 of each 32-chunk segment True.
|
||||
assert masks[4] == ([False] * 24 + [True] * 8) * 2
|
||||
|
||||
|
||||
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
|
||||
"""When every non-full-attn group already aligns to lcm_block_size, the
|
||||
fast path returns all-True without invoking find_longest_cache_hit."""
|
||||
full = _full(block_size=64)
|
||||
swa = _swa(block_size=64, sliding_window=128)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=64)
|
||||
assert coord.lcm_block_size == 64
|
||||
masks = coord.store_mask(256)
|
||||
# Every block in every group is True — no sub-lcm filtering possible.
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
|
||||
|
||||
def test_store_mask_fast_path_single_attention_group():
|
||||
"""Two groups sharing the same SWA spec collapse to one attention group;
|
||||
no lcm filter applies, every chunk is True."""
|
||||
swa = _swa(block_size=16, sliding_window=32)
|
||||
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
assert len(coord.attention_groups) == 1
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
@@ -0,0 +1,342 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
|
||||
(SWA + Full) attention config, using a dict-backed mock store."""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker as mooncake_store_worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
|
||||
KVCacheStoreRecvingThread,
|
||||
KVCacheStoreSendingThread,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
class _DictStore:
|
||||
"""In-memory MooncakeDistributedStore stand-in."""
|
||||
|
||||
def __init__(self):
|
||||
self._data: dict[str, bytes] = {}
|
||||
|
||||
def setup(self, *_args, **_kwargs):
|
||||
return 0
|
||||
|
||||
def register_buffer(self, addr, length):
|
||||
return 0
|
||||
|
||||
def batch_is_exist(self, keys):
|
||||
return [1 if k in self._data else 0 for k in keys]
|
||||
|
||||
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
|
||||
for k in keys:
|
||||
self._data[k] = b"x"
|
||||
return [0] * len(keys)
|
||||
|
||||
def batch_get_into_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
|
||||
return [0 if k in self._data else -1 for k in keys]
|
||||
|
||||
|
||||
def _minimal_vllm_config(cache_block_size=16):
|
||||
cfg = MagicMock()
|
||||
cfg.cache_config.block_size = cache_block_size
|
||||
cfg.cache_config.num_gpu_blocks = 4
|
||||
cfg.cache_config.hash_block_size = None
|
||||
cfg.cache_config.enable_prefix_caching = True
|
||||
cfg.parallel_config.prefill_context_parallel_size = 1
|
||||
cfg.parallel_config.decode_context_parallel_size = 1
|
||||
cfg.parallel_config.pipeline_parallel_size = 1
|
||||
cfg.parallel_config.world_size = 1
|
||||
cfg.parallel_config.rank = 0
|
||||
cfg.parallel_config.data_parallel_rank_local = 0
|
||||
cfg.parallel_config.data_parallel_size_local = 1
|
||||
cfg.kv_transfer_config.kv_role = "kv_both"
|
||||
cfg.kv_transfer_config.kv_connector_extra_config = {}
|
||||
cfg.kv_events_config = None
|
||||
cfg.model_config.model = "/tmp/m"
|
||||
cfg.model_config.use_mla = False
|
||||
cfg.model_config.get_num_layers.return_value = 2
|
||||
cfg.model_config.get_total_num_kv_heads.return_value = 8
|
||||
cfg.model_config.max_model_len = 4096
|
||||
cfg.scheduler_config.max_num_batched_tokens = 8192
|
||||
# Without this, MagicMock's truthy use_eagle() triggers the coordinator's
|
||||
# "use_eagle && nothing annotated → flag all groups" fallback.
|
||||
cfg.speculative_config = None
|
||||
return cfg
|
||||
|
||||
|
||||
def _build_worker_with_dict_store(vllm_config, kv_cache_config, store):
|
||||
"""Build a MooncakeStoreWorker patching all distributed dependencies."""
|
||||
fake_mooncake_store = types.ModuleType("mooncake.store")
|
||||
fake_mooncake_store.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined]
|
||||
fake_mooncake_store.ReplicateConfig = MagicMock # type: ignore[attr-defined]
|
||||
with (
|
||||
patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}),
|
||||
patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg,
|
||||
):
|
||||
sc = MCfg.load_from_env.return_value
|
||||
sc.metadata_server = ""
|
||||
sc.global_segment_size = 1 << 20
|
||||
sc.local_buffer_size = 1 << 20
|
||||
sc.protocol = "tcp"
|
||||
sc.device_name = ""
|
||||
sc.master_server_address = ""
|
||||
sc.enable_offload = False
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_pcp_group"
|
||||
) as mock_pcp,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_dcp_group"
|
||||
) as mock_dcp,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_ip",
|
||||
return_value="127.0.0.1",
|
||||
),
|
||||
):
|
||||
mock_pcp.return_value.world_size = 1
|
||||
mock_dcp.return_value.world_size = 1
|
||||
worker = mooncake_store_worker.MooncakeStoreWorker(
|
||||
vllm_config, kv_cache_config=kv_cache_config
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
"""
|
||||
E2E: build a SWA+Full hybrid worker, save all blocks via the sending
|
||||
thread (synchronously), then verify lookup returns the full hit length.
|
||||
Also verify that evicting SWA's early blocks (outside its window) still
|
||||
allows a full hit because the window covers the tail.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=32,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=8192, shared_by=["L0"]),
|
||||
KVCacheTensor(size=8192, shared_by=["L1"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], swa),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.put_step = 1
|
||||
worker.num_kv_head = 8
|
||||
|
||||
# Register kv_caches using mocked thread classes so register_kv_caches
|
||||
# doesn't try to start real background threads (which set ready_event).
|
||||
kv_caches = {
|
||||
"L0": torch.zeros(2, 4, 8, 8, 64),
|
||||
"L1": torch.zeros(2, 4, 8, 8, 64),
|
||||
}
|
||||
|
||||
def _fake_thread_init(*args, **kwargs):
|
||||
"""Mock thread that sets all threading.Event args so waits don't block."""
|
||||
for v in list(args) + list(kwargs.values()):
|
||||
if isinstance(v, threading.Event):
|
||||
v.set()
|
||||
m = MagicMock()
|
||||
m.start = lambda: None
|
||||
return m
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreSendingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreRecvingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
):
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
# Now build a real sending thread (no .start()) over the worker's token_dbs
|
||||
# and the dict-backed store, so _handle_request runs synchronously.
|
||||
ready = threading.Event()
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=worker.tp_rank,
|
||||
put_step=worker.put_step,
|
||||
kv_role=worker.kv_role,
|
||||
ready_event=ready,
|
||||
enable_kv_event=False,
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
send_thread.add_stored_request("r0")
|
||||
# Put the request in the queue so task_done() doesn't underflow.
|
||||
send_thread.request_queue.put(save_req)
|
||||
req = send_thread.request_queue.get()
|
||||
send_thread._handle_request(req)
|
||||
|
||||
# Point worker.store at the dict store (the worker constructor captured
|
||||
# the MagicMock; replace with the real dict store for lookup).
|
||||
worker.store = store
|
||||
|
||||
# Both groups stored all 4 blocks -> full hit.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
|
||||
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
|
||||
swa_keys_outside_window = [
|
||||
k
|
||||
for k in list(store._data.keys())
|
||||
if "@group:1" in k and (("@" + hs[0].hex()) in k or ("@" + hs[1].hex()) in k)
|
||||
]
|
||||
for k in swa_keys_outside_window:
|
||||
del store._data[k]
|
||||
|
||||
# SWA window=32 -> only last 2 blocks must be present in SWA group.
|
||||
# Full has all 4. Coordinator should still return 64.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
"""Producer stored every block for both groups; consumer must only fetch
|
||||
SWA blocks within the sliding window, not the head."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=32,
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], swa),
|
||||
]
|
||||
md0 = KeyMetadata("m", 0, 0, 0, 0, group_id=0)
|
||||
md1 = KeyMetadata("m", 0, 0, 0, 0, group_id=1)
|
||||
db_full = ChunkedTokenDatabase(md0, block_size=16, hash_block_size=16)
|
||||
db_swa = ChunkedTokenDatabase(md1, block_size=16, hash_block_size=16)
|
||||
db_full.set_kv_caches_base_addr([0])
|
||||
db_full.set_block_len([1024])
|
||||
db_swa.set_kv_caches_base_addr([1 << 20])
|
||||
db_swa.set_block_len([1024])
|
||||
|
||||
requested_keys: list[str] = []
|
||||
|
||||
class _CapturingStore:
|
||||
def batch_get_into_multi_buffers(self, keys, addrs, sizes):
|
||||
requested_keys.extend(keys)
|
||||
return [0] * len(keys)
|
||||
|
||||
ready = threading.Event()
|
||||
coord = MooncakeStoreCoordinator(
|
||||
groups, scheduler_block_size=16, hash_block_size=16
|
||||
)
|
||||
recv = KVCacheStoreRecvingThread(
|
||||
store=_CapturingStore(),
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=ready,
|
||||
coord=coord,
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
load_spec=LoadSpec(
|
||||
vllm_cached_tokens=0, kvpool_cached_tokens=64, can_load=True, token_len=64
|
||||
),
|
||||
)
|
||||
recv.request_queue.put(req)
|
||||
recv._handle_request(recv.request_queue.get())
|
||||
|
||||
full_keys = [k for k in requested_keys if "@group:0" in k]
|
||||
swa_keys = [k for k in requested_keys if "@group:1" in k]
|
||||
# Full attention: load every block (4).
|
||||
assert len(full_keys) == 4
|
||||
# SWA: only the 2 tail-window blocks (hashes hs[2], hs[3]).
|
||||
assert len(swa_keys) == 2
|
||||
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
|
||||
assert swa_hashes == {hs[2].hex(), hs[3].hex()}
|
||||
|
||||
|
||||
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
|
||||
must merge every 4 fine hashes into one chunk hash via
|
||||
BlockHashListWithBlockSize."""
|
||||
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
|
||||
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
|
||||
db.set_kv_caches_base_addr([0])
|
||||
db.set_block_len([512])
|
||||
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
|
||||
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
|
||||
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
|
||||
assert len(out) == 2
|
||||
assert out[0][0] == 0 and out[0][1] == 16
|
||||
assert out[1][0] == 16 and out[1][1] == 32
|
||||
# Each chunk's hash is the concatenation of 4 fine hashes.
|
||||
expected0 = b"".join(fine_hashes[0:4]).hex()
|
||||
expected1 = b"".join(fine_hashes[4:8]).hex()
|
||||
assert out[0][2].chunk_hash == expected0
|
||||
assert out[1][2].chunk_hash == expected1
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
|
||||
|
||||
def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.original_block_size = 16
|
||||
scheduler._block_size = 16
|
||||
scheduler._discard_partial_chunks = True
|
||||
scheduler.load_specs = {}
|
||||
scheduler._preempted_req_ids = set()
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
return scheduler
|
||||
|
||||
|
||||
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
num_computed_tokens=[44],
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens=(
|
||||
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _add_unfinished_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
token_ids: list[int],
|
||||
block_hashes: list[bytes],
|
||||
prefill_end_tokens: int,
|
||||
) -> None:
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=token_ids,
|
||||
block_hashes=block_hashes,
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=44,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=32,
|
||||
token_ids=token_ids[:44],
|
||||
prefill_end_tokens=prefill_end_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
|
||||
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
|
||||
# yet, so the tracker's token_len does not advance and num_tokens_to_save
|
||||
# stays below chunk_boundary — the save is naturally skipped.
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(44)),
|
||||
block_hashes=[b"h0", b"h1"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=[101, 102, 103])
|
||||
)
|
||||
|
||||
assert meta.requests == []
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.token_len == 44
|
||||
assert tracker.num_saved_tokens == 32
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
|
||||
|
||||
def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=None)
|
||||
)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 48
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.token_len == 48
|
||||
assert tracker.num_saved_tokens == 48
|
||||
@@ -19,7 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker as mooncake_store_worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
@@ -27,20 +30,37 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
)
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
return mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["layer0"], spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
|
||||
|
||||
def _make_store_sending_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
|
||||
token_databases: list[ChunkedTokenDatabase] | None = None,
|
||||
block_size: int = 16,
|
||||
replicate_config: object | None = None,
|
||||
) -> worker.KVCacheStoreSendingThread:
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
)
|
||||
token_database.set_kv_caches_base_addr([0x1000])
|
||||
token_database.set_block_len([256])
|
||||
thread = worker.KVCacheStoreSendingThread(
|
||||
) -> mooncake_store_worker.KVCacheStoreSendingThread:
|
||||
if coord is None:
|
||||
coord = _default_send_coord()
|
||||
if token_databases is None:
|
||||
db = ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([256])
|
||||
token_databases = [db]
|
||||
thread = mooncake_store_worker.KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_database=token_database,
|
||||
block_size=16,
|
||||
token_databases=token_databases,
|
||||
block_size=block_size,
|
||||
coord=coord,
|
||||
tp_rank=0,
|
||||
put_step=1,
|
||||
kv_role="kv_producer",
|
||||
@@ -55,18 +75,27 @@ def _make_store_recving_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
) -> worker.KVCacheStoreRecvingThread:
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
)
|
||||
token_database.set_kv_caches_base_addr([0x1000])
|
||||
token_database.set_block_len([256])
|
||||
thread = worker.KVCacheStoreRecvingThread(
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["layer0"], spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
|
||||
store=store,
|
||||
token_database=token_database,
|
||||
token_databases=[token_database],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
@@ -83,7 +112,7 @@ def _make_load_req(
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=token_len,
|
||||
block_ids=list(range(len(block_hashes))),
|
||||
block_ids=(list(range(len(block_hashes))),),
|
||||
block_hashes=block_hashes,
|
||||
load_spec=LoadSpec(
|
||||
vllm_cached_tokens=vllm_cached_tokens,
|
||||
@@ -98,7 +127,7 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=32,
|
||||
block_ids=[0, 1],
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=block_hashes,
|
||||
can_save=True,
|
||||
original_block_size=16,
|
||||
@@ -149,10 +178,31 @@ def _make_vllm_config(
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
rank=0,
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
),
|
||||
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
"""Minimal single-group KVCacheConfig for topology tests."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
)
|
||||
|
||||
|
||||
@@ -445,9 +495,9 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert sizes == [[256], [256], [256]]
|
||||
store.batch_get_replica_desc.assert_not_called()
|
||||
@@ -467,9 +517,9 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
|
||||
token_len=48,
|
||||
)
|
||||
expected_keys = [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
store.batch_get_replica_desc.return_value = {
|
||||
expected_keys[0]: [_ReplicaDesc("memory")],
|
||||
@@ -521,10 +571,10 @@ def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
|
||||
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
|
||||
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
]
|
||||
|
||||
|
||||
@@ -556,14 +606,14 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
|
||||
first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2]
|
||||
second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
base_addr = thread.token_database.kv_caches_base_addr[0]
|
||||
block_len = thread.token_database.block_len[0]
|
||||
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
|
||||
block_len = thread.token_databases[0].block_len[0]
|
||||
assert first_addrs == [[base_addr], [base_addr + block_len]]
|
||||
assert second_addrs == [[base_addr + 2 * block_len]]
|
||||
expected_size = block_len
|
||||
@@ -610,9 +660,9 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
|
||||
|
||||
@@ -654,7 +704,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert not hasattr(w, "_isolate_offload_resources")
|
||||
assert store.setup.call_args.args == (
|
||||
@@ -690,7 +740,7 @@ def test_requester_worker_init_prefers_local_hostname_override(
|
||||
},
|
||||
),
|
||||
)
|
||||
worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert store.setup.call_args.args[0] == "worker-a:50053"
|
||||
|
||||
@@ -718,7 +768,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
@@ -748,7 +798,8 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
extra_config={
|
||||
"preferred_segment": "10.0.0.7:50053",
|
||||
}
|
||||
)
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
@@ -760,6 +811,158 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
"""Partial chunks past the last lcm boundary aren't stored — cache hits
|
||||
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
# Default coord: single full-attn block_size=16, lcm=16.
|
||||
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=33,
|
||||
block_ids=([0, 1, 2],),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
original_block_size=16,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
assert len(keys) == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_skips_when_token_len_below_lcm():
|
||||
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
|
||||
so neither the existence check nor the put should be issued."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
store = MagicMock()
|
||||
# lcm=64 via single full-attn block_size=64.
|
||||
spec = FullAttentionSpec(block_size=64, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L"], spec)],
|
||||
scheduler_block_size=64,
|
||||
hash_block_size=64,
|
||||
)
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=64,
|
||||
hash_block_size=64,
|
||||
)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([1024])
|
||||
thread = _make_store_sending_thread(
|
||||
store, coord=coord, token_databases=[db], block_size=64
|
||||
)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=[b"a0", b"a1"],
|
||||
can_save=True,
|
||||
original_block_size=64,
|
||||
)
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["r0"] == 0
|
||||
|
||||
|
||||
def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
"""For SWA groups, only blocks within ``sliding_window`` of an
|
||||
lcm-aligned boundary are stored — a block outside every such window
|
||||
can never participate in any future hit (mirrors recv-side masking).
|
||||
|
||||
Setup: full-attn (block_size=32) + SWA (block_size=8, sliding_window=8)
|
||||
→ lcm=32. With token_len=64 there are two LCM boundaries (32, 64);
|
||||
the SWA group should store exactly one block per boundary: the block
|
||||
ending at 32 and the block ending at 64.
|
||||
"""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
|
||||
[256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
swa_spec = SlidingWindowSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=8,
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
|
||||
scheduler_block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([512])
|
||||
db_swa = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=8,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_swa.set_kv_caches_base_addr([0x2000])
|
||||
db_swa.set_block_len([128])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=32,
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
original_block_size=32,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
full_keys = [k for k in keys if "@group:0" in k]
|
||||
swa_keys = [k for k in keys if "@group:1" in k]
|
||||
# Full-attn: 2 blocks (chunks ending at 32 and 64).
|
||||
assert len(full_keys) == 2
|
||||
# SWA: only the two blocks ending at lcm boundaries (32 and 64), i.e.
|
||||
# blocks covering tokens [24,32) and [56,64) — hashes hs[3] and hs[7].
|
||||
assert len(swa_keys) == 2
|
||||
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
|
||||
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
|
||||
|
||||
|
||||
def _auto_set_ready_event(*args, **kwargs):
|
||||
"""Side effect for mocked thread constructors that auto-sets ready_event."""
|
||||
for arg in args:
|
||||
@@ -771,37 +974,108 @@ def _auto_set_ready_event(*args, **kwargs):
|
||||
return MagicMock()
|
||||
|
||||
|
||||
def _register_with_mocked_threads(
|
||||
worker: mooncake_store_worker.MooncakeStoreWorker,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
) -> None:
|
||||
"""Call register_kv_caches with the I/O transfer threads mocked out."""
|
||||
prefix = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker."
|
||||
with (
|
||||
patch(prefix + "KVCacheStoreSendingThread", side_effect=_auto_set_ready_event),
|
||||
patch(prefix + "KVCacheStoreRecvingThread", side_effect=_auto_set_ready_event),
|
||||
):
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
|
||||
def _make_bare_worker(
|
||||
*,
|
||||
num_gpu_blocks: int = 10,
|
||||
block_size: int = 16,
|
||||
kv_role: str = "kv_both",
|
||||
) -> worker.MooncakeStoreWorker:
|
||||
) -> mooncake_store_worker.MooncakeStoreWorker:
|
||||
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
|
||||
|
||||
Sets only the attributes that register_kv_caches() reads so we can
|
||||
test the stride-based layout detection without a real
|
||||
MooncakeDistributedStore.
|
||||
"""
|
||||
w = object.__new__(worker.MooncakeStoreWorker)
|
||||
w.cache_config = MagicMock()
|
||||
w.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
w.store = MagicMock()
|
||||
w.store.register_buffer.return_value = 0
|
||||
w.use_mla = False
|
||||
w.token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=block_size
|
||||
worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
|
||||
worker.cache_config = MagicMock()
|
||||
worker.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
worker.store = MagicMock()
|
||||
worker.store.register_buffer.return_value = 0
|
||||
worker.use_mla = False
|
||||
worker.kv_role = kv_role
|
||||
worker.block_size = block_size
|
||||
worker.tp_rank = 0
|
||||
worker.put_step = 1
|
||||
worker.enable_kv_events = False
|
||||
worker.kv_send_thread = None
|
||||
worker.kv_recv_thread = None
|
||||
worker.tp_size = 1
|
||||
worker.num_kv_head = 1
|
||||
worker.pp_size = 1
|
||||
# Minimal single-full-attention-group config so the coordinator-based
|
||||
# lookup path works (the connector no longer carries a legacy single-group
|
||||
# path; everything flows through the coordinator).
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
w.kv_role = kv_role
|
||||
w.block_size = block_size
|
||||
w.tp_rank = 0
|
||||
w.put_step = 1
|
||||
w.enable_kv_events = False
|
||||
w.disk_offload_buffer_budget_bytes = None
|
||||
w.kv_send_thread = None
|
||||
w.kv_recv_thread = None
|
||||
w.store_replicate_config = SimpleNamespace()
|
||||
return w
|
||||
|
||||
worker.disk_offload_buffer_budget_bytes = None
|
||||
worker.store_replicate_config = SimpleNamespace()
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
|
||||
worker._kv_cache_groups = [group]
|
||||
worker.pcp_size = 1
|
||||
worker.dcp_size = 1
|
||||
worker.hash_block_size = block_size
|
||||
worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0)
|
||||
# Pre-build a single-group token_dbs so lookup-only tests don't have to
|
||||
# call register_kv_caches.
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 0]
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
|
||||
|
||||
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
|
||||
tail. Coordinator-driven lookup returns full prefix even though the
|
||||
pre-window blocks are absent."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
|
||||
)
|
||||
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=worker.hash_block_size,
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -812,34 +1086,17 @@ def _make_bare_worker(
|
||||
def test_register_kv_caches_blocks_first_single_segment():
|
||||
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
|
||||
num_blocks = 10
|
||||
page_size_elements = 64 # elements per block
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
page_size_elements = 64
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Shape: (num_blocks, page_size_elements) — blocks outermost, no outer_dims
|
||||
tensor = torch.zeros(num_blocks, page_size_elements, dtype=torch.float16)
|
||||
_register_with_mocked_threads(worker, {"layer0": tensor})
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreSendingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreRecvingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w.register_kv_caches({"layer0": tensor})
|
||||
|
||||
assert len(w.kv_caches_base_addr) == 1
|
||||
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
|
||||
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
|
||||
assert len(w.block_len) == 1
|
||||
assert w.block_len[0] == expected_block_len
|
||||
|
||||
w.store.register_buffer.assert_called_once_with(
|
||||
db = worker.token_dbs[0]
|
||||
assert db.kv_caches_base_addr == [tensor.untyped_storage().data_ptr()]
|
||||
assert db.block_len == [tensor.untyped_storage().nbytes() // num_blocks]
|
||||
worker.store.register_buffer.assert_called_once_with(
|
||||
tensor.untyped_storage().data_ptr(),
|
||||
tensor.untyped_storage().nbytes(),
|
||||
)
|
||||
@@ -851,8 +1108,7 @@ def test_register_kv_caches_kv_first_two_segments():
|
||||
block_size_tokens = 16
|
||||
num_kv_heads = 4
|
||||
head_size = 8
|
||||
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Shape: (2, num_blocks, block_size, num_kv_heads, head_size) — K/V outermost
|
||||
tensor = torch.zeros(
|
||||
@@ -863,32 +1119,13 @@ def test_register_kv_caches_kv_first_two_segments():
|
||||
head_size,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
_register_with_mocked_threads(worker, {"layer0": tensor})
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreSendingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreRecvingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w.register_kv_caches({"layer0": tensor})
|
||||
|
||||
# K/V-first: dim 0 has stride > page_size, so 2 segments
|
||||
assert len(w.kv_caches_base_addr) == 2
|
||||
assert len(w.block_len) == 2
|
||||
|
||||
el = tensor.element_size()
|
||||
seg_stride = tensor.stride(0) * el # stride of the K/V dim in bytes
|
||||
db = worker.token_dbs[0]
|
||||
seg_stride = tensor.stride(0) * tensor.element_size()
|
||||
base = tensor.untyped_storage().data_ptr()
|
||||
assert w.kv_caches_base_addr[0] == base
|
||||
assert w.kv_caches_base_addr[1] == base + seg_stride
|
||||
assert w.block_len[0] == seg_stride // num_blocks
|
||||
assert w.block_len[1] == seg_stride // num_blocks
|
||||
assert db.kv_caches_base_addr == [base, base + seg_stride]
|
||||
assert db.block_len == [seg_stride // num_blocks] * 2
|
||||
|
||||
|
||||
def test_register_kv_caches_cross_layer_single_segment():
|
||||
@@ -897,7 +1134,7 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
num_layers = 4
|
||||
per_layer_page_elements = 64 # elements per layer per block
|
||||
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Cross-layer blocks-first tensor: all layers packed into a single
|
||||
# contiguous block. Shape (num_blocks, num_layers * per_layer_page)
|
||||
@@ -918,10 +1155,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
),
|
||||
):
|
||||
# Use the cross-layer wrapper key, same as register_cross_layers_kv_caches
|
||||
w.register_kv_caches({"__cross_layer__": tensor})
|
||||
worker.register_kv_caches({"__cross_layer__": tensor})
|
||||
|
||||
assert len(w.kv_caches_base_addr) == 1
|
||||
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
db = worker.token_dbs[0]
|
||||
assert len(db.kv_caches_base_addr) == 1
|
||||
assert db.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
|
||||
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
|
||||
# block_len should be per_layer_page_size * num_layers
|
||||
@@ -929,11 +1167,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
expected_block_len
|
||||
== num_layers * per_layer_page_elements * tensor.element_size()
|
||||
)
|
||||
assert len(w.block_len) == 1
|
||||
assert w.block_len[0] == expected_block_len
|
||||
assert len(db.block_len) == 1
|
||||
assert db.block_len[0] == expected_block_len
|
||||
|
||||
# Also verify via register_cross_layers_kv_caches wrapper
|
||||
w2 = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker2 = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
@@ -946,10 +1184,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w2.register_cross_layers_kv_caches(tensor)
|
||||
worker2.register_cross_layers_kv_caches(tensor)
|
||||
|
||||
assert w2.kv_caches_base_addr == w.kv_caches_base_addr
|
||||
assert w2.block_len == w.block_len
|
||||
db2 = worker2.token_dbs[0]
|
||||
assert db2.kv_caches_base_addr == db.kv_caches_base_addr
|
||||
assert db2.block_len == db.block_len
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1070,7 +1309,8 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"})
|
||||
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
# setup() receives global_segment_size=0 and the configured local buffer.
|
||||
@@ -1114,7 +1354,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
# setup() receives global_segment_size=4 GiB (rank contributes a segment).
|
||||
assert store.setup.call_args.args == (
|
||||
|
||||
Reference in New Issue
Block a user