diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index f1715c1d98..620b874531 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -9,12 +9,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( - connector, - worker, + connector as mooncake_store_connector, ) -from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( MooncakeStoreConnectorMetadata, ) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + KVCacheTensor, +) from vllm.v1.outputs import KVConnectorOutput from .utils import create_vllm_config @@ -27,6 +32,17 @@ def _make_vllm_config(): ) +def _make_kv_cache_config() -> KVCacheConfig: + """Single-group full-attention KVCacheConfig — enough for the connector + constructor's validate() pass.""" + spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + return KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[KVCacheTensor(size=8192, shared_by=["layer0"])], + kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)], + ) + + def _make_block_stored() -> BlockStored: return BlockStored( block_hashes=[b"hash"], @@ -41,6 +57,7 @@ def _make_block_stored() -> BlockStored: def test_scheduler_role_initializes_store_scheduler_only(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() with ( set_current_vllm_config(vllm_config), @@ -53,75 +70,19 @@ def test_scheduler_role_initializes_store_scheduler_only(): "connector.MooncakeStoreWorker" ) as mock_worker, ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) - mock_scheduler.assert_called_once_with(vllm_config) + mock_scheduler.assert_called_once_with(vllm_config, kv_cache_config) mock_worker.assert_not_called() - assert conn.connector_scheduler is mock_scheduler.return_value - assert conn.connector_worker is None - - -def test_worker_role_initializes_store_worker_on_rank0(): - vllm_config = _make_vllm_config() - - with ( - set_current_vllm_config(vllm_config), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "connector.MooncakeStoreScheduler" - ) as mock_scheduler, - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "connector.MooncakeStoreWorker" - ) as mock_worker, - ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) - - mock_scheduler.assert_not_called() - mock_worker.assert_called_once_with(vllm_config) - assert conn.connector_scheduler is None - assert conn.connector_worker is mock_worker.return_value - - -def test_worker_role_initializes_on_nonzero_rank(): - vllm_config = _make_vllm_config() - vllm_config.parallel_config.rank = 1 - - with ( - set_current_vllm_config(vllm_config), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "connector.MooncakeStoreWorker" - ) as mock_worker, - ): - connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) - - mock_worker.assert_called_once_with(vllm_config) - - -def test_lookup_rpc_path_uses_data_parallel_index_in_dense_dp(): - vllm_config = _make_vllm_config() - vllm_config.parallel_config.data_parallel_rank = 0 - vllm_config.parallel_config.data_parallel_index = 3 - - path = worker.get_zmq_rpc_path_lookup(vllm_config) - - assert path.endswith("_dp_rank3") - - -def test_lookup_rpc_path_uses_local_rank_when_local_engines_only(): - vllm_config = _make_vllm_config() - vllm_config.parallel_config.data_parallel_index = 7 - vllm_config.parallel_config.data_parallel_rank_local = 1 - vllm_config.parallel_config.data_parallel_hybrid_lb = True - - path = worker.get_zmq_rpc_path_lookup(vllm_config) - - assert path.endswith("_dp_rank1") + assert connector.connector_scheduler is mock_scheduler.return_value + assert connector.connector_worker is None def test_worker_methods_delegate_to_store_worker(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() kv_caches = {"layer0": MagicMock()} metadata = MooncakeStoreConnectorMetadata(set(), set()) finished_req_ids = {"req-1"} @@ -133,22 +94,25 @@ def test_worker_methods_delegate_to_store_worker(): "connector.MooncakeStoreWorker" ) as mock_worker_cls, ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) - worker_inst = mock_worker_cls.return_value - worker_inst.get_finished.return_value = ({"req-1"}, {"req-2"}) - conn.bind_connector_metadata(metadata) + worker = mock_worker_cls.return_value + worker.get_finished.return_value = ({"req-1"}, {"req-2"}) + connector.bind_connector_metadata(metadata) - conn.register_kv_caches(kv_caches) - result = conn.get_finished(finished_req_ids) + connector.register_kv_caches(kv_caches) + result = connector.get_finished(finished_req_ids) - worker_inst.register_kv_caches.assert_called_once_with(kv_caches) - worker_inst.get_finished.assert_called_once_with(finished_req_ids, metadata) + worker.register_kv_caches.assert_called_once_with(kv_caches) + worker.get_finished.assert_called_once_with(finished_req_ids, metadata) assert result == ({"req-1"}, {"req-2"}) def test_get_kv_connector_kv_cache_events_returns_none_when_empty(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() with ( set_current_vllm_config(vllm_config), @@ -157,14 +121,17 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty(): "connector.MooncakeStoreWorker" ) as mock_worker_cls, ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) mock_worker_cls.return_value.get_kv_events.return_value = [] - assert conn.get_kv_connector_kv_cache_events() is None + assert connector.get_kv_connector_kv_cache_events() is None def test_get_kv_connector_kv_cache_events_wraps_worker_events(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() event = _make_block_stored() with ( @@ -174,12 +141,14 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events(): "connector.MooncakeStoreWorker" ) as mock_worker_cls, ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) mock_worker_cls.return_value.get_kv_events.return_value = [event] - kv_events = conn.get_kv_connector_kv_cache_events() + kv_events = connector.get_kv_connector_kv_cache_events() - assert isinstance(kv_events, connector.MooncakeStoreKVEvents) + assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents) assert kv_events.get_number_of_workers() == 1 assert kv_events.get_all_events() == [event] @@ -187,6 +156,7 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events(): def test_prefer_cross_layer_blocks_from_config(): # Default: disabled vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() with ( set_current_vllm_config(vllm_config), patch( @@ -194,8 +164,10 @@ def test_prefer_cross_layer_blocks_from_config(): "connector.MooncakeStoreScheduler" ), ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER) - assert conn.prefer_cross_layer_blocks is False + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) + assert connector.prefer_cross_layer_blocks is False # Enabled via config vllm_config_enabled = create_vllm_config( @@ -210,14 +182,15 @@ def test_prefer_cross_layer_blocks_from_config(): "connector.MooncakeStoreScheduler" ), ): - conn_enabled = connector.MooncakeStoreConnector( - vllm_config_enabled, KVConnectorRole.SCHEDULER + connector_enabled = mooncake_store_connector.MooncakeStoreConnector( + vllm_config_enabled, KVConnectorRole.SCHEDULER, kv_cache_config ) - assert conn_enabled.prefer_cross_layer_blocks is True + assert connector_enabled.prefer_cross_layer_blocks is True def test_register_cross_layers_kv_cache_delegates_to_worker(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() with ( set_current_vllm_config(vllm_config), @@ -226,18 +199,21 @@ def test_register_cross_layers_kv_cache_delegates_to_worker(): "connector.MooncakeStoreWorker" ) as mock_worker_cls, ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) fake_tensor = MagicMock() fake_backend = MagicMock() - conn.register_cross_layers_kv_cache(fake_tensor, fake_backend) + connector.register_cross_layers_kv_cache(fake_tensor, fake_backend) - worker_inst = mock_worker_cls.return_value - worker_inst.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor) + worker = mock_worker_cls.return_value + worker.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor) def test_update_connector_output_and_take_events(): vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() event = _make_block_stored() with ( @@ -247,12 +223,14 @@ def test_update_connector_output_and_take_events(): "connector.MooncakeStoreScheduler" ), ): - conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER) + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) - kv_events = connector.MooncakeStoreKVEvents(num_workers=1) + kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1) kv_events.add_events([event]) - conn.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events)) + connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events)) - assert conn._kv_cache_events is kv_events - assert list(conn.take_events()) == [event] - assert conn._kv_cache_events is None + assert connector._kv_cache_events is kv_events + assert list(connector.take_events()) == [event] + assert connector._kv_cache_events is None diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py new file mode 100644 index 0000000000..1fd601af59 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py @@ -0,0 +1,302 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from math import lcm + +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 + ExternalCachedBlockPool, + MooncakeStoreCoordinator, +) +from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + SlidingWindowSpec, +) + + +def _make_coord(groups, hash_block_size, use_eagle=False): + """Construct a coordinator using the natural LCM of group block sizes as + the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for + the test fixtures.""" + block_sizes = [g.kv_cache_spec.block_size for g in groups] + scheduler_block_size = lcm(*block_sizes) + return MooncakeStoreCoordinator( + groups, + scheduler_block_size=scheduler_block_size, + hash_block_size=hash_block_size, + use_eagle=use_eagle, + ) + + +# ----- ExternalCachedBlockPool ----- + + +def test_external_cached_block_pool_tautological_returns_present_for_any_hash(): + cmap = ExternalCachedBlockPool() + h = BlockHash(b"\xaa" * 4) + res = cmap.get_cached_block(h, [0, 1]) + assert res is not None + assert len(res) == 2 + assert res[0] is not cmap.null_block + assert res[1] is not cmap.null_block + + +def test_external_cached_block_pool_hit_all_groups(): + h = BlockHash(b"\x11\x22\x33\x44") + cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))}) + res = cmap.get_cached_block(h, [0, 1]) + assert res is not None + assert len(res) == 2 + assert res[0] is not cmap.null_block + assert res[1] is not cmap.null_block + + +def test_external_cached_block_pool_miss_one_group(): + h = BlockHash(b"\x11\x22\x33\x44") + cmap = ExternalCachedBlockPool({(0, bytes(h))}) + assert cmap.get_cached_block(h, [0, 1]) is None + + +def test_external_cached_block_pool_unknown_hash(): + h_known = BlockHash(b"\x01" * 4) + h_unknown = BlockHash(b"\x02" * 4) + cmap = ExternalCachedBlockPool({(0, bytes(h_known))}) + assert cmap.get_cached_block(h_unknown, [0]) is None + + +# ----- Helpers ----- + + +def _full(block_size=16, sliding_window=None): + return FullAttentionSpec( + block_size=block_size, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=sliding_window, + ) + + +def _swa(block_size=16, sliding_window=32): + return SlidingWindowSpec( + block_size=block_size, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=sliding_window, + ) + + +def _hashes(n: int) -> list[BlockHash]: + return [BlockHash(bytes([i + 1]) * 4) for i in range(n)] + + +# ----- Single-group coordinator ----- + + +def test_coordinator_single_full_attention_all_hits(): + groups = [KVCacheGroupSpec(["L0"], _full(16))] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs}) + masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap) + assert hit == 64 + assert masks[0] == [True, True, True, True] + + +def test_coordinator_single_full_attention_partial_prefix(): + groups = [KVCacheGroupSpec(["L0"], _full(16))] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))}) + masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap) + assert hit == 32 + assert masks[0] == [True, True] + + +def test_coordinator_single_full_attention_no_hits(): + groups = [KVCacheGroupSpec(["L0"], _full(16))] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + cmap = ExternalCachedBlockPool(set()) + masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap) + assert hit == 0 + assert masks[0] == [] + + +def test_coordinator_single_swa_tautological_pool_masks_pre_window(): + """SWA tautological-pool: hit_length spans full prefix, mask is + tail-window only.""" + groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) # 4 chunks * 16 tokens + cmap = ExternalCachedBlockPool() + masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap) + assert hit == 64 + # ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks. + assert masks[0][-2:] == [True, True] + assert all(not m for m in masks[0][:-2]) + + +# ----- Hybrid coordinator (single-group worker, multi-group coordinator) ----- + + +def test_coordinator_hybrid_full_plus_swa_all_hit(): + groups = [ + KVCacheGroupSpec(["L0"], _full(16)), + KVCacheGroupSpec(["L1"], _swa(16, 32)), + ] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs}) + _masks, hit = coord.find_longest_cache_hit( + hs, max_length=64, cached_block_pool=cmap + ) + assert hit == 64 + + +def test_coordinator_hybrid_hole_in_full_clips_both(): + groups = [ + KVCacheGroupSpec(["L0"], _full(16)), + KVCacheGroupSpec(["L1"], _swa(16, 32)), + ] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))} + exists |= {(1, bytes(h)) for h in hs} + cmap = ExternalCachedBlockPool(exists) + _masks, hit = coord.find_longest_cache_hit( + hs, max_length=64, cached_block_pool=cmap + ) + assert hit == 16 + + +def test_coordinator_group_block_size_double_hash(): + """Group block_size=32 over hash_block_size=16 hashes: adjacent + hashes merge before pool lookup.""" + groups = [ + KVCacheGroupSpec(["L0"], _full(16)), + KVCacheGroupSpec(["L1"], _full(32)), + ] + coord = _make_coord(groups, hash_block_size=16) + hs = _hashes(4) + big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32)) + exists = {(0, bytes(h)) for h in hs} + exists |= {(1, bytes(bh)) for bh in big_hashes} + cmap = ExternalCachedBlockPool(exists) + _masks, hit = coord.find_longest_cache_hit( + hs, max_length=64, cached_block_pool=cmap + ) + assert hit == 64 + assert hit % 32 == 0 + + +# ----- store_mask ----- + + +def test_store_mask_full_attention_all_true(): + groups = [KVCacheGroupSpec(["L0"], _full(16))] + coord = _make_coord(groups, hash_block_size=16) + masks = coord.store_mask(64) + assert masks == ([True, True, True, True],) + + +def test_store_mask_zero_aligned_returns_empty_per_group(): + groups = [ + KVCacheGroupSpec(["L0"], _full(16)), + KVCacheGroupSpec(["L1"], _swa(16, 32)), + ] + coord = _make_coord(groups, hash_block_size=16) + masks = coord.store_mask(0) + assert masks == ([], []) + + +def test_store_mask_swa_only_window_around_each_lcm_boundary(): + """Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With + aligned=64 the SWA group should mark exactly the blocks ending at 32 + and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never + participate in any future hit.""" + full = _full(32) + swa = _swa(block_size=8, sliding_window=8) + groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)] + coord = _make_coord(groups, hash_block_size=8) + masks = coord.store_mask(64) + # Full-attn: 2 chunks * 32 tokens. + assert masks[0] == [True, True] + # SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored. + assert masks[1] == [False, False, False, True, False, False, False, True] + + +def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm(): + """Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm + boundary should now span two SWA tail blocks.""" + full = _full(32) + swa = _swa(block_size=8, sliding_window=16) + groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)] + coord = _make_coord(groups, hash_block_size=8) + masks = coord.store_mask(64) + assert masks[0] == [True, True] + # Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3. + # Boundary at 64: chunks 6 and 7. Others stay False. + assert masks[1] == [False, False, True, True, False, False, True, True] + + +def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa(): + """DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8} + and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments + (aligned_len=512). Validates that the tile-once strategy produces the + expected per-segment tail-window pattern, repeated.""" + full_mla = _full(block_size=256) + swa_64_sw128 = _swa(block_size=64, sliding_window=128) + swa_64_sw512 = _swa(block_size=64, sliding_window=512) + swa_4_sw16 = _swa(block_size=4, sliding_window=16) + swa_8_sw64 = _swa(block_size=8, sliding_window=64) + groups = [ + KVCacheGroupSpec(["L0"], full_mla), + KVCacheGroupSpec(["L1"], swa_64_sw128), + KVCacheGroupSpec(["L2"], swa_64_sw512), + KVCacheGroupSpec(["L3"], swa_4_sw16), + KVCacheGroupSpec(["L4"], swa_8_sw64), + ] + coord = _make_coord(groups, hash_block_size=4) + assert coord.lcm_block_size == 256 + masks = coord.store_mask(512) + + # Full-MLA: 2 chunks of 256, both stored. + assert masks[0] == [True, True] + # SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4. + # Per-segment template = [F,F,T,T]; tiled twice. + assert masks[1] == [False, False, True, True] * 2 + # SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True. + assert masks[2] == [True] * 8 + # SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64. + # Last 4 of each 64-chunk segment True. + assert masks[3] == ([False] * 60 + [True] * 4) * 2 + # SWA(8, sw=64): tail = ceil(63/8) = 8; C = 256/8 = 32. + # Last 8 of each 32-chunk segment True. + assert masks[4] == ([False] * 24 + [True] * 8) * 2 + + +def test_store_mask_fast_path_all_block_sizes_equal_lcm(): + """When every non-full-attn group already aligns to lcm_block_size, the + fast path returns all-True without invoking find_longest_cache_hit.""" + full = _full(block_size=64) + swa = _swa(block_size=64, sliding_window=128) + groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)] + coord = _make_coord(groups, hash_block_size=64) + assert coord.lcm_block_size == 64 + masks = coord.store_mask(256) + # Every block in every group is True — no sub-lcm filtering possible. + assert masks == ([True] * 4, [True] * 4) + + +def test_store_mask_fast_path_single_attention_group(): + """Two groups sharing the same SWA spec collapse to one attention group; + no lcm filter applies, every chunk is True.""" + swa = _swa(block_size=16, sliding_window=32) + groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)] + coord = _make_coord(groups, hash_block_size=16) + assert len(coord.attention_groups) == 1 + masks = coord.store_mask(64) + assert masks == ([True] * 4, [True] * 4) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py new file mode 100644 index 0000000000..01d4f4821e --- /dev/null +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -0,0 +1,342 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid +(SWA + Full) attention config, using a dict-backed mock store.""" + +import sys +import threading +import types +from unittest.mock import MagicMock, patch + +import torch + +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( + worker as mooncake_store_worker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 + MooncakeStoreCoordinator, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + ChunkedTokenDatabase, + KeyMetadata, + LoadSpec, + ReqMeta, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501 + KVCacheStoreRecvingThread, + KVCacheStoreSendingThread, +) +from vllm.v1.core.kv_cache_utils import BlockHash +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + KVCacheTensor, + SlidingWindowSpec, +) + + +class _DictStore: + """In-memory MooncakeDistributedStore stand-in.""" + + def __init__(self): + self._data: dict[str, bytes] = {} + + def setup(self, *_args, **_kwargs): + return 0 + + def register_buffer(self, addr, length): + return 0 + + def batch_is_exist(self, keys): + return [1 if k in self._data else 0 for k in keys] + + def batch_put_from_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs): + for k in keys: + self._data[k] = b"x" + return [0] * len(keys) + + def batch_get_into_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs): + return [0 if k in self._data else -1 for k in keys] + + +def _minimal_vllm_config(cache_block_size=16): + cfg = MagicMock() + cfg.cache_config.block_size = cache_block_size + cfg.cache_config.num_gpu_blocks = 4 + cfg.cache_config.hash_block_size = None + cfg.cache_config.enable_prefix_caching = True + cfg.parallel_config.prefill_context_parallel_size = 1 + cfg.parallel_config.decode_context_parallel_size = 1 + cfg.parallel_config.pipeline_parallel_size = 1 + cfg.parallel_config.world_size = 1 + cfg.parallel_config.rank = 0 + cfg.parallel_config.data_parallel_rank_local = 0 + cfg.parallel_config.data_parallel_size_local = 1 + cfg.kv_transfer_config.kv_role = "kv_both" + cfg.kv_transfer_config.kv_connector_extra_config = {} + cfg.kv_events_config = None + cfg.model_config.model = "/tmp/m" + cfg.model_config.use_mla = False + cfg.model_config.get_num_layers.return_value = 2 + cfg.model_config.get_total_num_kv_heads.return_value = 8 + cfg.model_config.max_model_len = 4096 + cfg.scheduler_config.max_num_batched_tokens = 8192 + # Without this, MagicMock's truthy use_eagle() triggers the coordinator's + # "use_eagle && nothing annotated → flag all groups" fallback. + cfg.speculative_config = None + return cfg + + +def _build_worker_with_dict_store(vllm_config, kv_cache_config, store): + """Build a MooncakeStoreWorker patching all distributed dependencies.""" + fake_mooncake_store = types.ModuleType("mooncake.store") + fake_mooncake_store.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined] + fake_mooncake_store.ReplicateConfig = MagicMock # type: ignore[attr-defined] + with ( + patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}), + patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg, + ): + sc = MCfg.load_from_env.return_value + sc.metadata_server = "" + sc.global_segment_size = 1 << 20 + sc.local_buffer_size = 1 << 20 + sc.protocol = "tcp" + sc.device_name = "" + sc.master_server_address = "" + sc.enable_offload = False + with ( + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake" + ".store.worker.get_tensor_model_parallel_rank", + return_value=0, + ), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake" + ".store.worker.get_tensor_model_parallel_world_size", + return_value=1, + ), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake" + ".store.worker.get_pcp_group" + ) as mock_pcp, + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake" + ".store.worker.get_dcp_group" + ) as mock_dcp, + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake" + ".store.worker.get_ip", + return_value="127.0.0.1", + ), + ): + mock_pcp.return_value.world_size = 1 + mock_dcp.return_value.world_size = 1 + worker = mooncake_store_worker.MooncakeStoreWorker( + vllm_config, kv_cache_config=kv_cache_config + ) + return worker + + +def test_e2e_swa_plus_full_save_then_lookup_hits(): + """ + E2E: build a SWA+Full hybrid worker, save all blocks via the sending + thread (synchronously), then verify lookup returns the full hit length. + Also verify that evicting SWA's early blocks (outside its window) still + allows a full hit because the window covers the tail. + """ + full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + swa = SlidingWindowSpec( + block_size=16, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=32, + ) + cfg = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor(size=8192, shared_by=["L0"]), + KVCacheTensor(size=8192, shared_by=["L1"]), + ], + kv_cache_groups=[ + KVCacheGroupSpec(["L0"], full), + KVCacheGroupSpec(["L1"], swa), + ], + ) + vllm_config = _minimal_vllm_config(cache_block_size=16) + store = _DictStore() + + worker = _build_worker_with_dict_store(vllm_config, cfg, store) + worker.tp_size = 1 + worker.pp_size = 1 + worker.put_step = 1 + worker.num_kv_head = 8 + + # Register kv_caches using mocked thread classes so register_kv_caches + # doesn't try to start real background threads (which set ready_event). + kv_caches = { + "L0": torch.zeros(2, 4, 8, 8, 64), + "L1": torch.zeros(2, 4, 8, 8, 64), + } + + def _fake_thread_init(*args, **kwargs): + """Mock thread that sets all threading.Event args so waits don't block.""" + for v in list(args) + list(kwargs.values()): + if isinstance(v, threading.Event): + v.set() + m = MagicMock() + m.start = lambda: None + return m + + with ( + patch.object( + mooncake_store_worker, + "KVCacheStoreSendingThread", + side_effect=_fake_thread_init, + ), + patch.object( + mooncake_store_worker, + "KVCacheStoreRecvingThread", + side_effect=_fake_thread_init, + ), + ): + worker.register_kv_caches(kv_caches) + + # Now build a real sending thread (no .start()) over the worker's token_dbs + # and the dict-backed store, so _handle_request runs synchronously. + ready = threading.Event() + send_thread = KVCacheStoreSendingThread( + store=store, + token_databases=worker.token_dbs, + block_size=worker.block_size, + coord=worker.coord, + tp_rank=worker.tp_rank, + put_step=worker.put_step, + kv_role=worker.kv_role, + ready_event=ready, + enable_kv_event=False, + ) + + hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)] + save_req = ReqMeta( + req_id="r0", + token_len_chunk=64, + block_ids=([0, 1, 2, 3], [0, 1, 2, 3]), + block_hashes=hs, + can_save=True, + ) + send_thread.add_stored_request("r0") + # Put the request in the queue so task_done() doesn't underflow. + send_thread.request_queue.put(save_req) + req = send_thread.request_queue.get() + send_thread._handle_request(req) + + # Point worker.store at the dict store (the worker constructor captured + # the MagicMock; replace with the real dict store for lookup). + worker.store = store + + # Both groups stored all 4 blocks -> full hit. + assert worker.lookup(token_len=64, block_hashes=hs) == 64 + + # Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks). + swa_keys_outside_window = [ + k + for k in list(store._data.keys()) + if "@group:1" in k and (("@" + hs[0].hex()) in k or ("@" + hs[1].hex()) in k) + ] + for k in swa_keys_outside_window: + del store._data[k] + + # SWA window=32 -> only last 2 blocks must be present in SWA group. + # Full has all 4. Coordinator should still return 64. + assert worker.lookup(token_len=64, block_hashes=hs) == 64 + + +def test_recv_skips_swa_blocks_before_window(): + """Producer stored every block for both groups; consumer must only fetch + SWA blocks within the sliding window, not the head.""" + full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + # sliding_window=32, block_size=16 → 2 contiguous blocks within window. + swa = SlidingWindowSpec( + block_size=16, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=32, + ) + groups = [ + KVCacheGroupSpec(["L0"], full), + KVCacheGroupSpec(["L1"], swa), + ] + md0 = KeyMetadata("m", 0, 0, 0, 0, group_id=0) + md1 = KeyMetadata("m", 0, 0, 0, 0, group_id=1) + db_full = ChunkedTokenDatabase(md0, block_size=16, hash_block_size=16) + db_swa = ChunkedTokenDatabase(md1, block_size=16, hash_block_size=16) + db_full.set_kv_caches_base_addr([0]) + db_full.set_block_len([1024]) + db_swa.set_kv_caches_base_addr([1 << 20]) + db_swa.set_block_len([1024]) + + requested_keys: list[str] = [] + + class _CapturingStore: + def batch_get_into_multi_buffers(self, keys, addrs, sizes): + requested_keys.extend(keys) + return [0] * len(keys) + + ready = threading.Event() + coord = MooncakeStoreCoordinator( + groups, scheduler_block_size=16, hash_block_size=16 + ) + recv = KVCacheStoreRecvingThread( + store=_CapturingStore(), + token_databases=[db_full, db_swa], + block_size=16, + tp_rank=0, + ready_event=ready, + coord=coord, + ) + + hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)] + req = ReqMeta( + req_id="r0", + token_len_chunk=64, + block_ids=([0, 1, 2, 3], [0, 1, 2, 3]), + block_hashes=hs, + load_spec=LoadSpec( + vllm_cached_tokens=0, kvpool_cached_tokens=64, can_load=True, token_len=64 + ), + ) + recv.request_queue.put(req) + recv._handle_request(recv.request_queue.get()) + + full_keys = [k for k in requested_keys if "@group:0" in k] + swa_keys = [k for k in requested_keys if "@group:1" in k] + # Full attention: load every block (4). + assert len(full_keys) == 4 + # SWA: only the 2 tail-window blocks (hashes hs[2], hs[3]). + assert len(swa_keys) == 2 + swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys} + assert swa_hashes == {hs[2].hex(), hs[3].hex()} + + +def test_chunked_token_database_hash_block_size_smaller_than_block_size(): + """DSv4-style: hash_block_size=4, group block_size=16 — process_tokens + must merge every 4 fine hashes into one chunk hash via + BlockHashListWithBlockSize.""" + md = KeyMetadata("m", 0, 0, 0, 0, group_id=3) + db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4) + db.set_kv_caches_base_addr([0]) + db.set_block_len([512]) + # 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks. + fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)] + out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes)) + assert len(out) == 2 + assert out[0][0] == 0 and out[0][1] == 16 + assert out[1][0] == 16 and out[1][1] == 32 + # Each chunk's hash is the concatenation of 4 fine hashes. + expected0 = b"".join(fine_hashes[0:4]).hex() + expected1 = b"".join(fine_hashes[4:8]).hex() + assert out[0][2].chunk_hash == expected0 + assert out[1][2].chunk_hash == expected1 diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py new file mode 100644 index 0000000000..7e8a6db27d --- /dev/null +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + RequestTracker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( + MooncakeStoreScheduler, +) + + +def _make_bare_scheduler() -> MooncakeStoreScheduler: + scheduler = object.__new__(MooncakeStoreScheduler) + scheduler.kv_role = "kv_both" + scheduler.original_block_size = 16 + scheduler._block_size = 16 + scheduler._discard_partial_chunks = True + scheduler.load_specs = {} + scheduler._preempted_req_ids = set() + scheduler._unfinished_request_ids = {"req-0"} + scheduler._unfinished_requests = {} + scheduler._request_trackers = {} + return scheduler + + +def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None): + return SimpleNamespace( + finished_req_ids=set(), + preempted_req_ids=set(), + scheduled_new_reqs=[], + scheduled_cached_reqs=SimpleNamespace( + req_ids=["req-0"], + new_block_ids=[([2],)], + num_computed_tokens=[44], + ), + num_scheduled_tokens={"req-0": 4}, + scheduled_spec_decode_tokens=( + {"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {} + ), + ) + + +def _add_unfinished_request( + scheduler: MooncakeStoreScheduler, + *, + token_ids: list[int], + block_hashes: list[bytes], + prefill_end_tokens: int, +) -> None: + request = SimpleNamespace( + all_token_ids=token_ids, + block_hashes=block_hashes, + num_output_placeholders=0, + ) + scheduler._unfinished_requests["req-0"] = (request, ([0, 1],)) + scheduler._request_trackers["req-0"] = RequestTracker( + req_id="req-0", + token_len=44, + allocated_block_ids=([0, 1],), + num_saved_tokens=32, + token_ids=token_ids[:44], + prefill_end_tokens=prefill_end_tokens, + ) + + +def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts(): + # Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids + # yet, so the tracker's token_len does not advance and num_tokens_to_save + # stays below chunk_boundary — the save is naturally skipped. + scheduler = _make_bare_scheduler() + _add_unfinished_request( + scheduler, + token_ids=list(range(44)), + block_hashes=[b"h0", b"h1"], + prefill_end_tokens=48, + ) + + meta = scheduler.build_connector_meta( + _make_scheduler_output(scheduled_spec_tokens=[101, 102, 103]) + ) + + assert meta.requests == [] + tracker = scheduler._request_trackers["req-0"] + assert tracker.token_len == 44 + assert tracker.num_saved_tokens == 32 + assert tracker.allocated_block_ids == ([0, 1, 2],) + + +def test_cached_request_without_spec_decode_keeps_current_step_save_overlap(): + scheduler = _make_bare_scheduler() + _add_unfinished_request( + scheduler, + token_ids=list(range(48)), + block_hashes=[b"h0", b"h1", b"h2"], + prefill_end_tokens=48, + ) + + meta = scheduler.build_connector_meta( + _make_scheduler_output(scheduled_spec_tokens=None) + ) + + assert len(meta.requests) == 1 + req_meta = meta.requests[0] + assert req_meta.req_id == "req-0" + assert req_meta.can_save is True + assert req_meta.token_len_chunk == 48 + tracker = scheduler._request_trackers["req-0"] + assert tracker.token_len == 48 + assert tracker.num_saved_tokens == 48 diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index c8b9437616..201c0af26f 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -19,7 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( worker, ) -from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( + worker as mooncake_store_worker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( ChunkedTokenDatabase, KeyMetadata, LoadSpec, @@ -27,20 +30,37 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( ) +def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator: + from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec + + spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + return mooncake_store_worker.MooncakeStoreCoordinator( + [KVCacheGroupSpec(["layer0"], spec)], + scheduler_block_size=16, + hash_block_size=16, + ) + + def _make_store_sending_thread( store: MagicMock, *, + coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None, + token_databases: list[ChunkedTokenDatabase] | None = None, + block_size: int = 16, replicate_config: object | None = None, -) -> worker.KVCacheStoreSendingThread: - token_database = ChunkedTokenDatabase( - KeyMetadata("test-model", 0, 0, 0, 0), block_size=16 - ) - token_database.set_kv_caches_base_addr([0x1000]) - token_database.set_block_len([256]) - thread = worker.KVCacheStoreSendingThread( +) -> mooncake_store_worker.KVCacheStoreSendingThread: + if coord is None: + coord = _default_send_coord() + if token_databases is None: + db = ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16) + db.set_kv_caches_base_addr([0x1000]) + db.set_block_len([256]) + token_databases = [db] + thread = mooncake_store_worker.KVCacheStoreSendingThread( store=store, - token_database=token_database, - block_size=16, + token_databases=token_databases, + block_size=block_size, + coord=coord, tp_rank=0, put_step=1, kv_role="kv_producer", @@ -55,18 +75,27 @@ def _make_store_recving_thread( store: MagicMock, *, disk_offload_buffer_budget_bytes: int | None = None, -) -> worker.KVCacheStoreRecvingThread: +) -> mooncake_store_worker.KVCacheStoreRecvingThread: + from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec + token_database = ChunkedTokenDatabase( KeyMetadata("test-model", 0, 0, 0, 0), block_size=16 ) token_database.set_kv_caches_base_addr([0x1000]) token_database.set_block_len([256]) - thread = worker.KVCacheStoreRecvingThread( + spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None) + coord = mooncake_store_worker.MooncakeStoreCoordinator( + [KVCacheGroupSpec(["layer0"], spec)], + scheduler_block_size=16, + hash_block_size=16, + ) + thread = mooncake_store_worker.KVCacheStoreRecvingThread( store=store, - token_database=token_database, + token_databases=[token_database], block_size=16, tp_rank=0, ready_event=threading.Event(), + coord=coord, disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes, ) thread.request_queue.task_done = MagicMock() @@ -83,7 +112,7 @@ def _make_load_req( return ReqMeta( req_id=req_id, token_len_chunk=token_len, - block_ids=list(range(len(block_hashes))), + block_ids=(list(range(len(block_hashes))),), block_hashes=block_hashes, load_spec=LoadSpec( vllm_cached_tokens=vllm_cached_tokens, @@ -98,7 +127,7 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta: return ReqMeta( req_id=req_id, token_len_chunk=32, - block_ids=[0, 1], + block_ids=([0, 1],), block_hashes=block_hashes, can_save=True, original_block_size=16, @@ -149,10 +178,31 @@ def _make_vllm_config( parallel_config=SimpleNamespace( pipeline_parallel_size=1, rank=0, + decode_context_parallel_size=1, + prefill_context_parallel_size=1, ), kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config), cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10), kv_events_config=SimpleNamespace(enable_kv_cache_events=False), + speculative_config=None, + ) + + +def _make_kv_cache_config(*, block_size: int = 16) -> object: + """Minimal single-group KVCacheConfig for topology tests.""" + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + ) + + spec = FullAttentionSpec( + block_size=block_size, num_kv_heads=8, head_size=64, dtype=None + ) + return KVCacheConfig( + num_blocks=10, + kv_cache_tensors=[], + kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)], ) @@ -445,9 +495,9 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch): assert store.batch_get_into_multi_buffers.call_count == 1 keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args assert keys == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", ] assert sizes == [[256], [256], [256]] store.batch_get_replica_desc.assert_not_called() @@ -467,9 +517,9 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm): token_len=48, ) expected_keys = [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", ] store.batch_get_replica_desc.return_value = { expected_keys[0]: [_ReplicaDesc("memory")], @@ -521,10 +571,10 @@ def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split(): first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0] second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0] assert first_keys == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", ] assert second_keys == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", ] @@ -556,14 +606,14 @@ def test_recv_thread_splits_disk_offload_loads_by_budget(): first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2] second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2] assert first_keys == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", ] assert second_keys == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", ] - base_addr = thread.token_database.kv_caches_base_addr[0] - block_len = thread.token_database.block_len[0] + base_addr = thread.token_databases[0].kv_caches_base_addr[0] + block_len = thread.token_databases[0].block_len[0] assert first_addrs == [[base_addr], [base_addr + block_len]] assert second_addrs == [[base_addr + 2 * block_len]] expected_size = block_len @@ -610,9 +660,9 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys(): assert store.batch_get_into_multi_buffers.call_count == 1 assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [ - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131", - "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", ] @@ -654,7 +704,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch): }, ), ) - w = worker.MooncakeStoreWorker(_make_vllm_config()) + w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config()) assert not hasattr(w, "_isolate_offload_resources") assert store.setup.call_args.args == ( @@ -690,7 +740,7 @@ def test_requester_worker_init_prefers_local_hostname_override( }, ), ) - worker.MooncakeStoreWorker(_make_vllm_config()) + worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config()) assert store.setup.call_args.args[0] == "worker-a:50053" @@ -718,7 +768,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled( }, ), ) - w = worker.MooncakeStoreWorker(_make_vllm_config()) + w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config()) assert w.disk_offload_buffer_budget_bytes is None @@ -748,7 +798,8 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment( extra_config={ "preferred_segment": "10.0.0.7:50053", } - ) + ), + _make_kv_cache_config(), ) assert isinstance(w.store_replicate_config, fake_replicate_config_cls) @@ -760,6 +811,158 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment( # --------------------------------------------------------------------------- +def test_store_sending_thread_clamps_token_len_to_lcm(): + """Partial chunks past the last lcm boundary aren't stored — cache hits + are always lcm-aligned (mirrors HybridKVCacheCoordinator).""" + store = MagicMock() + store.batch_is_exist.return_value = [0, 0] + store.batch_put_from_multi_buffers.return_value = [256, 256] + # Default coord: single full-attn block_size=16, lcm=16. + # token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk). + thread = _make_store_sending_thread(store) + + thread.add_stored_request("r0") + thread._handle_request( + ReqMeta( + req_id="r0", + token_len_chunk=33, + block_ids=([0, 1, 2],), + block_hashes=[b"a0", b"a1", b"a2"], + can_save=True, + original_block_size=16, + ) + ) + + keys = store.batch_put_from_multi_buffers.call_args.args[0] + assert len(keys) == 2 + + +def test_store_sending_thread_skips_when_token_len_below_lcm(): + """Requests shorter than lcm_block_size cannot produce any aligned chunk, + so neither the existence check nor the put should be issued.""" + from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec + + store = MagicMock() + # lcm=64 via single full-attn block_size=64. + spec = FullAttentionSpec(block_size=64, num_kv_heads=8, head_size=64, dtype=None) + coord = mooncake_store_worker.MooncakeStoreCoordinator( + [KVCacheGroupSpec(["L"], spec)], + scheduler_block_size=64, + hash_block_size=64, + ) + db = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=64, + hash_block_size=64, + ) + db.set_kv_caches_base_addr([0x1000]) + db.set_block_len([1024]) + thread = _make_store_sending_thread( + store, coord=coord, token_databases=[db], block_size=64 + ) + + thread.add_stored_request("r0") + thread._handle_request( + ReqMeta( + req_id="r0", + token_len_chunk=32, + block_ids=([0, 1],), + block_hashes=[b"a0", b"a1"], + can_save=True, + original_block_size=64, + ) + ) + + store.batch_is_exist.assert_not_called() + store.batch_put_from_multi_buffers.assert_not_called() + assert thread.stored_requests["r0"] == 0 + + +def test_store_sending_thread_only_stores_swa_blocks_in_window(): + """For SWA groups, only blocks within ``sliding_window`` of an + lcm-aligned boundary are stored — a block outside every such window + can never participate in any future hit (mirrors recv-side masking). + + Setup: full-attn (block_size=32) + SWA (block_size=8, sliding_window=8) + → lcm=32. With token_len=64 there are two LCM boundaries (32, 64); + the SWA group should store exactly one block per boundary: the block + ending at 32 and the block ending at 64. + """ + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + SlidingWindowSpec, + ) + + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: ( + [256] * len(keys) + ) + + full_spec = FullAttentionSpec( + block_size=32, num_kv_heads=8, head_size=64, dtype=None + ) + swa_spec = SlidingWindowSpec( + block_size=8, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=8, + ) + coord = mooncake_store_worker.MooncakeStoreCoordinator( + [KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)], + scheduler_block_size=32, + hash_block_size=8, + ) + + db_full = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=32, + hash_block_size=8, + ) + db_full.set_kv_caches_base_addr([0x1000]) + db_full.set_block_len([512]) + db_swa = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), + block_size=8, + hash_block_size=8, + ) + db_swa.set_kv_caches_base_addr([0x2000]) + db_swa.set_block_len([128]) + + thread = _make_store_sending_thread( + store, + coord=coord, + token_databases=[db_full, db_swa], + block_size=32, + ) + + hs = [bytes([i + 1]) * 4 for i in range(8)] + thread.add_stored_request("r0") + thread._handle_request( + ReqMeta( + req_id="r0", + token_len_chunk=64, + block_ids=([0, 1], list(range(8))), + block_hashes=hs, + can_save=True, + original_block_size=32, + ) + ) + + keys = store.batch_put_from_multi_buffers.call_args.args[0] + full_keys = [k for k in keys if "@group:0" in k] + swa_keys = [k for k in keys if "@group:1" in k] + # Full-attn: 2 blocks (chunks ending at 32 and 64). + assert len(full_keys) == 2 + # SWA: only the two blocks ending at lcm boundaries (32 and 64), i.e. + # blocks covering tokens [24,32) and [56,64) — hashes hs[3] and hs[7]. + assert len(swa_keys) == 2 + swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys} + assert swa_hashes == {hs[3].hex(), hs[7].hex()} + + def _auto_set_ready_event(*args, **kwargs): """Side effect for mocked thread constructors that auto-sets ready_event.""" for arg in args: @@ -771,37 +974,108 @@ def _auto_set_ready_event(*args, **kwargs): return MagicMock() +def _register_with_mocked_threads( + worker: mooncake_store_worker.MooncakeStoreWorker, + kv_caches: dict[str, torch.Tensor], +) -> None: + """Call register_kv_caches with the I/O transfer threads mocked out.""" + prefix = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker." + with ( + patch(prefix + "KVCacheStoreSendingThread", side_effect=_auto_set_ready_event), + patch(prefix + "KVCacheStoreRecvingThread", side_effect=_auto_set_ready_event), + ): + worker.register_kv_caches(kv_caches) + + def _make_bare_worker( *, num_gpu_blocks: int = 10, block_size: int = 16, kv_role: str = "kv_both", -) -> worker.MooncakeStoreWorker: +) -> mooncake_store_worker.MooncakeStoreWorker: """Construct a MooncakeStoreWorker via __new__, bypassing __init__. Sets only the attributes that register_kv_caches() reads so we can test the stride-based layout detection without a real MooncakeDistributedStore. """ - w = object.__new__(worker.MooncakeStoreWorker) - w.cache_config = MagicMock() - w.cache_config.num_gpu_blocks = num_gpu_blocks - w.store = MagicMock() - w.store.register_buffer.return_value = 0 - w.use_mla = False - w.token_database = ChunkedTokenDatabase( - KeyMetadata("test-model", 0, 0, 0, 0), block_size=block_size + worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker) + worker.cache_config = MagicMock() + worker.cache_config.num_gpu_blocks = num_gpu_blocks + worker.store = MagicMock() + worker.store.register_buffer.return_value = 0 + worker.use_mla = False + worker.kv_role = kv_role + worker.block_size = block_size + worker.tp_rank = 0 + worker.put_step = 1 + worker.enable_kv_events = False + worker.kv_send_thread = None + worker.kv_recv_thread = None + worker.tp_size = 1 + worker.num_kv_head = 1 + worker.pp_size = 1 + # Minimal single-full-attention-group config so the coordinator-based + # lookup path works (the connector no longer carries a legacy single-group + # path; everything flows through the coordinator). + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, ) - w.kv_role = kv_role - w.block_size = block_size - w.tp_rank = 0 - w.put_step = 1 - w.enable_kv_events = False - w.disk_offload_buffer_budget_bytes = None - w.kv_send_thread = None - w.kv_recv_thread = None - w.store_replicate_config = SimpleNamespace() - return w + + worker.disk_offload_buffer_budget_bytes = None + worker.store_replicate_config = SimpleNamespace() + + spec = FullAttentionSpec( + block_size=block_size, num_kv_heads=8, head_size=64, dtype=None + ) + group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec) + worker._kv_cache_groups = [group] + worker.pcp_size = 1 + worker.dcp_size = 1 + worker.hash_block_size = block_size + worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0) + # Pre-build a single-group token_dbs so lookup-only tests don't have to + # call register_kv_caches. + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=block_size, + hash_block_size=block_size, + ) + ] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=block_size, + hash_block_size=block_size, + ) + return worker + + +def test_lookup_partial_prefix_returns_first_hit_length(): + worker = _make_bare_worker() + worker.store.batch_is_exist.return_value = [1, 1, 0] + assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32 + + +def test_lookup_swa_single_group_returns_full_when_tail_window_present(): + """Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the + tail. Coordinator-driven lookup returns full prefix even though the + pre-window blocks are absent.""" + from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec + + worker = _make_bare_worker(block_size=16) + swa = SlidingWindowSpec( + block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32 + ) + worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=worker.hash_block_size, + hash_block_size=worker.hash_block_size, + ) + worker.store.batch_is_exist.return_value = [0, 0, 1, 1] + assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64 # --------------------------------------------------------------------------- @@ -812,34 +1086,17 @@ def _make_bare_worker( def test_register_kv_caches_blocks_first_single_segment(): """Blocks-first layout (FlashInfer/MLA): one segment per layer.""" num_blocks = 10 - page_size_elements = 64 # elements per block - w = _make_bare_worker(num_gpu_blocks=num_blocks) + page_size_elements = 64 + worker = _make_bare_worker(num_gpu_blocks=num_blocks) # Shape: (num_blocks, page_size_elements) — blocks outermost, no outer_dims tensor = torch.zeros(num_blocks, page_size_elements, dtype=torch.float16) + _register_with_mocked_threads(worker, {"layer0": tensor}) - with ( - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "worker.KVCacheStoreSendingThread", - side_effect=_auto_set_ready_event, - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "worker.KVCacheStoreRecvingThread", - side_effect=_auto_set_ready_event, - ), - ): - w.register_kv_caches({"layer0": tensor}) - - assert len(w.kv_caches_base_addr) == 1 - assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr() - - expected_block_len = tensor.untyped_storage().nbytes() // num_blocks - assert len(w.block_len) == 1 - assert w.block_len[0] == expected_block_len - - w.store.register_buffer.assert_called_once_with( + db = worker.token_dbs[0] + assert db.kv_caches_base_addr == [tensor.untyped_storage().data_ptr()] + assert db.block_len == [tensor.untyped_storage().nbytes() // num_blocks] + worker.store.register_buffer.assert_called_once_with( tensor.untyped_storage().data_ptr(), tensor.untyped_storage().nbytes(), ) @@ -851,8 +1108,7 @@ def test_register_kv_caches_kv_first_two_segments(): block_size_tokens = 16 num_kv_heads = 4 head_size = 8 - - w = _make_bare_worker(num_gpu_blocks=num_blocks) + worker = _make_bare_worker(num_gpu_blocks=num_blocks) # Shape: (2, num_blocks, block_size, num_kv_heads, head_size) — K/V outermost tensor = torch.zeros( @@ -863,32 +1119,13 @@ def test_register_kv_caches_kv_first_two_segments(): head_size, dtype=torch.float16, ) + _register_with_mocked_threads(worker, {"layer0": tensor}) - with ( - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "worker.KVCacheStoreSendingThread", - side_effect=_auto_set_ready_event, - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." - "worker.KVCacheStoreRecvingThread", - side_effect=_auto_set_ready_event, - ), - ): - w.register_kv_caches({"layer0": tensor}) - - # K/V-first: dim 0 has stride > page_size, so 2 segments - assert len(w.kv_caches_base_addr) == 2 - assert len(w.block_len) == 2 - - el = tensor.element_size() - seg_stride = tensor.stride(0) * el # stride of the K/V dim in bytes + db = worker.token_dbs[0] + seg_stride = tensor.stride(0) * tensor.element_size() base = tensor.untyped_storage().data_ptr() - assert w.kv_caches_base_addr[0] == base - assert w.kv_caches_base_addr[1] == base + seg_stride - assert w.block_len[0] == seg_stride // num_blocks - assert w.block_len[1] == seg_stride // num_blocks + assert db.kv_caches_base_addr == [base, base + seg_stride] + assert db.block_len == [seg_stride // num_blocks] * 2 def test_register_kv_caches_cross_layer_single_segment(): @@ -897,7 +1134,7 @@ def test_register_kv_caches_cross_layer_single_segment(): num_layers = 4 per_layer_page_elements = 64 # elements per layer per block - w = _make_bare_worker(num_gpu_blocks=num_blocks) + worker = _make_bare_worker(num_gpu_blocks=num_blocks) # Cross-layer blocks-first tensor: all layers packed into a single # contiguous block. Shape (num_blocks, num_layers * per_layer_page) @@ -918,10 +1155,11 @@ def test_register_kv_caches_cross_layer_single_segment(): ), ): # Use the cross-layer wrapper key, same as register_cross_layers_kv_caches - w.register_kv_caches({"__cross_layer__": tensor}) + worker.register_kv_caches({"__cross_layer__": tensor}) - assert len(w.kv_caches_base_addr) == 1 - assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr() + db = worker.token_dbs[0] + assert len(db.kv_caches_base_addr) == 1 + assert db.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr() expected_block_len = tensor.untyped_storage().nbytes() // num_blocks # block_len should be per_layer_page_size * num_layers @@ -929,11 +1167,11 @@ def test_register_kv_caches_cross_layer_single_segment(): expected_block_len == num_layers * per_layer_page_elements * tensor.element_size() ) - assert len(w.block_len) == 1 - assert w.block_len[0] == expected_block_len + assert len(db.block_len) == 1 + assert db.block_len[0] == expected_block_len # Also verify via register_cross_layers_kv_caches wrapper - w2 = _make_bare_worker(num_gpu_blocks=num_blocks) + worker2 = _make_bare_worker(num_gpu_blocks=num_blocks) with ( patch( "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." @@ -946,10 +1184,11 @@ def test_register_kv_caches_cross_layer_single_segment(): side_effect=_auto_set_ready_event, ), ): - w2.register_cross_layers_kv_caches(tensor) + worker2.register_cross_layers_kv_caches(tensor) - assert w2.kv_caches_base_addr == w.kv_caches_base_addr - assert w2.block_len == w.block_len + db2 = worker2.token_dbs[0] + assert db2.kv_caches_base_addr == db.kv_caches_base_addr + assert db2.block_len == db.block_len # --------------------------------------------------------------------------- @@ -1070,7 +1309,8 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch): ) w = worker.MooncakeStoreWorker( - _make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}) + _make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}), + _make_kv_cache_config(), ) # setup() receives global_segment_size=0 and the configured local buffer. @@ -1114,7 +1354,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch): ), ) - w = worker.MooncakeStoreWorker(_make_vllm_config()) + w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config()) # setup() receives global_segment_size=4 GiB (rank contributes a segment). assert store.setup.call_args.args == ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 184501a96a..5d6136c587 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, + SupportsHMA, ) from vllm.forward_context import ForwardContext from vllm.logger import init_logger @@ -76,7 +77,7 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents): return f"" -class MooncakeStoreConnector(KVConnectorBase_V1): +class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): """KV connector using MooncakeDistributedStore as shared KV pool.""" @property @@ -87,6 +88,36 @@ class MooncakeStoreConnector(KVConnectorBase_V1): == "true" ) + @staticmethod + def _validate_kv_cache_config( + vllm_config: VllmConfig, kv_cache_config: KVCacheConfig + ) -> None: + from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec + + unsupported: list[str] = [] + cache_block_size = vllm_config.cache_config.block_size + for g_idx, g in enumerate(kv_cache_config.kv_cache_groups): + spec = g.kv_cache_spec + if isinstance(spec, CrossAttentionSpec): + unsupported.append(f"group {g_idx}: CrossAttentionSpec") + # Enforce Mamba align mode + if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size: + unsupported.append( + f"group {g_idx}: MambaSpec with block_size=" + f"{spec.block_size} != cache_config.block_size=" + f"{cache_block_size} (mamba_cache_mode != 'align')" + ) + pcp = vllm_config.parallel_config.prefill_context_parallel_size + dcp = vllm_config.parallel_config.decode_context_parallel_size + if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1: + unsupported.append( + f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention" + ) + if unsupported: + raise ValueError( + "MooncakeStoreConnector does not support: " + "; ".join(unsupported) + ) + def __init__( self, vllm_config: VllmConfig, @@ -99,6 +130,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1): kv_cache_config=kv_cache_config, # type: ignore[arg-type] ) assert vllm_config.kv_transfer_config is not None + assert kv_cache_config is not None, "kv_cache_config is required" + self._validate_kv_cache_config(vllm_config, kv_cache_config) + self._kv_cache_config = kv_cache_config self.kv_role = vllm_config.kv_transfer_config.kv_role self._kv_cache_events: MooncakeStoreKVEvents | None = None @@ -106,9 +140,11 @@ class MooncakeStoreConnector(KVConnectorBase_V1): self.connector_worker: MooncakeStoreWorker | None = None if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler = MooncakeStoreScheduler(vllm_config) + self.connector_scheduler = MooncakeStoreScheduler( + vllm_config, kv_cache_config + ) else: - self.connector_worker = MooncakeStoreWorker(vllm_config) + self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config) # ============================================================ # Scheduler-side methods @@ -146,6 +182,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1): self, request: Request, block_ids: list[int], + ) -> tuple[bool, dict[str, Any] | None]: + return self.request_finished_all_groups(request, (block_ids,)) + + def request_finished_all_groups( + self, + request: Request, + block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: assert self.connector_scheduler is not None return self.connector_scheduler.request_finished(request, block_ids) @@ -184,6 +227,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1): self, kv_cache: torch.Tensor, attn_backend: type ): assert self.connector_worker is not None + assert ( + self._kv_cache_config is not None + and len(self._kv_cache_config.kv_cache_groups) == 1 + ), "Cross-layer KV cache does not supported with hybrid models" self.connector_worker.register_cross_layers_kv_caches(kv_cache) def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py new file mode 100644 index 0000000000..26f3aa2632 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""External-store cache-hit coordinator for MooncakeStoreConnector.""" + +from typing import cast + +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.kv_cache_utils import ( + BlockHash, + BlockHashList, + BlockHashListWithBlockSize, + KVCacheBlock, +) +from vllm.v1.core.single_type_kv_cache_manager import ( + SingleTypeKVCacheManager, + spec_manager_map, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + KVCacheSpec, + UniformTypeKVCacheSpecs, +) + +# Dummy placeholder hash for store_mask's template computation. +_DUMMY_BLOCK_HASH = BlockHash(b"\x00" * 32) + + +class ExternalCachedBlockPool: + """Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set.""" + + def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None: + # ``exists=None`` is used on the recv side where hit_length is already + # determined and we just want each spec's manager to apply its own mask. + self._exists = exists + self.null_block = KVCacheBlock(block_id=0) + # Dummy ID 1 for present block for duck-typing. + self._present_block = KVCacheBlock(block_id=1) + + def get_cached_block( + self, + block_hash: BlockHash, + group_ids: list[int], + ) -> list[KVCacheBlock] | None: + # Mirrors BlockPool.get_cached_block: hit only when every group_id + # (groups sharing a spec) has the hash cached. + if self._exists is None: + return [self._present_block] * len(group_ids) + h = bytes(block_hash) + if all((g, h) in self._exists for g in group_ids): + return [self._present_block] * len(group_ids) + return None + + +class MooncakeStoreCoordinator: + """Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an + ``ExternalCachedBlockPool``.""" + + def __init__( + self, + kv_cache_groups: list[KVCacheGroupSpec], + scheduler_block_size: int, + hash_block_size: int, + use_eagle: bool = False, + ) -> None: + assert all( + g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups + ), "block_size must be divisible by hash_block_size" + assert scheduler_block_size % hash_block_size == 0, ( + f"scheduler_block_size ({scheduler_block_size}) must be a multiple of " + f"hash_block_size ({hash_block_size})" + ) + assert all( + scheduler_block_size % g.kv_cache_spec.block_size == 0 + for g in kv_cache_groups + ), "scheduler_block_size must be a multiple of each group's block_size" + self.kv_cache_groups = kv_cache_groups + self.hash_block_size = hash_block_size + self.lcm_block_size = scheduler_block_size + self.use_eagle = use_eagle + self._verify_and_split_kv_cache_groups() + + def _verify_and_split_kv_cache_groups(self) -> None: + """Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but + dispatches via spec_manager_map (we don't allocate managers). + """ + attention_groups: list[ + tuple[KVCacheSpec, list[int], type[SingleTypeKVCacheManager]] + ] = [] + for i, g in enumerate(self.kv_cache_groups): + spec = _unwrap_spec(g.kv_cache_spec) + manager_cls = spec_manager_map[type(spec)] + for existing_spec, group_ids, existing_cls in attention_groups: + if existing_spec == spec: + assert manager_cls is existing_cls + group_ids.append(i) + break + else: + attention_groups.append((spec, [i], manager_cls)) + # Full attention first (matches upstream convergence ordering). + self.attention_groups = sorted( + attention_groups, + key=lambda x: not isinstance(x[0], FullAttentionSpec), + ) + self.eagle_attn_group_indices: set[int] = { + i + for i, (_, group_ids, _) in enumerate(self.attention_groups) + if any(self.kv_cache_groups[gid].is_eagle_group for gid in group_ids) + } + if self.use_eagle and not self.eagle_attn_group_indices: + self.eagle_attn_group_indices = set(range(len(self.attention_groups))) + + def find_longest_cache_hit( + self, + block_hashes: list[BlockHash], + max_length: int, + cached_block_pool: ExternalCachedBlockPool, + ) -> tuple[tuple[list[bool], ...], int]: + """Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff + group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only); + recv-side callers skip False slots.""" + blocks_per_group, hit_length = self._find_hit_blocks( + block_hashes, max_length, cached_block_pool + ) + masks = tuple( + [blk is not cached_block_pool.null_block for blk in blocks] + for blocks in blocks_per_group + ) + return masks, hit_length + + def load_mask( + self, + block_hashes: list[BlockHash], + token_len: int, + ) -> tuple[list[bool], ...]: + """Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s + spec would populate chunk ``i`` locally at length ``token_len`` + (e.g. SWA / Mamba tail-only). + """ + masks, _ = self.find_longest_cache_hit( + block_hashes, token_len, ExternalCachedBlockPool() + ) + return masks + + def store_mask(self, aligned_token_len: int) -> tuple[list[bool], ...]: + """Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of + group ``g`` would be populated by some future cache hit at length + ``L = N * lcm_block_size <= aligned_token_len``. + """ + assert aligned_token_len % self.lcm_block_size == 0, ( + f"aligned_token_len ({aligned_token_len}) must be a multiple of " + f"lcm_block_size ({self.lcm_block_size})" + ) + if aligned_token_len == 0: + return tuple([] for _ in self.kv_cache_groups) + + num_chunks_per_group = [ + aligned_token_len // g.kv_cache_spec.block_size + for g in self.kv_cache_groups + ] + + # Fast path: single group or full attn groups or uniform block_sizes + if all( + isinstance(spec, FullAttentionSpec) + or spec.block_size == self.lcm_block_size + for spec, _, _ in self.attention_groups + ): + return tuple([True] * n for n in num_chunks_per_group) + + n_segments = aligned_token_len // self.lcm_block_size + dummy_hashes: list[BlockHash] = [_DUMMY_BLOCK_HASH] * ( + self.lcm_block_size // self.hash_block_size + ) + template_masks, _ = self.find_longest_cache_hit( + dummy_hashes, + max_length=self.lcm_block_size, + cached_block_pool=ExternalCachedBlockPool(), + ) + return tuple( + list(template_masks[g]) * n_segments + for g in range(len(self.kv_cache_groups)) + ) + + def block_hashes_for_spec( + self, block_hashes: list[BlockHash], spec: KVCacheSpec + ) -> BlockHashList: + if spec.block_size == self.hash_block_size: + return block_hashes + return BlockHashListWithBlockSize( + block_hashes, self.hash_block_size, spec.block_size + ) + + def _find_hit_blocks( + self, + block_hashes: list[BlockHash], + max_length: int, + cached_block_pool: ExternalCachedBlockPool, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + """Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but + dispatches via spec_manager_map (we don't allocate managers). + """ + if len(self.attention_groups) == 1: + spec, group_ids, manager_cls = self.attention_groups[0] + hashes = self.block_hashes_for_spec(block_hashes, spec) + hit_blocks = manager_cls.find_longest_cache_hit( + block_hashes=hashes, + max_length=max_length, + kv_cache_group_ids=group_ids, + block_pool=cast(BlockPool, cached_block_pool), + kv_cache_spec=spec, + use_eagle=(0 in self.eagle_attn_group_indices), + alignment_tokens=spec.block_size, + ) + num_groups = len(self.kv_cache_groups) + blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)] + for gid, blks in zip(group_ids, hit_blocks, strict=True): + blocks_by_group[gid] = blks + return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size + + num_groups = len(self.kv_cache_groups) + hit_length = max_length + hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups + + is_simple_hybrid = len(self.attention_groups) == 2 and isinstance( + self.attention_groups[0][0], FullAttentionSpec + ) + eagle_verified: set[int] = set() + + while True: + curr_hit_length = hit_length + + for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups): + cached = hit_blocks_by_group[group_ids[0]] + if isinstance(spec, FullAttentionSpec) and cached is not None: + curr_hit_length = ( + curr_hit_length // spec.block_size * spec.block_size + ) + continue + + use_eagle = ( + idx in self.eagle_attn_group_indices and idx not in eagle_verified + ) + _max_length = curr_hit_length + if use_eagle: + _max_length = min(curr_hit_length + spec.block_size, max_length) + hashes = self.block_hashes_for_spec(block_hashes, spec) + hit_blocks = manager_cls.find_longest_cache_hit( + block_hashes=hashes, + max_length=_max_length, + kv_cache_group_ids=group_ids, + block_pool=cast(BlockPool, cached_block_pool), + kv_cache_spec=spec, + use_eagle=use_eagle, + alignment_tokens=self.lcm_block_size, + ) + _new_hit_length = len(hit_blocks[0]) * spec.block_size + if use_eagle: + eagle_verified.add(idx) + elif _new_hit_length < curr_hit_length: + eagle_verified.clear() + curr_hit_length = _new_hit_length + for gid, blocks in zip(group_ids, hit_blocks, strict=True): + hit_blocks_by_group[gid] = blocks + + if curr_hit_length >= hit_length: + break + hit_length = curr_hit_length + if is_simple_hybrid: + break + + # Truncate full-attention hit_blocks to final converged length; + # other specs already trim themselves inside their hit logic. + spec0, group_ids0, _ = self.attention_groups[0] + if isinstance(spec0, FullAttentionSpec): + num_blocks = hit_length // spec0.block_size + for gid in group_ids0: + full_blks = hit_blocks_by_group[gid] + assert full_blks is not None + del full_blks[num_blocks:] + + return ( + tuple(blks if blks is not None else [] for blks in hit_blocks_by_group), + hit_length, + ) + + +def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec: + if isinstance(spec, UniformTypeKVCacheSpecs): + return next(iter(spec.kv_cache_specs.values())) + return spec diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index acdeedceea..ab1df0413d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -15,7 +15,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( ) from vllm.logger import init_logger from vllm.utils.math_utils import cdiv -from vllm.v1.core.kv_cache_utils import BlockHash +from vllm.v1.core.kv_cache_utils import ( + BlockHash, + BlockHashListWithBlockSize, +) logger = init_logger(__name__) @@ -29,6 +32,7 @@ class KeyMetadata: pcp_rank: int dcp_rank: int pp_rank: int + group_id: int = 0 @dataclass(order=True) @@ -46,6 +50,7 @@ class PoolKey: self.key_metadata.pcp_rank, self.key_metadata.dcp_rank, self.key_metadata.pp_rank, + self.key_metadata.group_id, self.chunk_hash, ) ) @@ -57,6 +62,7 @@ class PoolKey: f"@pcp{self.key_metadata.pcp_rank}" f"@dcp{self.key_metadata.dcp_rank}" f"@pp_rank:{self.key_metadata.pp_rank}" + f"@group:{self.key_metadata.group_id}" f"@{self.chunk_hash}" ) @@ -64,9 +70,20 @@ class PoolKey: class ChunkedTokenDatabase: """Maps token positions to store keys and GPU memory addresses.""" - def __init__(self, metadata: KeyMetadata, block_size: int): + def __init__( + self, + metadata: KeyMetadata, + block_size: int, + hash_block_size: int | None = None, + ): self.metadata = metadata self.block_size = block_size + self.hash_block_size = hash_block_size or block_size + if self.block_size % self.hash_block_size != 0: + raise ValueError( + f"block_size ({self.block_size}) must be a multiple of " + f"hash_block_size ({self.hash_block_size})" + ) self.kv_caches_base_addr: list[int] = [] self.block_len: list[int] = [] @@ -77,9 +94,6 @@ class ChunkedTokenDatabase: self.kv_caches_base_addr = kv_caches_base_addr def set_block_len(self, block_len: list[int]): - for length in block_len: - if length % self.block_size != 0: - raise ValueError(f"block_len {length} % {self.block_size} != 0") self.block_len = block_len def prepare_value( @@ -96,7 +110,8 @@ class ChunkedTokenDatabase: length = len(self.block_len) for index, base_addr in enumerate(self.kv_caches_base_addr): addr = base_addr + block_id * self.block_len[index % length] - size = self.block_len[index % length] // self.block_size * (end - start) + assert (end - start) % self.block_size == 0 + size = self.block_len[index % length] * cdiv(end - start, self.block_size) addr_list.append(addr) size_list.append(size) return addr_list, size_list, block_id @@ -104,38 +119,34 @@ class ChunkedTokenDatabase: def process_tokens( self, token_len: int, - block_hashes: list[BlockHash] | list[str], + block_hashes: list[BlockHash], mask_num: int = 0, ) -> Iterable[tuple[int, int, PoolKey]]: """Process tokens and yield (start_idx, end_idx, pool_key) tuples. Args: token_len: Total number of tokens. - block_hashes: Block hashes for each block. + block_hashes: Block hashes computed at ``hash_block_size`` granularity. + When ``block_size > hash_block_size`` consecutive hashes are merged + up to the group's ``block_size`` via ``BlockHashListWithBlockSize``. mask_num: Number of tokens to skip from the beginning. """ if not block_hashes: return - if not isinstance(block_hashes[0], str): - block_hashes = [ - h.hex() # type: ignore[union-attr] - for h in block_hashes - ] - for chunk_id, hash_val in enumerate(block_hashes): + if self.block_size == self.hash_block_size: + chunk_hashes: Iterable[BlockHash] = block_hashes + else: + chunk_hashes = BlockHashListWithBlockSize( + block_hashes, self.hash_block_size, self.block_size + ) + for chunk_id, h in enumerate(chunk_hashes): start_idx = chunk_id * self.block_size if start_idx >= token_len: break end_idx = min(start_idx + self.block_size, token_len) if start_idx < mask_num: continue - else: - yield ( - start_idx, - end_idx, - self._make_key_by_hash( - hash_val # type: ignore[arg-type] - ), - ) + yield start_idx, end_idx, self._make_key_by_hash(h.hex()) @dataclass @@ -154,7 +165,7 @@ class RequestTracker: req_id: str token_len: int - allocated_block_ids: list[int] + allocated_block_ids: tuple[list[int], ...] num_saved_tokens: int = 0 token_ids: list[int] | None = None # Snapshot of the prefill range length at tracker creation time. @@ -166,15 +177,18 @@ class RequestTracker: self, new_block_ids: tuple[list[int], ...] | list[int], ) -> None: - if len(new_block_ids) == 0: - new_block_ids = [] - elif isinstance(new_block_ids, tuple): - new_block_ids = new_block_ids[0] - elif isinstance(new_block_ids, list): - pass - else: - raise ValueError(f"Unsupported new_block_ids type {type(new_block_ids)}") - self.allocated_block_ids.extend(new_block_ids) + # Backward-compat: accept a single list (broadcast to single group). + if isinstance(new_block_ids, list): + new_block_ids = (new_block_ids,) + if len(new_block_ids) != len(self.allocated_block_ids): + raise ValueError( + f"Group count mismatch: tracker has " + f"{len(self.allocated_block_ids)} groups, update has " + f"{len(new_block_ids)}" + ) + for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True): + if new: + existing.extend(new) @dataclass @@ -183,7 +197,7 @@ class ReqMeta: req_id: str token_len_chunk: int - block_ids: list[int] + block_ids: tuple[list[int], ...] block_hashes: list[BlockHash] can_save: bool | None = None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 5ce3278fee..5922965974 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -22,7 +22,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( ) from vllm.logger import init_logger from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput +from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.request import Request logger = init_logger(__name__) @@ -45,7 +47,11 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]: class MooncakeStoreScheduler: """Scheduler-side component for MooncakeStoreConnector.""" - def __init__(self, vllm_config: VllmConfig): + def __init__( + self, + vllm_config: VllmConfig, + kv_cache_config: KVCacheConfig, + ): assert vllm_config.kv_transfer_config is not None self.kv_role = vllm_config.kv_transfer_config.kv_role self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get( @@ -56,11 +62,11 @@ class MooncakeStoreScheduler: self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size self.original_block_size = vllm_config.cache_config.block_size - self._block_size = vllm_config.cache_config.block_size - if self.pcp_size > 1: - self._block_size *= self.pcp_size - if self.dcp_size > 1: - self._block_size *= self.dcp_size + # LCM for multi-group HMA; bs * pcp * dcp for single-group. Matches + # the engine's own scheduler block size by construction. + self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes( + kv_cache_config, vllm_config + ) self._discard_partial_chunks = ( vllm_config.kv_transfer_config.get_from_extra_config( @@ -72,7 +78,7 @@ class MooncakeStoreScheduler: self.load_specs: dict[str, LoadSpec] = {} # to be loaded self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests self._preempted_req_ids: set[str] = set() # preempted requests - self._unfinished_requests: dict[str, tuple[Request, list[int]]] = {} + self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {} self._unfinished_request_ids: set[str] = set() def get_num_new_matched_tokens( @@ -126,9 +132,9 @@ class MooncakeStoreScheduler: num_external_tokens: int, ): """Update state after block allocation.""" - local_block_ids: list[int] = [] + local_block_ids: tuple[list[int], ...] = () if num_external_tokens > 0: - local_block_ids = blocks.get_block_ids()[0] + local_block_ids = blocks.get_block_ids() self._unfinished_requests[request.request_id] = (request, local_block_ids) self._unfinished_request_ids.add(request.request_id) @@ -189,11 +195,12 @@ class MooncakeStoreScheduler: request_tuple = self._unfinished_requests.get(request.req_id) request_real = request_tuple[0] # type: ignore[index] - if not isinstance(request.block_ids[0], list): - unfolded_block_ids = request.block_ids.copy() + if isinstance(request.block_ids, tuple): + # Multi-group: preserve per-group structure. + unfolded_block_ids = tuple(b.copy() for b in request.block_ids) else: - # TODO: support HMA - unfolded_block_ids = request.block_ids[0].copy() + # Single-group legacy: list[int] -> 1-tuple. + unfolded_block_ids = (request.block_ids.copy(),) prefill_tokens = _new_req_prefill_tokens(request) request_tracker = RequestTracker( @@ -237,9 +244,9 @@ class MooncakeStoreScheduler: if req_id in self._preempted_req_ids: # Resumed after preemption if isinstance(new_block_ids, tuple): - block_ids_list = new_block_ids[0].copy() + new_block_ids = tuple(b.copy() for b in new_block_ids) else: - block_ids_list = new_block_ids.copy() + new_block_ids = (new_block_ids.copy(),) self._preempted_req_ids.discard(req_id) load_spec = self.load_specs.pop(req_id, None) request_tuple = self._unfinished_requests.get(req_id) @@ -254,7 +261,7 @@ class MooncakeStoreScheduler: request_tracker = RequestTracker( req_id=req_id, token_len=num_tokens_to_compute, - allocated_block_ids=block_ids_list, + allocated_block_ids=new_block_ids, num_saved_tokens=0, token_ids=prefill_tokens[:num_tokens_to_compute].copy(), prefill_end_tokens=len(prefill_tokens), @@ -361,7 +368,7 @@ class MooncakeStoreScheduler: def request_finished( self, request: Request, - block_ids: list[int], + block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: """Determine whether to delay freeing blocks for async save.""" if self.kv_role == "kv_consumer": @@ -370,11 +377,12 @@ class MooncakeStoreScheduler: assert tracker is not None if tracker.num_saved_tokens <= 0: return False, None - delay_free_blocks = len(block_ids) > 0 + total_blocks = sum(len(g) for g in block_ids) + delay_free_blocks = total_blocks > 0 if delay_free_blocks: logger.debug( "Delaying free of %d blocks for request %s", - len(block_ids), + total_blocks, request.request_id, ) return delay_free_blocks, None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 3e83204297..2183cc730c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -10,6 +10,7 @@ Includes the store worker, transfer threads, lookup server, and MooncakeDistributedStore integration. """ +import dataclasses import json import os import queue @@ -36,15 +37,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( get_mooncake_dp_engine_index, ) -from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 + ExternalCachedBlockPool, + MooncakeStoreCoordinator, +) +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 ChunkedTokenDatabase, KeyMetadata, MooncakeStoreConnectorMetadata, + PoolKey, ReqMeta, ) from vllm.logger import init_logger from vllm.utils.network_utils import get_ip, make_zmq_socket -from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash +from vllm.v1.core.kv_cache_utils import ( + BlockHash, + maybe_convert_block_hash, + resolve_kv_cache_block_sizes, +) +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder logger = init_logger(__name__) @@ -321,7 +332,7 @@ class KVTransferThread(threading.Thread): def __init__( self, store: Any, - token_database: ChunkedTokenDatabase, + token_databases: list[ChunkedTokenDatabase], block_size: int, tp_rank: int, ready_event: threading.Event, @@ -332,7 +343,7 @@ class KVTransferThread(threading.Thread): self.ready_event = ready_event self.block_size = block_size self.tp_rank = tp_rank - self.token_database = token_database + self.token_databases = token_databases self.done_task_lock = threading.Lock() self.request_queue: queue.Queue[Any] = queue.Queue() self.finished_requests: set[str] = set() @@ -385,7 +396,8 @@ class KVCacheStoreSendingThread(KVTransferThread): def __init__( self, store: Any, - token_database: ChunkedTokenDatabase, + coord: MooncakeStoreCoordinator, + token_databases: list[ChunkedTokenDatabase], block_size: int, tp_rank: int, put_step: int, @@ -396,13 +408,14 @@ class KVCacheStoreSendingThread(KVTransferThread): ): super().__init__( store, - token_database, + token_databases, block_size, tp_rank, ready_event, name="KVCacheStoreSendingThread", ) self.put_step = put_step + self.coord = coord self.kv_role = kv_role self.stored_requests: defaultdict[str, int] = defaultdict(int) self.enable_kv_event = enable_kv_event @@ -449,14 +462,20 @@ class KVCacheStoreSendingThread(KVTransferThread): return True def _handle_request(self, req_meta: ReqMeta): - token_len = req_meta.token_len_chunk - block_ids = req_meta.block_ids + # Cache hits are always a multiple of ``lcm_block_size`` tokens + lcm_block_size = self.coord.lcm_block_size + token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size + block_ids_per_group = req_meta.block_ids req_id = req_meta.req_id current_event = req_meta.current_event if req_id not in self.stored_requests: self.request_queue.task_done() return + if token_len == 0: + self.dec_stored_request(req_id) + self.request_queue.task_done() + return if self._should_skip_request(req_id): logger.debug( "Skipping Mooncake store for request %s while CPU/disk offloading " @@ -467,23 +486,34 @@ class KVCacheStoreSendingThread(KVTransferThread): self.request_queue.task_done() return - starts = [] - ends = [] - keys = [] + # Within each lcm region only per-spec relevant chunks are loaded + # (e.g., SWA or linear attn), so mask out irrelevant chunks + store_masks = self.coord.store_mask(token_len) + starts: list[int] = [] + ends: list[int] = [] + keys: list[str] = [] block_hashes: list[BlockHash] = [] - for index, (start, end, key) in enumerate( - self.token_database.process_tokens(token_len, req_meta.block_hashes) - ): - starts.append(start) - ends.append(end) - keys.append(key.to_string()) - block_hashes.append(req_meta.block_hashes[index]) + group_indices: list[int] = [] + for g_idx, db in enumerate(self.token_databases): + mask = store_masks[g_idx] + for chunk_idx, (start, end, key) in enumerate( + db.process_tokens(token_len, req_meta.block_hashes) + ): + if chunk_idx >= len(mask) or not mask[chunk_idx]: + continue + starts.append(start) + ends.append(end) + keys.append(key.to_string()) + block_hashes.append(req_meta.block_hashes[chunk_idx]) + group_indices.append(g_idx) # Apply put_step striding for TP - starts = starts[self.tp_rank % self.put_step :: self.put_step] - ends = ends[self.tp_rank % self.put_step :: self.put_step] - keys = keys[self.tp_rank % self.put_step :: self.put_step] - block_hashes = block_hashes[self.tp_rank % self.put_step :: self.put_step] + sl = slice(self.tp_rank % self.put_step, None, self.put_step) + starts = starts[sl] + ends = ends[sl] + keys = keys[sl] + block_hashes = block_hashes[sl] + group_indices = group_indices[sl] if not keys: self.dec_stored_request(req_id) @@ -501,38 +531,37 @@ class KVCacheStoreSendingThread(KVTransferThread): ends = [ends[i] for i in missing_indices] keys = [keys[i] for i in missing_indices] block_hashes = [block_hashes[i] for i in missing_indices] + group_indices = [group_indices[i] for i in missing_indices] logger.debug( - "Storing KV cache for %d out of %d blocks " - "(missing_count=%d) for request %s", + "Storing KV cache for %d blocks (groups=%s) for request %s", len(keys), - token_len // self.block_size, - len(missing_indices), + set(group_indices), req_id, ) - addrs = [] - sizes = [] + addrs: list[list[int]] = [] + sizes: list[list[int]] = [] stored_events: list[BlockStored] = [] - prev_key = None + # parent_block_hash chains live within a group, not across. + prev_key_per_group: dict[int, Any] = {} new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes] - for index, start in enumerate(starts): - addr, size, _ = self.token_database.prepare_value( - start, ends[index], block_ids - ) + for idx, (s, e, g_idx) in enumerate( + zip(starts, ends, group_indices, strict=True) + ): + db = self.token_databases[g_idx] + addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx]) addrs.append(addr) sizes.append(size) if self.enable_kv_event: token_ids = ( - req_meta.token_ids[start : ends[index]] - if req_meta.token_ids is not None - else None + req_meta.token_ids[s:e] if req_meta.token_ids is not None else None ) stored_event = BlockStored( - block_hashes=[new_block_hashes[index]], - parent_block_hash=prev_key, + block_hashes=[new_block_hashes[idx]], + parent_block_hash=prev_key_per_group.get(g_idx), token_ids=token_ids, block_size=req_meta.original_block_size, lora_id=None, @@ -540,7 +569,7 @@ class KVCacheStoreSendingThread(KVTransferThread): lora_name=None, ) stored_events.append(stored_event) - prev_key = new_block_hashes[index] + prev_key_per_group[g_idx] = new_block_hashes[idx] if current_event is not None: current_event.synchronize() @@ -600,7 +629,8 @@ class KVCacheStoreRecvingThread(KVTransferThread): def __init__( self, store: Any, - token_database: ChunkedTokenDatabase, + coord: MooncakeStoreCoordinator, + token_databases: list[ChunkedTokenDatabase], block_size: int, tp_rank: int, ready_event: threading.Event, @@ -608,7 +638,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): ): super().__init__( store, - token_database, + token_databases, block_size, tp_rank, ready_event, @@ -622,6 +652,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): disk_offload_buffer_budget_bytes ) ) + self.coord = coord def _handle_request(self, req_meta: ReqMeta): token_len = req_meta.load_spec.token_len # type: ignore[union-attr] @@ -632,32 +663,31 @@ class KVCacheStoreRecvingThread(KVTransferThread): * self.block_size ) - addr_list = [] - size_list = [] - key_list = [] - for start, end, key in self.token_database.process_tokens( - token_len, req_meta.block_hashes, mask_num - ): - addr, size, _ = self.token_database.prepare_value( - start, end, req_meta.block_ids - ) - key_list.append(key.to_string()) - addr_list.append(addr) - size_list.append(size) + # Skip chunks the consumer's per-group spec wouldn't populate + # locally (e.g. SWA pre-window) even if the producer stored them. + load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len) + + addr_list: list[list[int]] = [] + size_list: list[list[int]] = [] + key_list: list[str] = [] + for g_idx, db in enumerate(self.token_databases): + mask = load_mask_per_group[g_idx] + for start, end, key in db.process_tokens( + token_len, req_meta.block_hashes, mask_num + ): + chunk_idx = start // db.block_size + if chunk_idx >= len(mask) or not mask[chunk_idx]: + continue + addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx]) + key_list.append(key.to_string()) + addr_list.append(addr) + size_list.append(size) # Rotate lists by tp_rank for load balancing - key_list_c = ( - key_list[self.tp_rank % len(key_list) :] - + key_list[: self.tp_rank % len(key_list)] - ) - addr_list_c = ( - addr_list[self.tp_rank % len(addr_list) :] - + addr_list[: self.tp_rank % len(addr_list)] - ) - size_list_c = ( - size_list[self.tp_rank % len(size_list) :] - + size_list[: self.tp_rank % len(size_list)] - ) + rotation = self.tp_rank % len(key_list) + key_list_c = key_list[rotation:] + key_list[:rotation] + addr_list_c = addr_list[rotation:] + addr_list[:rotation] + size_list_c = size_list[rotation:] + size_list[:rotation] load_batches = [(key_list_c, addr_list_c, size_list_c)] if self.usable_disk_offload_buffer_budget_bytes is not None: @@ -737,7 +767,11 @@ class KVCacheStoreRecvingThread(KVTransferThread): class MooncakeStoreWorker: """Worker-side component for MooncakeStoreConnector.""" - def __init__(self, vllm_config: VllmConfig): + def __init__( + self, + vllm_config: VllmConfig, + kv_cache_config: KVCacheConfig, + ): try: from mooncake.store import ( # type: ignore MooncakeDistributedStore, @@ -771,11 +805,9 @@ class MooncakeStoreWorker: ) self.cache_config = vllm_config.cache_config self.original_block_size = self.cache_config.block_size - self.block_size = self.cache_config.block_size - if self.pcp_size > 1: - self.block_size *= self.pcp_size - if self.dcp_size > 1: - self.block_size *= self.dcp_size + self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( + kv_cache_config, vllm_config + ) self.num_layers = model_config.get_num_layers(parallel_config) self.use_mla = False @@ -806,8 +838,6 @@ class MooncakeStoreWorker: pp_rank=self.pp_rank, ) - self.token_database = ChunkedTokenDatabase(self.metadata, self.block_size) - # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_env() extra_config = ( @@ -892,6 +922,45 @@ class MooncakeStoreWorker: self.kv_recv_thread: KVCacheStoreRecvingThread | None = None self.finished_store_req: set[str] = set() + self._kv_cache_config = kv_cache_config + # Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to + # self.block_size (= scheduler_block_size) so the coordinator's + # ``block_size % hash_block_size == 0`` invariant holds. + groups = list(kv_cache_config.kv_cache_groups) + if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size: + g = groups[0] + groups = [ + dataclasses.replace( + g, + kv_cache_spec=dataclasses.replace( + g.kv_cache_spec, block_size=self.block_size + ), + ) + ] + self._kv_cache_groups: list[KVCacheGroupSpec] = groups + spec_cfg = getattr(vllm_config, "speculative_config", None) + use_eagle = bool( + spec_cfg.use_eagle() + if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None)) + else False + ) + self.coord = MooncakeStoreCoordinator( + self._kv_cache_groups, + scheduler_block_size=self.block_size, + hash_block_size=self.hash_block_size, + use_eagle=use_eagle, + ) + # One ChunkedTokenDatabase per group; addresses populated in + # register_kv_caches once the kv-cache layout is known. + self.token_dbs: list[ChunkedTokenDatabase] = [ + ChunkedTokenDatabase( + dataclasses.replace(self.metadata, group_id=g_idx), + g.kv_cache_spec.block_size, + hash_block_size=self.hash_block_size, + ) + for g_idx, g in enumerate(self._kv_cache_groups) + ] + def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None: """Register a cross-layers KV cache tensor. @@ -901,93 +970,86 @@ class MooncakeStoreWorker: """ self.register_kv_caches({"__cross_layer__": kv_cache}) - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): + def register_kv_caches( + self, + kv_caches: dict[str, torch.Tensor | list[torch.Tensor]], + ) -> None: """Register KV cache tensors and start transfer threads.""" - # TODO(yifan): we haven't supported HMA yet. - first_kv_cache = next(iter(kv_caches.values())) + if not kv_caches: + logger.warning("No KV caches to offload.") + return + + # Resolve each entry to a representative tensor for storage + # deduplication. For attention layers the value is already a tensor; + # for Mamba layers it is a list of tensors that all share the same + # underlying raw storage, so we take the first one. + def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor: + assert isinstance(v, torch.Tensor | list) + return v if isinstance(v, torch.Tensor) else v[0] - # num_blocks from cache_config is authoritative (set after - # profiling, before KV cache allocation). assert self.cache_config.num_gpu_blocks is not None self.num_blocks = self.cache_config.num_gpu_blocks - # Detect the KV cache memory layout using the stride-based - # approach from simple_kv_offload/worker.py. - # - # The physical layout varies across attention backends: - # FlashAttn/ROCm : (2, num_blocks, ...) → K/V outermost - # FlashInfer/MLA : (num_blocks, ...) → blocks outermost - # - # We derive page_size_bytes = storage.nbytes() // num_blocks, - # then classify dims: any dim whose byte-stride exceeds - # page_size_bytes must be an outer segment dim (e.g. the K/V - # dim of size 2). For those backends we register each segment - # (K, V) as a separate base-address so that the per-block - # offset arithmetic in prepare_value() stays correct. - storage = first_kv_cache.untyped_storage() - el = first_kv_cache.element_size() - page_size_bytes = storage.nbytes() // self.num_blocks - outer_dims = [ - d - for d in range(first_kv_cache.ndim) - if first_kv_cache.stride(d) * el > page_size_bytes - ] - - # Register buffers with the store (deduplicate shared storages) - # and record per-segment base addresses for every layer. seen_ptrs: set[int] = set() - self.kv_caches_base_addr: list[int] = [] - self.block_len: list[int] = [] + addrs: list[int] = [] + block_lens: list[int] = [] - for cache in kv_caches.values(): + for value in kv_caches.values(): + cache = _repr_tensor(value) cache_storage = cache.untyped_storage() base_addr = cache_storage.data_ptr() + if base_addr in seen_ptrs: + continue + seen_ptrs.add(base_addr) region_len = cache_storage.nbytes() - if base_addr not in seen_ptrs: - seen_ptrs.add(base_addr) - ret = self.store.register_buffer(base_addr, region_len) - if ret != 0: - logger.error( - "register_buffer failed for addr %#x len %d: %d", - base_addr, - region_len, - ret, - ) + ret = self.store.register_buffer(base_addr, region_len) + if ret != 0: + logger.error( + "register_buffer failed for addr %#x len %d: %d", + base_addr, + region_len, + ret, + ) + # Detect layout via stride: a dim whose byte-stride exceeds + # page_size_bytes is an outer segment dim (e.g. the K/V dim of + # FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks- + # outermost layout has no such dim and yields a single segment. + el = cache.element_size() + page_size_bytes = region_len // self.num_blocks + outer_dims = [ + d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes + ] if not outer_dims: # Blocks-first layout (FlashInfer / MLA): one segment. - self.kv_caches_base_addr.append(base_addr) - self.block_len.append(page_size_bytes) + addrs.append(base_addr) + block_lens.append(page_size_bytes) else: # K/V-first layout (FlashAttn / ROCm): split segments. seg_stride = cache.stride(outer_dims[0]) * el for idx in range(cache.shape[outer_dims[0]]): - self.kv_caches_base_addr.append(base_addr + idx * seg_stride) - self.block_len.append(seg_stride // self.num_blocks) + addrs.append(base_addr + idx * seg_stride) + block_lens.append(seg_stride // self.num_blocks) logger.info( - "Registering KV_Caches. use_mla: %s, shape %s, " - "num_blocks: %d, block_len: %s, " - "per_key_bytes: %d, " - "num_segments: %d", - self.use_mla, - first_kv_cache.shape, + "Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d", + len(self.token_dbs), + len(addrs), self.num_blocks, - list(set(self.block_len)), - sum(self.block_len), - len(self.kv_caches_base_addr), ) - self.token_database.set_kv_caches_base_addr(self.kv_caches_base_addr) - self.token_database.set_block_len(self.block_len) + for db in self.token_dbs: + db.set_kv_caches_base_addr(addrs) + db.set_block_len(block_lens) # Start transfer threads if self.kv_role in ["kv_producer", "kv_both"]: ready_event_sending = threading.Event() self.kv_send_thread = KVCacheStoreSendingThread( self.store, - self.token_database, + self.coord, + self.token_dbs, self.block_size, self.tp_rank, self.put_step, @@ -1001,7 +1063,8 @@ class MooncakeStoreWorker: ready_event_recving = threading.Event() self.kv_recv_thread = KVCacheStoreRecvingThread( self.store, - self.token_database, + self.coord, + self.token_dbs, self.block_size, self.tp_rank, ready_event_recving, @@ -1122,62 +1185,55 @@ class MooncakeStoreWorker: return finished_sending - def lookup( - self, - token_len: int, - block_hashes: list[BlockHash], - ) -> int: + def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int: """Check how many prefix tokens exist in the store. Checks across all TP ranks and PP ranks. """ - end = 0 - keys: list[str] = [] + if not block_hashes or token_len <= 0: + return 0 + + # Build per-(group, hash) candidate keys expanded across TP/PP. + # candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i]. + candidate_keys: list[str] = [] + candidate_meta: list[tuple[int, bytes]] = [] + tp_count = min(self.tp_size, self.num_kv_head) + for g_idx, db in enumerate(self.token_dbs): + spec_block_size = db.block_size + group_hashes = self.coord.block_hashes_for_spec( + block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec + ) + for chunk_id, h in enumerate(group_hashes): + start_idx = chunk_id * spec_block_size + if start_idx >= token_len: + break + for tp in range(tp_count): + for pp in range(self.pp_size): + md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp) + candidate_keys.append(PoolKey(md, h.hex()).to_string()) + candidate_meta.append((g_idx, bytes(h))) + + if not candidate_keys: + return 0 + try: - starts: list[int] = [] - for start, end, key in self.token_database.process_tokens( - token_len, block_hashes - ): - keys.append(key.to_string()) - starts.append(start) - - # Expand keys for all TP ranks - multi_tp_keys = keys[:] - for i in range(1, min(self.tp_size, self.num_kv_head)): - for item in keys: - new_str = item.replace("@tp_rank:0", f"@tp_rank:{i}", 1) - multi_tp_keys.append(new_str) - - # Expand keys for all PP ranks - pp_base_keys = multi_tp_keys.copy() - for i in range(1, self.pp_size): - for item in pp_base_keys: - new_str = item.replace("@pp_rank:0", f"@pp_rank:{i}", 1) - multi_tp_keys.append(new_str) - - res = self.store.batch_is_exist(multi_tp_keys) - - num_block = len(keys) - multi_tp_values = [ - res[i * num_block : (i + 1) * num_block] - for i in range(min(self.tp_size, self.num_kv_head) * self.pp_size) - ] - index = self._find_min_first_non_one_index(multi_tp_values) - if index != -1: - return starts[index] + res = self.store.batch_is_exist(candidate_keys) except Exception as e: logger.error("Remote connection failed in lookup: %s", e) return 0 - return end - @staticmethod - def _find_min_first_non_one_index( - arr: list[list[int]], - ) -> int: - try: - return min(idx for row in arr for idx, val in enumerate(row) if val != 1) - except ValueError: - return -1 + # A (group, hash) is "present" only when every TP*PP rank has it. + expected_per_key = max(1, tp_count * self.pp_size) + present_count: dict[tuple[int, bytes], int] = {} + for gh, exists in zip(candidate_meta, res, strict=True): + if exists == 1: + present_count[gh] = present_count.get(gh, 0) + 1 + exists_set = {gh for gh, c in present_count.items() if c >= expected_per_key} + + _masks, hit_length = self.coord.find_longest_cache_hit( + block_hashes, token_len, ExternalCachedBlockPool(exists_set) + ) + return hit_length def get_kv_events(self) -> list[BlockStored]: if self.enable_kv_events and self.kv_send_thread is not None: @@ -1220,7 +1276,8 @@ class LookupKeyServer: token_len = int.from_bytes(all_frames[0], byteorder="big") hash_frames = all_frames[1:] hashes_str = self.decoder.decode(hash_frames) - result = self.store_worker.lookup(token_len, hashes_str) + block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str] + result = self.store_worker.lookup(token_len, block_hashes) response = result.to_bytes(4, "big") self.socket.send(response)