From 8082b62f3202539e969b090b53723257ca409607 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:13:58 -0400 Subject: [PATCH] [Mooncake] Allow Mamba cache mode `none` for capacity-only workers (#60712) Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Signed-off-by: wzhao18 Co-authored-by: Codex Co-authored-by: Claude Opus 5.5 --- .../unit/test_mooncake_store_connector.py | 103 ++++++++++++++++-- .../unit/test_mooncake_store_scheduler.py | 1 + .../v1/mooncake/store/scheduler.py | 26 +++-- 3 files changed, 112 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index 824f806f48..4e5d9d2e7f 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( MooncakeStoreConnectorStats, ) +from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( CircularBufferSpec, FullAttentionSpec, @@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly(): ) -def test_scheduler_requires_align_mode_for_mamba(): - vllm_config = _make_vllm_config() - mamba_spec = MambaSpec( - block_size=16, - shapes=((1, 1),), - dtypes=(torch.float32,), - mamba_cache_mode="none", +def _make_mamba_none_kv_cache_config( + attention_block_size: int | None, mamba_block_size: int +) -> KVCacheConfig: + groups = [ + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=mamba_block_size, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="none", + ), + ) + ] + if attention_block_size is not None: + # Kimi K3 layout without prefix caching: the LCM hash unit exceeds both + # group block sizes, so no lookup coordinator can be built. + groups.insert( + 0, + KVCacheGroupSpec( + ["attention"], + FullAttentionSpec( + block_size=attention_block_size, + num_kv_heads=8, + head_size=64, + dtype=torch.float32, + ), + ), + ) + return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups) + + +def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool): + vllm_config = create_vllm_config( + kv_connector="MooncakeStoreConnector", + kv_role=kv_role, + kv_connector_extra_config={ + "enable_lookup": enable_lookup, + "save_decode_cache": save_decode_cache, + }, ) - kv_cache_config = KVCacheConfig( - num_blocks=4, - kv_cache_tensors=[], - kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)], + vllm_config.cache_config.enable_prefix_caching = False + return vllm_config + + +_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize( + "attention_block_size,mamba_block_size", + [(None, 16), (1_048_576, 6_400)], + ids=["mamba-only", "hybrid-no-prefix"], +) + + +@_MAMBA_NONE_LAYOUTS +@pytest.mark.parametrize( + "kv_role,enable_lookup,save_decode_cache", + [ + ("kv_consumer", True, False), + ("kv_consumer", False, True), + ("kv_producer", False, False), + ("kv_both", False, False), + ], +) +def test_scheduler_requires_align_mode_for_mamba_kv_transfer( + kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size +): + vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache) + kv_cache_config = _make_mamba_none_kv_cache_config( + attention_block_size, mamba_block_size ) with ( @@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba(): scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config) +@_MAMBA_NONE_LAYOUTS +def test_capacity_only_scheduler_allows_mamba_none_mode( + attention_block_size, mamba_block_size +): + """Capacity-only consumers transfer no KV, so any Mamba mode is allowed.""" + vllm_config = _make_role_vllm_config( + "kv_consumer", enable_lookup=False, save_decode_cache=False + ) + kv_cache_config = _make_mamba_none_kv_cache_config( + attention_block_size, mamba_block_size + ) + + with patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." + "scheduler.LookupKeyClient" + ) as mock_client: + store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config) + assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False) + meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty()) + + mock_client.return_value.lookup.assert_not_called() + assert meta.requests == [] + + def _make_block_stored( block_hash: bytes = b"hash", group_idx: int | None = None ) -> BlockStored: diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 47796eb147..ec92fb9102 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -49,6 +49,7 @@ def _make_bare_scheduler( scheduler.enable_kv_events = False scheduler.lookup_async = False scheduler.enable_lookup = True + scheduler._capacity_only = False scheduler._block_size = 16 scheduler._hash_block_size = hash_block_size scheduler.enable_partial_hash_hits = enable_partial_hash_hits diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 239dad7364..18d961abb2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -172,15 +172,24 @@ class MooncakeStoreScheduler: for group_id, group in enumerate(store_groups) if isinstance(group.kv_cache_spec, MambaSpec) } - assert all( - spec.mamba_cache_mode == "align" for spec in mamba_groups.values() - ), "MooncakeStoreScheduler requires mamba_cache_mode='align'" self._boundary_state_group_ids = frozenset(mamba_groups) - self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config( - kv_cache_config, vllm_config, self._block_size, self._hash_block_size + # Mirrors MooncakeStoreConnector._capacity_only. + self._capacity_only = ( + self.kv_role == "kv_consumer" + and not self.enable_lookup + and not self.save_decode_cache ) - self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits + self._store_coord: MooncakeStoreCoordinator | None = None + self.enable_partial_hash_hits = False + if not self._capacity_only: + assert all( + spec.mamba_cache_mode == "align" for spec in mamba_groups.values() + ), "MooncakeStoreScheduler requires mamba_cache_mode='align'" + self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config( + kv_cache_config, vllm_config, self._block_size, self._hash_block_size + ) + self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits self._gpu_block_pool: BlockPool | None = None self._num_workers = vllm_config.parallel_config.world_size @@ -511,7 +520,8 @@ class MooncakeStoreScheduler: meta.add_request(req_meta) self._finished_partial_tail_metas.clear() - self._reference_save_blocks(meta) + if not self._capacity_only: + self._reference_save_blocks(meta) return meta def _apply_current_save_block_ids( @@ -546,6 +556,7 @@ class MooncakeStoreScheduler: """ pool = self._gpu_block_pool coord = self._store_coord + assert coord is not None block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups] for req_meta in meta.requests: if not req_meta.can_save: @@ -629,6 +640,7 @@ class MooncakeStoreScheduler: completed_token_len=request.num_computed_tokens, ) coord = self._store_coord + assert coord is not None req_meta.boundary_puts = [ *_partial_tail_non_mamba_puts( coord,