[Mooncake] Allow Mamba cache mode none for capacity-only workers (#60712)

Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Wei Zhao
2026-10-13 23:13:58 +00:00
committed by GitHub
co-authored by Codex Claude Opus 5.5
parent 96fc2da315
commit 8082b62f32
3 changed files with 112 additions and 18 deletions
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
@@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly():
)
def test_scheduler_requires_align_mode_for_mamba():
vllm_config = _make_vllm_config()
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
def _make_mamba_none_kv_cache_config(
attention_block_size: int | None, mamba_block_size: int
) -> KVCacheConfig:
groups = [
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
),
)
]
if attention_block_size is not None:
# Kimi K3 layout without prefix caching: the LCM hash unit exceeds both
# group block sizes, so no lookup coordinator can be built.
groups.insert(
0,
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=8,
head_size=64,
dtype=torch.float32,
),
),
)
return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups)
def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool):
vllm_config = create_vllm_config(
kv_connector="MooncakeStoreConnector",
kv_role=kv_role,
kv_connector_extra_config={
"enable_lookup": enable_lookup,
"save_decode_cache": save_decode_cache,
},
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
vllm_config.cache_config.enable_prefix_caching = False
return vllm_config
_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize(
"attention_block_size,mamba_block_size",
[(None, 16), (1_048_576, 6_400)],
ids=["mamba-only", "hybrid-no-prefix"],
)
@_MAMBA_NONE_LAYOUTS
@pytest.mark.parametrize(
"kv_role,enable_lookup,save_decode_cache",
[
("kv_consumer", True, False),
("kv_consumer", False, True),
("kv_producer", False, False),
("kv_both", False, False),
],
)
def test_scheduler_requires_align_mode_for_mamba_kv_transfer(
kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size
):
vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with (
@@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba():
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
@_MAMBA_NONE_LAYOUTS
def test_capacity_only_scheduler_allows_mamba_none_mode(
attention_block_size, mamba_block_size
):
"""Capacity-only consumers transfer no KV, so any Mamba mode is allowed."""
vllm_config = _make_role_vllm_config(
"kv_consumer", enable_lookup=False, save_decode_cache=False
)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
) as mock_client:
store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False)
meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty())
mock_client.return_value.lookup.assert_not_called()
assert meta.requests == []
def _make_block_stored(
block_hash: bytes = b"hash", group_idx: int | None = None
) -> BlockStored:
@@ -49,6 +49,7 @@ def _make_bare_scheduler(
scheduler.enable_kv_events = False
scheduler.lookup_async = False
scheduler.enable_lookup = True
scheduler._capacity_only = False
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
@@ -172,15 +172,24 @@ class MooncakeStoreScheduler:
for group_id, group in enumerate(store_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
}
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._boundary_state_group_ids = frozenset(mamba_groups)
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not self.enable_lookup
and not self.save_decode_cache
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._store_coord: MooncakeStoreCoordinator | None = None
self.enable_partial_hash_hits = False
if not self._capacity_only:
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
@@ -511,7 +520,8 @@ class MooncakeStoreScheduler:
meta.add_request(req_meta)
self._finished_partial_tail_metas.clear()
self._reference_save_blocks(meta)
if not self._capacity_only:
self._reference_save_blocks(meta)
return meta
def _apply_current_save_block_ids(
@@ -546,6 +556,7 @@ class MooncakeStoreScheduler:
"""
pool = self._gpu_block_pool
coord = self._store_coord
assert coord is not None
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
for req_meta in meta.requests:
if not req_meta.can_save:
@@ -629,6 +640,7 @@ class MooncakeStoreScheduler:
completed_token_len=request.num_computed_tokens,
)
coord = self._store_coord
assert coord is not None
req_meta.boundary_puts = [
*_partial_tail_non_mamba_puts(
coord,