[Mooncake] Allow Mamba cache mode none for capacity-only workers (#60712)

Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Wei Zhao
2026-10-13 23:13:58 +00:00
committed by GitHub
co-authored by Codex Claude Opus 5.5
parent 96fc2da315
commit 8082b62f32
3 changed files with 112 additions and 18 deletions
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
@@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly():
)
def test_scheduler_requires_align_mode_for_mamba():
vllm_config = _make_vllm_config()
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
def _make_mamba_none_kv_cache_config(
attention_block_size: int | None, mamba_block_size: int
) -> KVCacheConfig:
groups = [
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
),
)
]
if attention_block_size is not None:
# Kimi K3 layout without prefix caching: the LCM hash unit exceeds both
# group block sizes, so no lookup coordinator can be built.
groups.insert(
0,
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=8,
head_size=64,
dtype=torch.float32,
),
),
)
return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups)
def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool):
vllm_config = create_vllm_config(
kv_connector="MooncakeStoreConnector",
kv_role=kv_role,
kv_connector_extra_config={
"enable_lookup": enable_lookup,
"save_decode_cache": save_decode_cache,
},
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
vllm_config.cache_config.enable_prefix_caching = False
return vllm_config
_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize(
"attention_block_size,mamba_block_size",
[(None, 16), (1_048_576, 6_400)],
ids=["mamba-only", "hybrid-no-prefix"],
)
@_MAMBA_NONE_LAYOUTS
@pytest.mark.parametrize(
"kv_role,enable_lookup,save_decode_cache",
[
("kv_consumer", True, False),
("kv_consumer", False, True),
("kv_producer", False, False),
("kv_both", False, False),
],
)
def test_scheduler_requires_align_mode_for_mamba_kv_transfer(
kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size
):
vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with (
@@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba():
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
@_MAMBA_NONE_LAYOUTS
def test_capacity_only_scheduler_allows_mamba_none_mode(
attention_block_size, mamba_block_size
):
"""Capacity-only consumers transfer no KV, so any Mamba mode is allowed."""
vllm_config = _make_role_vllm_config(
"kv_consumer", enable_lookup=False, save_decode_cache=False
)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
) as mock_client:
store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False)
meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty())
mock_client.return_value.lookup.assert_not_called()
assert meta.requests == []
def _make_block_stored(
block_hash: bytes = b"hash", group_idx: int | None = None
) -> BlockStored:
@@ -49,6 +49,7 @@ def _make_bare_scheduler(
scheduler.enable_kv_events = False
scheduler.lookup_async = False
scheduler.enable_lookup = True
scheduler._capacity_only = False
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits