[Mooncake] Allow Mamba cache mode none for capacity-only workers (#60712)
Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
parent
96fc2da315
commit
8082b62f32
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly():
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_requires_align_mode_for_mamba():
|
||||
vllm_config = _make_vllm_config()
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
def _make_mamba_none_kv_cache_config(
|
||||
attention_block_size: int | None, mamba_block_size: int
|
||||
) -> KVCacheConfig:
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
),
|
||||
)
|
||||
]
|
||||
if attention_block_size is not None:
|
||||
# Kimi K3 layout without prefix caching: the LCM hash unit exceeds both
|
||||
# group block sizes, so no lookup coordinator can be built.
|
||||
groups.insert(
|
||||
0,
|
||||
KVCacheGroupSpec(
|
||||
["attention"],
|
||||
FullAttentionSpec(
|
||||
block_size=attention_block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
)
|
||||
return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups)
|
||||
|
||||
|
||||
def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeStoreConnector",
|
||||
kv_role=kv_role,
|
||||
kv_connector_extra_config={
|
||||
"enable_lookup": enable_lookup,
|
||||
"save_decode_cache": save_decode_cache,
|
||||
},
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
return vllm_config
|
||||
|
||||
|
||||
_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize(
|
||||
"attention_block_size,mamba_block_size",
|
||||
[(None, 16), (1_048_576, 6_400)],
|
||||
ids=["mamba-only", "hybrid-no-prefix"],
|
||||
)
|
||||
|
||||
|
||||
@_MAMBA_NONE_LAYOUTS
|
||||
@pytest.mark.parametrize(
|
||||
"kv_role,enable_lookup,save_decode_cache",
|
||||
[
|
||||
("kv_consumer", True, False),
|
||||
("kv_consumer", False, True),
|
||||
("kv_producer", False, False),
|
||||
("kv_both", False, False),
|
||||
],
|
||||
)
|
||||
def test_scheduler_requires_align_mode_for_mamba_kv_transfer(
|
||||
kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size
|
||||
):
|
||||
vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache)
|
||||
kv_cache_config = _make_mamba_none_kv_cache_config(
|
||||
attention_block_size, mamba_block_size
|
||||
)
|
||||
|
||||
with (
|
||||
@@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba():
|
||||
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
@_MAMBA_NONE_LAYOUTS
|
||||
def test_capacity_only_scheduler_allows_mamba_none_mode(
|
||||
attention_block_size, mamba_block_size
|
||||
):
|
||||
"""Capacity-only consumers transfer no KV, so any Mamba mode is allowed."""
|
||||
vllm_config = _make_role_vllm_config(
|
||||
"kv_consumer", enable_lookup=False, save_decode_cache=False
|
||||
)
|
||||
kv_cache_config = _make_mamba_none_kv_cache_config(
|
||||
attention_block_size, mamba_block_size
|
||||
)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
) as mock_client:
|
||||
store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False)
|
||||
meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty())
|
||||
|
||||
mock_client.return_value.lookup.assert_not_called()
|
||||
assert meta.requests == []
|
||||
|
||||
|
||||
def _make_block_stored(
|
||||
block_hash: bytes = b"hash", group_idx: int | None = None
|
||||
) -> BlockStored:
|
||||
|
||||
@@ -49,6 +49,7 @@ def _make_bare_scheduler(
|
||||
scheduler.enable_kv_events = False
|
||||
scheduler.lookup_async = False
|
||||
scheduler.enable_lookup = True
|
||||
scheduler._capacity_only = False
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
|
||||
@@ -172,15 +172,24 @@ class MooncakeStoreScheduler:
|
||||
for group_id, group in enumerate(store_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
}
|
||||
assert all(
|
||||
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._boundary_state_group_ids = frozenset(mamba_groups)
|
||||
|
||||
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
|
||||
# Mirrors MooncakeStoreConnector._capacity_only.
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not self.enable_lookup
|
||||
and not self.save_decode_cache
|
||||
)
|
||||
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
|
||||
self._store_coord: MooncakeStoreCoordinator | None = None
|
||||
self.enable_partial_hash_hits = False
|
||||
if not self._capacity_only:
|
||||
assert all(
|
||||
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
|
||||
)
|
||||
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
@@ -511,7 +520,8 @@ class MooncakeStoreScheduler:
|
||||
meta.add_request(req_meta)
|
||||
self._finished_partial_tail_metas.clear()
|
||||
|
||||
self._reference_save_blocks(meta)
|
||||
if not self._capacity_only:
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def _apply_current_save_block_ids(
|
||||
@@ -546,6 +556,7 @@ class MooncakeStoreScheduler:
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
@@ -629,6 +640,7 @@ class MooncakeStoreScheduler:
|
||||
completed_token_len=request.num_computed_tokens,
|
||||
)
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
req_meta.boundary_puts = [
|
||||
*_partial_tail_non_mamba_puts(
|
||||
coord,
|
||||
|
||||
Reference in New Issue
Block a user