[Core][Mamba] Remove leftovers of mamba_cache_mode="all" (#60043)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Jiangyun Zhu <zhujiangyun@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-10-13 00:58:59 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bc3ddb198a
commit 2cc2e68d1c
33 changed files with 161 additions and 344 deletions
@@ -180,7 +180,6 @@ def _run_k3_partial_prefix_reuse(
block_size=block_size,
enable_chunked_prefill=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
) as runner:
assert (
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
@@ -23,4 +23,4 @@ generation_config:
extra_body:
chat_template_kwargs:
enable_thinking: true
server_args: "--tensor-parallel-size 4 --mamba-cache-mode align"
server_args: "--tensor-parallel-size 4"
+6 -4
View File
@@ -96,15 +96,17 @@ def mixer2_gated_norm_tensor_parallel(
gate_states = torch.randn(batch_size, seq_len, hidden_size)
# create gated-norm with TP
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
with ensure_current_vllm_config():
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
mixer.weight.weight_loader(mixer.weight, weight) # load
# create gated-norm without TP to compute reference
# - utilize mock patching to disable TP when
with (
ensure_current_vllm_config(),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_world_size",
@@ -38,9 +38,7 @@ def _manager(
num_prefill_lookahead=0,
):
init_none_hash(sha256)
config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
)
config = _make_hybrid_kv_cache_config(block_size, num_blocks, ["full", "mamba"])
if eagle_group is not None:
groups = list(config.kv_cache_groups)
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
+8 -1
View File
@@ -278,7 +278,14 @@ class TestCSALinearGrouping:
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs().items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=BYTES_PER_BLOCK * 64
)
+25 -65
View File
@@ -1617,6 +1617,7 @@ def make_kv_cache_config_hybrid_model(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
return KVCacheConfig(
@@ -1652,6 +1653,7 @@ def make_kv_cache_config_three_types(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
elif third_spec_type == "sliding_window":
third_spec = SlidingWindowSpec(
@@ -1732,6 +1734,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
@@ -1757,10 +1760,13 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, len(req0.prompt_token_ids), 0, computed_blocks
)
assert blocks is not None
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the shared prefix.
for num_new_tokens in (len(common_token_ids), 7):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
@@ -2285,7 +2291,7 @@ def _make_hybrid_kv_cache_config(
- "full": FullAttentionSpec
- "sliding_window": SlidingWindowSpec with window=2*block_size
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
- "mamba": MambaSpec
- "mamba": MambaSpec in prefix-caching ("align") mode
"""
spec_map = {
@@ -2313,11 +2319,6 @@ def _make_hybrid_kv_cache_config(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
),
"mamba_align": lambda: MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
}
@@ -2427,14 +2428,16 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
assert not computed_blocks.blocks[0] # No cache hit initially
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, 55, len(computed_blocks.blocks[0]) * block_size, computed_blocks
)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
manager.new_step_starts()
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the common prefix.
for num_new_tokens in (len(common_token_ids), len(unique_token_ids)):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
manager.new_step_starts()
# Second request: should hit cached blocks for common prefix
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
@@ -2549,7 +2552,7 @@ def test_prefill_hybrid_model_mamba_align():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = make_kv_cache_manager(
kv_cache_config,
@@ -2585,7 +2588,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
"""
block_size = 16
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
@@ -2660,7 +2663,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = KVCacheManager(
kv_cache_config,
@@ -5896,49 +5899,6 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
assert retained(0, None) == {14}
def test_mamba_shared_prefix_survives_zero_retention():
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
token_ids = [i for i in range(16) for _ in range(block_size)]
def cached_mamba_blocks(shared_prefix_boundary):
# Fresh manager per scenario so cached blocks don't leak between runs.
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
)
req = make_request("r", token_ids, block_size, sha256)
req.shared_prefix_boundary = shared_prefix_boundary
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req, len(token_ids), num_computed, computed_blocks
)
assert blocks is not None
pool = manager.block_pool
return {
i
for i in range(16)
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
}
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
assert cached_mamba_blocks(0) == {14}
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
# later request sharing that prefix can hit the Mamba state.
assert cached_mamba_blocks(96) == {5, 14}
def test_mamba_shared_prefix_reuse_under_zero_retention():
"""Full cross-request Marconi flow: a partial shared prefix cached by the
detecting request must stay reusable by a later request under
@@ -5949,7 +5909,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
def last_req_hit(retention, pin):
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
@@ -898,7 +898,6 @@ def _run_mamba_prefix_cache_mrv1(
load_format="dummy",
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1158,7 +1157,6 @@ def _run_mamba_prefix_cache_mrv2(
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
trust_remote_code=True,
enable_chunked_prefill=True,
enable_prefix_caching=enable_prefix_caching,
mamba_cache_mode="align" if enable_prefix_caching else "none",
) as llm:
sampling_params = SamplingParams(
temperature=0.0,
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
enforce_eager=True,
enable_chunked_prefill=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
) as runner:
llm = runner.get_llm()
-4
View File
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
tensor_parallel_size=tensor_parallel_size,
mamba_backend=mamba_backend,
)
@@ -130,7 +129,6 @@ def test_replayssm_flashinfer_spec_decode_matches_baseline(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
speculative_config={
"method": "ngram",
"num_speculative_tokens": 3,
@@ -182,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
mamba_backend="flashinfer",
tensor_parallel_size=2,
disable_log_stats=False,
@@ -250,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
trust_remote_code=True,
enable_prefix_caching=True,
enable_chunked_prefill=True,
mamba_cache_mode="align",
disable_log_stats=False, # required for llm.get_metrics()
tensor_parallel_size=tensor_parallel_size,
)
@@ -60,7 +60,6 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -78,7 +77,6 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -583,7 +583,6 @@ def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
kv_transfer_config=kv_transfer_config,
language_model_only=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
# Use lossless state storage so this exact-equality regression isolates
# offload boundary selection from low-precision Mamba checkpointing.
mamba_cache_dtype="float32",
@@ -632,6 +632,8 @@ def test_p_side_chunked_prefill_mamba(mock_platform):
block_size=BLOCK_SIZE,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
# mamba_cache_mode "none" only runs without prefix caching.
vllm_config.cache_config.enable_prefix_caching = False
kv_cache_config = make_kv_cache_config(
block_size=BLOCK_SIZE,
@@ -855,6 +857,7 @@ def test_async_load_does_not_double_reserve_mamba_spec_blocks(
disable_hybrid_kv_cache_manager=False,
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.enable_prefix_caching = mamba_cache_mode == "align"
kv_cache_config = make_kv_cache_config(
block_size,
mamba_enabled=True,
+58 -16
View File
@@ -49,6 +49,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVCacheTensor,
MambaSpec,
SlidingWindowSpec,
@@ -2355,8 +2356,6 @@ def _make_hybrid_attention_mamba_scheduler(
enable_kv_cache_events: bool = False,
) -> SchedulerFixture:
"""Build a scheduler for one attention group plus one Mamba group."""
scheduler_block_size = scheduler_block_size or block_size
hash_block_size = hash_block_size or block_size
attention_spec = FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=NUM_KV_HEADS,
@@ -2369,9 +2368,38 @@ def _make_hybrid_attention_mamba_scheduler(
dtypes=(torch.float32,),
mamba_cache_mode=mamba_cache_mode,
)
return _make_two_group_scheduler(
attention_spec,
("mamba", mamba_spec),
num_cpu_blocks=num_cpu_blocks,
num_gpu_blocks=num_gpu_blocks,
scheduler_block_size=scheduler_block_size or block_size,
hash_block_size=hash_block_size or block_size,
dcp_world_size=dcp_world_size,
lazy=lazy,
mamba_cache_mode=mamba_cache_mode,
enable_kv_cache_events=enable_kv_cache_events,
)
def _make_two_group_scheduler(
attention_spec: FullAttentionSpec,
second_group: tuple[str, KVCacheSpec],
*,
num_cpu_blocks: int,
num_gpu_blocks: int,
scheduler_block_size: int,
hash_block_size: int,
dcp_world_size: int,
lazy: bool = False,
mamba_cache_mode: MambaCacheMode = "align",
enable_kv_cache_events: bool = False,
) -> SchedulerFixture:
"""Build a scheduler for one attention group plus one named second group."""
second_name, second_spec = second_group
groups = [
KVCacheGroupSpec(["attention"], attention_spec),
KVCacheGroupSpec(["mamba"], mamba_spec),
KVCacheGroupSpec([second_name], second_spec),
]
tensors = [
KVCacheTensor(
@@ -2380,7 +2408,7 @@ def _make_hybrid_attention_mamba_scheduler(
layer_stride=spec.page_size_bytes * num_gpu_blocks,
block_stride=spec.page_size_bytes,
)
for group, spec in zip(groups, (attention_spec, mamba_spec))
for group, spec in zip(groups, (attention_spec, second_spec))
]
kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
@@ -2428,34 +2456,48 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
silently offloads only a fraction of them.
"""
attention_block_size = BLOCK_SIZE
mamba_block_size = 4 * BLOCK_SIZE
replicated_block_size = 4 * BLOCK_SIZE
dcp_world_size = 2
fix = _make_hybrid_attention_mamba_scheduler(
fix = _make_two_group_scheduler(
FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
),
(
"replicated_attention",
FullAttentionSpec(
block_size=replicated_block_size,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
dcp_sharded=False,
),
),
num_cpu_blocks=32,
num_gpu_blocks=32,
attention_block_size=attention_block_size,
block_size=mamba_block_size,
scheduler_block_size=replicated_block_size,
# Every prefix-cacheable group's resolved block size must be a multiple
# of the hash block, so it cannot exceed the smallest of them.
hash_block_size=BLOCK_SIZE,
dcp_world_size=dcp_world_size,
mamba_cache_mode="none",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_size, mamba_size = sched.group_block_sizes
attention_size, replicated_size = sched.group_block_sizes
assert attention_size == attention_block_size * dcp_world_size
assert mamba_size == mamba_block_size
assert replicated_size == replicated_block_size
confirmed = 2 * sched.block_size
req = _make_cp_request(num_blocks=8, virtual_block_size=BLOCK_SIZE)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, confirmed // attention_size, attention_size, group_id=0
)
mamba_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, confirmed // mamba_size, mamba_size, group_id=1
replicated_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, confirmed // replicated_size, replicated_size, group_id=1
)
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, replicated_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
sched.build_connector_meta(
make_scheduler_output(
@@ -2476,11 +2518,11 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
# count.
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [
confirmed // attention_size,
confirmed // mamba_size,
confirmed // replicated_size,
]
assert set(meta.store_gpu_blocks) == {
*(b.block_id for b in attention_blocks),
*(b.block_id for b in mamba_blocks),
*(b.block_id for b in replicated_blocks),
}