[Mamba] enable prefix cache by default (#50991)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-04 15:20:51 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7b50d2c0bc
commit f9c74b4b9c
4 changed files with 30 additions and 36 deletions
+24 -17
View File
@@ -42,11 +42,6 @@ HYBRID_MODELS = [
"tiny-random/qwen3-next-moe",
]
HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL = {
"LiquidAI/LFM2-1.2B",
"tiny-random/qwen3-next-moe",
}
FULL_CUDA_GRAPH_MODELS = [
"ai21labs/Jamba-tiny-dev",
"Zyphra/Zamba2-1.2B-instruct",
@@ -95,15 +90,11 @@ def test_models(
example_prompts, max_tokens, num_logprobs
)
extra_kwargs = {}
if model in HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL:
extra_kwargs["enable_chunked_prefill"] = True
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
**extra_kwargs,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -140,7 +131,9 @@ def test_batching(
_set_conv_state_layout(monkeypatch, conv_state_layout)
for_loop_outputs = []
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for prompt in example_prompts:
(single_output,) = vllm_model.generate_greedy_logprobs(
[prompt], max_tokens, num_logprobs
@@ -224,7 +217,7 @@ def test_mamba_cache_cg_padding(
example_prompts.append(example_prompts[0])
try:
with vllm_runner(model) as vllm_model:
with vllm_runner(model, enable_chunked_prefill=True) as vllm_model:
vllm_model.generate_greedy(example_prompts, max_tokens)
except RuntimeError:
pytest.fail(
@@ -250,7 +243,9 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
a single step.
"""
try:
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
except ValueError:
pytest.fail(
@@ -272,7 +267,9 @@ def test_state_cleanup(
If it's not cleaned, an error would be expected.
"""
try:
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for _ in range(10):
vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
except ValueError:
@@ -294,14 +291,20 @@ def test_distributed_correctness(
num_logprobs: int,
) -> None:
with vllm_runner(
model, tensor_parallel_size=1, max_num_seqs=MAX_NUM_SEQS
model,
tensor_parallel_size=1,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(
model, tensor_parallel_size=2, max_num_seqs=MAX_NUM_SEQS
model,
tensor_parallel_size=2,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -340,7 +343,10 @@ def test_full_cuda_graph(
)
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -388,6 +394,7 @@ def test_fp32_cache_state(
model,
max_num_seqs=MAX_NUM_SEQS,
gpu_memory_utilization=0.9,
enable_chunked_prefill=True,
**{cache_dtype_param: "float32"},
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
+3 -3
View File
@@ -135,10 +135,10 @@ class CacheConfig:
"""The cache strategy for Mamba layers:
- "none": set when prefix caching is disabled.
- "all": cache the mamba state of all tokens at position i * block_size. This is
the default behavior (for models that support it) when prefix caching is enabled.
- "all": cache the mamba state of all tokens at position i * block_size.
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size.
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
"""
replayssm_buffer_len: int = Field(default=16, gt=0)
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
+1 -5
View File
@@ -2587,11 +2587,7 @@ class EngineArgs:
self, model_config: ModelConfig
) -> None:
default_chunked_prefill = model_config.is_chunked_prefill_supported
# Hybrid models support prefix caching but keep it opt-in for now
# while the feature matures.
default_prefix_caching = (
model_config.is_prefix_caching_supported and not model_config.is_hybrid
)
default_prefix_caching = model_config.is_prefix_caching_supported
if self.enable_chunked_prefill is None:
self.enable_chunked_prefill = default_chunked_prefill
+2 -11
View File
@@ -595,10 +595,8 @@ class MambaModelConfig(VerifyAndUpdateConfig):
if cache_config.enable_prefix_caching:
if cache_config.mamba_cache_mode == "none":
cache_config.mamba_cache_mode = (
"all" if model_config.supports_mamba_prefix_caching else "align"
)
logger.warning(
cache_config.mamba_cache_mode = "align"
logger.info(
"Mamba cache mode is set to '%s' for %s by default "
"when prefix caching is enabled",
cache_config.mamba_cache_mode,
@@ -618,13 +616,6 @@ class MambaModelConfig(VerifyAndUpdateConfig):
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
logger.info(
"Warning: Prefix caching in Mamba cache '%s' "
"mode is currently enabled. "
"Its support for Mamba layers is experimental. "
"Please report any issues you may observe.",
cache_config.mamba_cache_mode,
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
# to the block size as the basic granularity for prefix caching.