[Mamba] enable prefix cache by default (#50991)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-04 15:20:51 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7b50d2c0bc
commit f9c74b4b9c
4 changed files with 30 additions and 36 deletions
+24 -17
View File
@@ -42,11 +42,6 @@ HYBRID_MODELS = [
"tiny-random/qwen3-next-moe",
]
HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL = {
"LiquidAI/LFM2-1.2B",
"tiny-random/qwen3-next-moe",
}
FULL_CUDA_GRAPH_MODELS = [
"ai21labs/Jamba-tiny-dev",
"Zyphra/Zamba2-1.2B-instruct",
@@ -95,15 +90,11 @@ def test_models(
example_prompts, max_tokens, num_logprobs
)
extra_kwargs = {}
if model in HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL:
extra_kwargs["enable_chunked_prefill"] = True
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
**extra_kwargs,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -140,7 +131,9 @@ def test_batching(
_set_conv_state_layout(monkeypatch, conv_state_layout)
for_loop_outputs = []
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for prompt in example_prompts:
(single_output,) = vllm_model.generate_greedy_logprobs(
[prompt], max_tokens, num_logprobs
@@ -224,7 +217,7 @@ def test_mamba_cache_cg_padding(
example_prompts.append(example_prompts[0])
try:
with vllm_runner(model) as vllm_model:
with vllm_runner(model, enable_chunked_prefill=True) as vllm_model:
vllm_model.generate_greedy(example_prompts, max_tokens)
except RuntimeError:
pytest.fail(
@@ -250,7 +243,9 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
a single step.
"""
try:
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
except ValueError:
pytest.fail(
@@ -272,7 +267,9 @@ def test_state_cleanup(
If it's not cleaned, an error would be expected.
"""
try:
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for _ in range(10):
vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
except ValueError:
@@ -294,14 +291,20 @@ def test_distributed_correctness(
num_logprobs: int,
) -> None:
with vllm_runner(
model, tensor_parallel_size=1, max_num_seqs=MAX_NUM_SEQS
model,
tensor_parallel_size=1,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(
model, tensor_parallel_size=2, max_num_seqs=MAX_NUM_SEQS
model,
tensor_parallel_size=2,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -340,7 +343,10 @@ def test_full_cuda_graph(
)
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
@@ -388,6 +394,7 @@ def test_fp32_cache_state(
model,
max_num_seqs=MAX_NUM_SEQS,
gpu_memory_utilization=0.9,
enable_chunked_prefill=True,
**{cache_dtype_param: "float32"},
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(