[Mamba] enable prefix cache by default (#50991)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7b50d2c0bc
commit
f9c74b4b9c
@@ -42,11 +42,6 @@ HYBRID_MODELS = [
|
||||
"tiny-random/qwen3-next-moe",
|
||||
]
|
||||
|
||||
HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL = {
|
||||
"LiquidAI/LFM2-1.2B",
|
||||
"tiny-random/qwen3-next-moe",
|
||||
}
|
||||
|
||||
FULL_CUDA_GRAPH_MODELS = [
|
||||
"ai21labs/Jamba-tiny-dev",
|
||||
"Zyphra/Zamba2-1.2B-instruct",
|
||||
@@ -95,15 +90,11 @@ def test_models(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
extra_kwargs = {}
|
||||
if model in HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL:
|
||||
extra_kwargs["enable_chunked_prefill"] = True
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
**extra_kwargs,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
@@ -140,7 +131,9 @@ def test_batching(
|
||||
_set_conv_state_layout(monkeypatch, conv_state_layout)
|
||||
|
||||
for_loop_outputs = []
|
||||
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
for prompt in example_prompts:
|
||||
(single_output,) = vllm_model.generate_greedy_logprobs(
|
||||
[prompt], max_tokens, num_logprobs
|
||||
@@ -224,7 +217,7 @@ def test_mamba_cache_cg_padding(
|
||||
example_prompts.append(example_prompts[0])
|
||||
|
||||
try:
|
||||
with vllm_runner(model) as vllm_model:
|
||||
with vllm_runner(model, enable_chunked_prefill=True) as vllm_model:
|
||||
vllm_model.generate_greedy(example_prompts, max_tokens)
|
||||
except RuntimeError:
|
||||
pytest.fail(
|
||||
@@ -250,7 +243,9 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
|
||||
a single step.
|
||||
"""
|
||||
try:
|
||||
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
|
||||
except ValueError:
|
||||
pytest.fail(
|
||||
@@ -272,7 +267,9 @@ def test_state_cleanup(
|
||||
If it's not cleaned, an error would be expected.
|
||||
"""
|
||||
try:
|
||||
with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
for _ in range(10):
|
||||
vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
|
||||
except ValueError:
|
||||
@@ -294,14 +291,20 @@ def test_distributed_correctness(
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model, tensor_parallel_size=1, max_num_seqs=MAX_NUM_SEQS
|
||||
model,
|
||||
tensor_parallel_size=1,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model, tensor_parallel_size=2, max_num_seqs=MAX_NUM_SEQS
|
||||
model,
|
||||
tensor_parallel_size=2,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
@@ -340,7 +343,10 @@ def test_full_cuda_graph(
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
@@ -388,6 +394,7 @@ def test_fp32_cache_state(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
gpu_memory_utilization=0.9,
|
||||
enable_chunked_prefill=True,
|
||||
**{cache_dtype_param: "float32"},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
|
||||
@@ -135,10 +135,10 @@ class CacheConfig:
|
||||
"""The cache strategy for Mamba layers:
|
||||
|
||||
- "none": set when prefix caching is disabled.
|
||||
- "all": cache the mamba state of all tokens at position i * block_size. This is
|
||||
the default behavior (for models that support it) when prefix caching is enabled.
|
||||
- "all": cache the mamba state of all tokens at position i * block_size.
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size.
|
||||
when the token is at position i * block_size. This is the default when prefix
|
||||
caching is enabled.
|
||||
"""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
|
||||
|
||||
@@ -2587,11 +2587,7 @@ class EngineArgs:
|
||||
self, model_config: ModelConfig
|
||||
) -> None:
|
||||
default_chunked_prefill = model_config.is_chunked_prefill_supported
|
||||
# Hybrid models support prefix caching but keep it opt-in for now
|
||||
# while the feature matures.
|
||||
default_prefix_caching = (
|
||||
model_config.is_prefix_caching_supported and not model_config.is_hybrid
|
||||
)
|
||||
default_prefix_caching = model_config.is_prefix_caching_supported
|
||||
|
||||
if self.enable_chunked_prefill is None:
|
||||
self.enable_chunked_prefill = default_chunked_prefill
|
||||
|
||||
@@ -595,10 +595,8 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
if cache_config.enable_prefix_caching:
|
||||
if cache_config.mamba_cache_mode == "none":
|
||||
cache_config.mamba_cache_mode = (
|
||||
"all" if model_config.supports_mamba_prefix_caching else "align"
|
||||
)
|
||||
logger.warning(
|
||||
cache_config.mamba_cache_mode = "align"
|
||||
logger.info(
|
||||
"Mamba cache mode is set to '%s' for %s by default "
|
||||
"when prefix caching is enabled",
|
||||
cache_config.mamba_cache_mode,
|
||||
@@ -618,13 +616,6 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill, (
|
||||
"Chunked prefill is required for mamba cache mode 'align'."
|
||||
)
|
||||
logger.info(
|
||||
"Warning: Prefix caching in Mamba cache '%s' "
|
||||
"mode is currently enabled. "
|
||||
"Its support for Mamba layers is experimental. "
|
||||
"Please report any issues you may observe.",
|
||||
cache_config.mamba_cache_mode,
|
||||
)
|
||||
# By default, mamba block size will be set to max_model_len (see
|
||||
# below). When enabling prefix caching, we align mamba block size
|
||||
# to the block size as the basic granularity for prefix caching.
|
||||
|
||||
Reference in New Issue
Block a user