[Perf] Support internal prefill checkpoints for Mamba prefix caching, 9%~25% TTFT improvement (#52789)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wentao Ye
2026-08-22 10:32:40 -07:00
committed by GitHub
co-authored by Ziming Huang OpenAI Codex
parent 020829f66b
commit 9eb9d9d395
12 changed files with 562 additions and 48 deletions
+5 -1
View File
@@ -815,6 +815,7 @@ class MambaSpec(KVCacheSpec):
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2
mamba_cache_mode: str = "none"
num_speculative_blocks: int = 0
num_prefill_checkpoint_blocks: int = 0
num_heads: int = 1
tokens_per_state: int = -1
@@ -843,7 +844,9 @@ class MambaSpec(KVCacheSpec):
cdiv(max_model_len, self.block_size) + self.num_speculative_blocks
) * self.page_size_bytes
elif vllm_config.cache_config.mamba_cache_mode == "align":
return self.page_size_bytes * (2 + self.num_speculative_blocks)
return self.page_size_bytes * (
2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks
)
else:
return self.page_size_bytes * (1 + self.num_speculative_blocks)
@@ -865,6 +868,7 @@ class MambaSpec(KVCacheSpec):
return all(
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == self.num_speculative_blocks
and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks
for spec in kv_cache_specs.values()
)