[Perf] Support internal prefill checkpoints for Mamba prefix caching, 9%~25% TTFT improvement (#52789)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wentao Ye
2026-08-22 10:32:40 -07:00
committed by GitHub
co-authored by Ziming Huang OpenAI Codex
parent 020829f66b
commit 9eb9d9d395
12 changed files with 562 additions and 48 deletions
+81
View File
@@ -21,6 +21,8 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
fused_recurrent_kda_packed_decode as fused_recurrent_kda_packed_decode_amd,
)
from vllm.models.kimi_k3.nvidia.kda import (
_flashkda_prefill,
_store_cache_checkpoints_kernel,
is_flashkda_supported,
is_fused_kda_decode_supported,
)
@@ -40,6 +42,7 @@ from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
)
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
@@ -1088,6 +1091,17 @@ def test_flashkda_correctness():
expected_states.append(final_state)
expected_out = torch.cat(expected_outputs, dim=1)
expected_state = torch.cat(expected_states).transpose(-1, -2).contiguous()
_, expected_checkpoint = naive_recurrent_kda(
q_norm[:, :16],
k_norm[:, :16],
v[:, :16],
gate[:, :16],
beta[:, :16],
initial_state=initial_state[0:1].transpose(-1, -2),
output_final_state=True,
)
assert expected_checkpoint is not None
expected_checkpoint = expected_checkpoint.transpose(-1, -2).contiguous()
actual_out = torch.empty_like(v)
actual_state = torch.empty_like(initial_state)
@@ -1115,3 +1129,70 @@ def test_flashkda_correctness():
assert_close("o", expected_out, actual_out, 0.01)
assert_close("ht", expected_state, actual_state, 0.01)
checkpoint_out = torch.empty_like(v)
checkpoint_final_state = torch.empty_like(initial_state)
checkpoint_state = torch.empty_like(initial_state)
checkpoint_offsets = torch.tensor([16, 31], dtype=torch.int32, device=DEVICE)
_flashkda_prefill(
q=q,
k=k,
v=v,
g=raw_g,
beta=beta_logits,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=initial_state,
cu_seqlens=cu_seqlens,
out=checkpoint_out,
final_state=checkpoint_final_state,
workspace=workspace,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
)
assert_close("checkpoint_o", expected_out, checkpoint_out, 0.01)
assert_close("checkpoint_ht", expected_state, checkpoint_final_state, 0.01)
assert_close("checkpoint", expected_checkpoint, checkpoint_state[:1], 0.01)
conv_state = torch.zeros(2, H * D, 3, dtype=q.dtype, device=DEVICE)
recurrent_storage = torch.zeros(2, H * D * D + 8, device=DEVICE)
recurrent_state = recurrent_storage[:, : H * D * D].view(2, H, D, D)
conv_input = q[0].flatten(1)
checkpoint_state_indices = torch.tensor(
[1, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
)
state_len = conv_state.shape[-1]
width = H * D
recurrent_row_size = checkpoint_state[0].numel()
block_size = 256
_store_cache_checkpoints_kernel[
(
checkpoint_state_indices.numel(),
(max(width * state_len, recurrent_row_size) + block_size - 1) // block_size,
)
](
conv_input,
conv_state,
checkpoint_state,
recurrent_state,
cu_seqlens,
checkpoint_offsets,
checkpoint_state_indices,
conv_input.stride(0),
conv_input.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
checkpoint_state.stride(0),
recurrent_state.stride(0),
checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
block_size,
)
torch.testing.assert_close(conv_state[1], q[0, 13:16].flatten(1).transpose(0, 1))
torch.testing.assert_close(recurrent_state[1], checkpoint_state[0])
+50
View File
@@ -87,6 +87,7 @@ def _make_builder(
device: torch.device = DEVICE,
mamba_cache_mode: str = "none",
use_recoverssm: bool = False,
num_prefill_checkpoint_blocks: int = 0,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -110,6 +111,7 @@ def _make_builder(
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
),
layer_names=["layer.0"],
vllm_config=vllm_config,
@@ -121,6 +123,54 @@ def _make_builder(
return builder
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_targets_last_aligned_boundary():
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 32], query_lens=[50, 16])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
).replace(is_prefilling=torch.tensor([True, True]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=0,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
device=device,
).build(0, common_attn_metadata)
assert actual.checkpoint is not None
torch.testing.assert_close(
actual.checkpoint.state_indices,
torch.tensor([2, NULL_BLOCK_ID], dtype=torch.int32, device=device),
)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets,
torch.tensor([48, 0], dtype=torch.int32, device=device),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_internal_checkpoint_metadata_skips_unaligned_offset():
device = torch.device("cuda")
# The checkpoint block boundary is 48, but this query starts at token 1,
# making the real checkpoint offset 47, which is not FlashKDA-aligned.
batch = BatchSpec(seq_lens=[50], query_lens=[49])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
).replace(is_prefilling=torch.tensor([True]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=0,
full_cuda_graph=False,
mamba_cache_mode="align",
num_prefill_checkpoint_blocks=1,
device=device,
).build(0, common_attn_metadata)
assert actual.checkpoint is None
@pytest.mark.parametrize(
(
"batch",