[Core][KDA] Generalize Mamba prefill checkpoint builder and exporter (#57783)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Chauncey
2026-09-21 04:46:11 +00:00
committed by GitHub
co-authored by zjy0516
parent ff0c885494
commit 3df4ae153e
5 changed files with 314 additions and 220 deletions
+11 -31
View File
@@ -14,6 +14,10 @@ import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
from vllm.model_executor.layers.mamba.checkpoint import MambaPrefillCheckpointMetadata
from vllm.model_executor.layers.mamba.kda_checkpoint import (
FlashKDAPrefillCheckpointExporter,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_update
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
@@ -25,7 +29,6 @@ from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
from vllm.models.kimi_k3.nvidia.kda import (
_flashinfer_kda_prefill,
_flashkda_prefill,
_store_cache_checkpoints_kernel,
is_flashinfer_fused_kda_decode_supported,
is_flashinfer_recurrent_kda_prefill_supported,
is_flashkda_supported,
@@ -1498,36 +1501,13 @@ def test_flashkda_checkpoint_correctness(state_dtype: torch.dtype, tolerance: fl
checkpoint_state_indices = torch.tensor(
[1, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
)
state_len = conv_state.shape[-1]
width = H * D
recurrent_row_size = checkpoint_state[0].numel()
block_size = 256
_store_cache_checkpoints_kernel[
(
checkpoint_state_indices.numel(),
(max(width * state_len, recurrent_row_size) + block_size - 1) // block_size,
)
](
conv_input,
conv_state,
checkpoint_state,
recurrent_state,
cu_seqlens,
checkpoint_offsets,
checkpoint_state_indices,
conv_input.stride(0),
conv_input.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
checkpoint_state.stride(0),
recurrent_state.stride(0),
checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
block_size,
FlashKDAPrefillCheckpointExporter().export(
MambaPrefillCheckpointMetadata(checkpoint_offsets, checkpoint_state_indices),
raw_qkv=conv_input,
conv_state=conv_state,
recurrent_checkpoint=checkpoint_state,
recurrent_state=recurrent_state,
cu_seqlens=cu_seqlens,
)
torch.testing.assert_close(conv_state[1], q[0, 13:16].flatten(1).transpose(0, 1))
torch.testing.assert_close(recurrent_state[1], checkpoint_state[0])