[Feat][Mamba2] Enable internal prefill checkpoints (#57329)

Signed-off-by: Roi Koren <roik@nvidia.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
roikoren755
2026-10-06 07:53:51 +00:00
committed by GitHub
co-authored by zjy0516
parent edfccc9469
commit e11962fc1c
8 changed files with 586 additions and 23 deletions
@@ -70,6 +70,9 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
offsets: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -139,6 +142,7 @@ class MambaPrefillCheckpointBuilder:
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -3,11 +3,12 @@
from collections.abc import Sequence
from dataclasses import replace
import torch
from torch import nn
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
divide,
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
selective_state_update_replayssm_output_only,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
logger = init_logger(__name__)
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
checkpoint_meta = attn_metadata.checkpoint_meta
if attn_metadata is None:
# V1 profile run -- warm up SSD kernels so that autotuning
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
# NOTE: final output is an in-place update of out tensor
assert preallocated_ssm_out_p is not None
has_checkpoints = checkpoint_chunk_idx is not None
varlen_states = mamba_chunk_scan_combined_varlen(
hidden_states_p.view(
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=has_checkpoints,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
final_states = (
varlen_states[last_chunk_indices_p]
if has_checkpoints
else varlen_states
)
if has_checkpoints:
assert checkpoint_meta is not None
assert query_start_loc_p is not None
store_prefill_checkpoint(
checkpoint_meta,
checkpoint_chunk_idx,
x,
conv_state,
varlen_states,
ssm_state,
query_start_loc_p,
)
ssm_state[state_indices_tensor_p] = final_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
is_blackwell=self.is_blackwell,
)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not isinstance(spec, MambaSpec):
return spec
# The block-boundary state can be exported from inside one forward
# pass, so the scheduler need not split the prefill; every reader
# acts on this only in align mode. The metadata builder puts a chunk
# end exactly on the checkpoint, so any token position works.
return replace(
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
)
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _store_prefill_checkpoint_kernel(
x_ptr,
conv_state_ptr,
chunk_states_ptr,
ssm_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
state_indices_ptr,
chunk_idx_ptr,
stride_x_dim,
stride_x_token,
stride_conv_block,
stride_conv_dim,
stride_conv_token,
stride_chunk_states,
stride_ssm_block,
DIM: tl.constexpr,
STATE_LEN: tl.constexpr,
SSM_ROW_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
valid = checkpoint_offset > 0
dst = tl.load(state_indices_ptr + row).to(tl.int64)
conv_mask = valid & (cols < DIM * STATE_LEN)
dim_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
token_idx = (
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
).to(tl.int64)
conv = tl.load(
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
)
tl.store(
conv_state_ptr
+ dst * stride_conv_block
+ dim_idx * stride_conv_dim
+ history_idx * stride_conv_token,
conv,
mask=conv_mask,
)
ssm_mask = valid & (cols < SSM_ROW_SIZE)
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
def store_prefill_checkpoint(
checkpoint: MambaPrefillCheckpointMetadata,
chunk_idx: torch.Tensor,
x: torch.Tensor,
conv_state: torch.Tensor,
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
Rows without a checkpoint (offset 0) write nothing.
Args:
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
chunk_idx.numel(),
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
)
_store_prefill_checkpoint_kernel[grid](
x,
conv_state,
chunk_states,
ssm_state,
query_start_loc,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
chunk_idx,
x.stride(0),
x.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
chunk_states.stride(0),
ssm_state.stride(0),
DIM=dim,
STATE_LEN=state_len,
SSM_ROW_SIZE=ssm_row_size,
BLOCK_SIZE=block_size,
)