[Feat][Mamba2] Enable internal prefill checkpoints (#57329)
Signed-off-by: Roi Koren <roik@nvidia.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -70,6 +70,9 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
offsets: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -139,6 +142,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
_gather_checkpoint_state_indices(
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
num_decodes = attn_metadata.num_decodes
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
|
||||
checkpoint_meta = attn_metadata.checkpoint_meta
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run -- warm up SSD kernels so that autotuning
|
||||
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
|
||||
# NOTE: final output is an in-place update of out tensor
|
||||
assert preallocated_ssm_out_p is not None
|
||||
has_checkpoints = checkpoint_chunk_idx is not None
|
||||
varlen_states = mamba_chunk_scan_combined_varlen(
|
||||
hidden_states_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
cu_chunk_seqlens=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
initial_states=initial_states,
|
||||
return_intermediate_states=has_checkpoints,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
final_states = (
|
||||
varlen_states[last_chunk_indices_p]
|
||||
if has_checkpoints
|
||||
else varlen_states
|
||||
)
|
||||
if has_checkpoints:
|
||||
assert checkpoint_meta is not None
|
||||
assert query_start_loc_p is not None
|
||||
store_prefill_checkpoint(
|
||||
checkpoint_meta,
|
||||
checkpoint_chunk_idx,
|
||||
x,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
is_blackwell=self.is_blackwell,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# The block-boundary state can be exported from inside one forward
|
||||
# pass, so the scheduler need not split the prefill; every reader
|
||||
# acts on this only in align mode. The metadata builder puts a chunk
|
||||
# end exactly on the checkpoint, so any token position works.
|
||||
return replace(
|
||||
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_prefill_checkpoint_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
chunk_states_ptr,
|
||||
ssm_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
state_indices_ptr,
|
||||
chunk_idx_ptr,
|
||||
stride_x_dim,
|
||||
stride_x_token,
|
||||
stride_conv_block,
|
||||
stride_conv_dim,
|
||||
stride_conv_token,
|
||||
stride_chunk_states,
|
||||
stride_ssm_block,
|
||||
DIM: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
SSM_ROW_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
|
||||
valid = checkpoint_offset > 0
|
||||
dst = tl.load(state_indices_ptr + row).to(tl.int64)
|
||||
|
||||
conv_mask = valid & (cols < DIM * STATE_LEN)
|
||||
dim_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
token_idx = (
|
||||
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
|
||||
).to(tl.int64)
|
||||
conv = tl.load(
|
||||
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ dst * stride_conv_block
|
||||
+ dim_idx * stride_conv_dim
|
||||
+ history_idx * stride_conv_token,
|
||||
conv,
|
||||
mask=conv_mask,
|
||||
)
|
||||
|
||||
ssm_mask = valid & (cols < SSM_ROW_SIZE)
|
||||
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
|
||||
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
|
||||
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
|
||||
|
||||
|
||||
def store_prefill_checkpoint(
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
chunk_idx: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
Rows without a checkpoint (offset 0) write nothing.
|
||||
|
||||
Args:
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
chunk_idx.numel(),
|
||||
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
|
||||
)
|
||||
_store_prefill_checkpoint_kernel[grid](
|
||||
x,
|
||||
conv_state,
|
||||
chunk_states,
|
||||
ssm_state,
|
||||
query_start_loc,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
chunk_idx,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
chunk_states.stride(0),
|
||||
ssm_state.stride(0),
|
||||
DIM=dim,
|
||||
STATE_LEN=state_len,
|
||||
SSM_ROW_SIZE=ssm_row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
Reference in New Issue
Block a user