[Perf][GDN] Internal prefill checkpoints for the FlashInfer GDN prefill backend (#60050)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2fb482732
commit
badfcdbf91
@@ -2,11 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
from math import lcm, prod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
"""Token alignment of the checkpoint offsets this layer can export from
|
||||
inside a prefill, if any; in align mode, the platform makes Mamba blocks
|
||||
multiples of it.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
assert mamba_block_size is not None
|
||||
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
"""Get the attention backend class for this Mamba layer."""
|
||||
return get_mamba_attn_backend(self.mamba_type)
|
||||
|
||||
|
||||
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
|
||||
"""LCM of the Mamba layers' prefill checkpoint alignments.
|
||||
|
||||
None unless every Mamba layer declares one: the scheduler takes internal
|
||||
checkpoints only when every Mamba group can export them.
|
||||
"""
|
||||
alignment = 1
|
||||
for layer in get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
MambaBase, # type: ignore[type-abstract]
|
||||
).values():
|
||||
layer_alignment = layer.get_prefill_checkpoint_alignment()
|
||||
if layer_alignment is None:
|
||||
return None
|
||||
alignment = lcm(alignment, layer_alignment)
|
||||
return alignment
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
|
||||
# backends that lay out the checkpoint before any tensor exists (Mamba2's
|
||||
# SSD chunk split, FlashInfer GDN's checkpoint plan).
|
||||
offsets: list[int] | None = None
|
||||
query_lens: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
query_lens=query_lens,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
|
||||
class MambaPrefillCheckpointExporter(ABC):
|
||||
"""Export a mid-prefill checkpoint into backend-specific paged states."""
|
||||
@dataclass(frozen=True)
|
||||
class MambaPrefillCheckpointExporter:
|
||||
"""Store each checkpoint's pre-convolution window and recurrent state.
|
||||
|
||||
The recurrent states are rows of a buffer the prefill kernel emitted;
|
||||
``recurrent_rows`` picks each checkpoint row's state out of it, and by
|
||||
default row ``i`` holds checkpoint ``i``'s.
|
||||
"""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
@abstractmethod
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*args,
|
||||
**kwargs,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
recurrent_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Write checkpoint state into the paged cache."""
|
||||
raise NotImplementedError
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
recurrent_rows is not None,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
recurrent_rows_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
HAS_RECURRENT_ROWS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
if HAS_RECURRENT_ROWS:
|
||||
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
|
||||
else:
|
||||
recurrent_row = seq_idx.to(tl.int64)
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3-Next/Qwen3.5 model."""
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
@@ -15,6 +19,7 @@ from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import (
|
||||
get_mamba_prefill_checkpoint_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
|
||||
GDNAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
# Optional ROCm AITER Triton kernels for the GDN decode path.
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
@@ -241,6 +258,10 @@ def fi_chunk_gated_delta_rule(
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
state_checkpoints: torch.Tensor | None = None,
|
||||
checkpoint_cu_starts: torch.Tensor | None = None,
|
||||
checkpoint_every_n_tokens: int = 0,
|
||||
use_cp: Literal["auto"] | bool = "auto",
|
||||
):
|
||||
from flashinfer.gdn_prefill import (
|
||||
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
|
||||
@@ -271,6 +292,10 @@ def fi_chunk_gated_delta_rule(
|
||||
initial_state=fi_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts,
|
||||
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
backend="flashinfer",
|
||||
)
|
||||
# FlashInfer returns (output, state) when output_final_state=True,
|
||||
@@ -283,6 +308,20 @@ def fi_chunk_gated_delta_rule(
|
||||
return result.unsqueeze(0), None
|
||||
|
||||
|
||||
def _checkpoint_grid(cache_config: CacheConfig) -> int:
|
||||
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
|
||||
|
||||
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
|
||||
alignment; the platform already keeps blocks on it.
|
||||
"""
|
||||
if cache_config.prefix_match_unit is not None:
|
||||
return math.lcm(
|
||||
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
)
|
||||
assert cache_config.mamba_block_size is not None
|
||||
return cache_config.mamba_block_size
|
||||
|
||||
|
||||
@CustomOp.register("chunk_gated_delta_rule")
|
||||
class ChunkGatedDeltaRule(CustomOp):
|
||||
def __init__(self) -> None:
|
||||
@@ -591,6 +630,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
|
||||
self._prefill_kernels_warmed_up = False
|
||||
# FlashInfer's prefill emits mid-prefill states, so an align-mode
|
||||
# checkpoint needs no split prefill.
|
||||
self._checkpoint_alignment: int | None = None
|
||||
if (
|
||||
vllm_config.cache_config.mamba_cache_mode == "align"
|
||||
and self.gdn_prefill_backend == "flashinfer"
|
||||
):
|
||||
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
self._vllm_config = vllm_config
|
||||
self._max_num_batched_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
|
||||
state_len=self.conv_kernel_size - 1
|
||||
)
|
||||
self.enable_packed_recurrent_decode = (
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
@@ -624,6 +678,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
return self._checkpoint_alignment
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# FlashInfer emits states at one stride for the whole batch, the gcd of
|
||||
# the offsets; keeping every offset on the prefix-cache grid keeps that
|
||||
# stride a multiple of the grid, which bounds the states emitted.
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _exports_checkpoints(self) -> bool:
|
||||
# Known only once every layer is built: a Mamba layer that cannot
|
||||
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
|
||||
# splitting the prefill instead.
|
||||
return (
|
||||
self._checkpoint_alignment is not None
|
||||
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
|
||||
# Offsets on the grid emit at most one state per grid step of a batch.
|
||||
# Read lazily: the block size is final only after the model is built.
|
||||
return (
|
||||
(
|
||||
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.head_v_dim,
|
||||
self.head_k_dim,
|
||||
),
|
||||
torch.float32,
|
||||
)
|
||||
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
@@ -1167,6 +1261,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
if self._prefill_kernels_warmed_up:
|
||||
return
|
||||
self._prefill_kernels_warmed_up = True
|
||||
if self._exports_checkpoints:
|
||||
# Size the workspace while profiling; it is locked afterwards.
|
||||
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
|
||||
|
||||
device = qkv_or_qkvz.device
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
@@ -1242,6 +1339,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
if self.gdn_prefill_backend == "flashinfer":
|
||||
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"GDN prefill kernel warmup (T=%d) failed for "
|
||||
@@ -1276,6 +1375,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def _warmup_flashinfer_prefill_variants(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
|
||||
|
||||
FlashInfer builds separate kernels for its context-parallel route (few
|
||||
sequences) and its regular route, each with and without checkpoint
|
||||
output. A single sequence under ``use_cp="auto"`` takes the CP route
|
||||
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
|
||||
"""
|
||||
checkpoint_buffers: list[torch.Tensor | None] = [None]
|
||||
if self._exports_checkpoints:
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
checkpoint_buffers.append(state_checkpoints[:1])
|
||||
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
|
||||
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
|
||||
for use_cp, state_checkpoints in itertools.product(
|
||||
("auto", False), checkpoint_buffers
|
||||
):
|
||||
has_checkpoint = state_checkpoints is not None
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
use_cp=use_cp,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
|
||||
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
|
||||
)
|
||||
|
||||
def _forward_core_rocm(
|
||||
self,
|
||||
qkvz: torch.Tensor,
|
||||
@@ -1443,6 +1586,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
# 1.2: Process the remaining part
|
||||
conv_input_non_spec = mixed_qkv_non_spec
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert mixed_qkv_non_spec is not None
|
||||
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
|
||||
@@ -1598,23 +1742,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
assert prefill_has_initial_state is not None
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~prefill_has_initial_state, ...] = 0
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
plan = attn_metadata.flashinfer_checkpoint_plan
|
||||
if plan is None:
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
else:
|
||||
# Only FlashInfer's prefill emits checkpoints.
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = fi_chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=plan.cu_starts,
|
||||
checkpoint_every_n_tokens=plan.every_n_tokens,
|
||||
)
|
||||
assert attn_metadata.checkpoint is not None
|
||||
assert non_spec_query_start_loc is not None
|
||||
self._checkpoint_exporter.export(
|
||||
attn_metadata.checkpoint,
|
||||
raw_qkv=conv_input_non_spec,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
recurrent_rows=plan.rows,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
|
||||
@@ -1,130 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> None:
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user