Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
+27 -2
View File
@@ -2,11 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Iterable
from math import prod
from math import lcm, prod
import torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import AttentionBackend
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
def get_prefill_checkpoint_alignment(self) -> int | None:
"""Token alignment of the checkpoint offsets this layer can export from
inside a prefill, if any; in align mode, the platform makes Mamba blocks
multiples of it.
"""
return None
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
mamba_block_size = vllm_config.cache_config.mamba_block_size
assert mamba_block_size is not None
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
def get_attn_backend(self) -> type[AttentionBackend]:
"""Get the attention backend class for this Mamba layer."""
return get_mamba_attn_backend(self.mamba_type)
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
"""LCM of the Mamba layers' prefill checkpoint alignments.
None unless every Mamba layer declares one: the scheduler takes internal
checkpoints only when every Mamba group can export them.
"""
alignment = 1
for layer in get_layers_from_vllm_config(
vllm_config,
MambaBase, # type: ignore[type-abstract]
).values():
layer_alignment = layer.get_prefill_checkpoint_alignment()
if layer_alignment is None:
return None
alignment = lcm(alignment, layer_alignment)
return alignment
+126 -10
View File
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass, replace
import torch
from vllm.config import VllmConfig
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
# backends that lay out the checkpoint before any tensor exists (Mamba2's
# SSD chunk split, FlashInfer GDN's checkpoint plan).
offsets: list[int] | None = None
query_lens: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
query_lens=query_lens,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
class MambaPrefillCheckpointExporter(ABC):
"""Export a mid-prefill checkpoint into backend-specific paged states."""
@dataclass(frozen=True)
class MambaPrefillCheckpointExporter:
"""Store each checkpoint's pre-convolution window and recurrent state.
The recurrent states are rows of a buffer the prefill kernel emitted;
``recurrent_rows`` picks each checkpoint row's state out of it, and by
default row ``i`` holds checkpoint ``i``'s.
"""
state_len: int | None = None
@abstractmethod
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*args,
**kwargs,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
recurrent_rows: torch.Tensor | None = None,
) -> None:
"""Write checkpoint state into the paged cache."""
raise NotImplementedError
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
recurrent_rows is not None,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
recurrent_rows_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
HAS_RECURRENT_ROWS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
if HAS_RECURRENT_ROWS:
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
else:
recurrent_row = seq_idx.to(tl.int64)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)
@@ -4,6 +4,7 @@ import torch
from einops import rearrange
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
VllmConfig,
get_current_vllm_config,
@@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
return weight_loader
@eager_break_during_capture
def olmo_hybrid_gdn_full_forward(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -2,7 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3-Next/Qwen3.5 model."""
import functools
import itertools
import math
import os
from dataclasses import replace
from typing import Literal
import torch
@@ -15,6 +19,7 @@ from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
CacheConfig,
VllmConfig,
get_current_vllm_config,
)
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import (
get_mamba_prefill_checkpoint_alignment,
)
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
)
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
GDNAttentionMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
from vllm.v1.worker.workspace import current_workspace_manager
# Optional ROCm AITER Triton kernels for the GDN decode path.
# Availability is checked centrally via rocm_aiter_ops; the actual function
@@ -90,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _aiter_flydsl_unsupported_reasons(
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
) -> list[str]:
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
model, or an empty list if they can."""
reasons = []
if head_k_dim != 128 or head_v_dim != 128:
reasons.append(
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
"FlyDSL requires 128/128"
)
dtype = vllm_config.model_config.dtype
if dtype != torch.bfloat16:
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
return reasons
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
@@ -108,7 +142,7 @@ def _resolve_gdn_prefill_backend(
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
@@ -136,24 +170,28 @@ def _resolve_gdn_prefill_backend(
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
reasons = _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
if reasons:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
"; ".join(reasons),
)
return backend, "triton"
return backend, "aiter_flydsl"
if backend == "auto":
from vllm.platforms.rocm import on_mi3xx
if (
on_mi3xx()
and not _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
):
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
@@ -241,6 +279,10 @@ def fi_chunk_gated_delta_rule(
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
state_checkpoints: torch.Tensor | None = None,
checkpoint_cu_starts: torch.Tensor | None = None,
checkpoint_every_n_tokens: int = 0,
use_cp: Literal["auto"] | bool = "auto",
):
from flashinfer.gdn_prefill import (
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
@@ -271,6 +313,10 @@ def fi_chunk_gated_delta_rule(
initial_state=fi_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts,
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
use_cp=use_cp,
backend="flashinfer",
)
# FlashInfer returns (output, state) when output_final_state=True,
@@ -283,6 +329,20 @@ def fi_chunk_gated_delta_rule(
return result.unsqueeze(0), None
def _checkpoint_grid(cache_config: CacheConfig) -> int:
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
alignment; the platform already keeps blocks on it.
"""
if cache_config.prefix_match_unit is not None:
return math.lcm(
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
)
assert cache_config.mamba_block_size is not None
return cache_config.mamba_block_size
@CustomOp.register("chunk_gated_delta_rule")
class ChunkGatedDeltaRule(CustomOp):
def __init__(self) -> None:
@@ -622,6 +682,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
self._prefill_kernels_warmed_up = False
# FlashInfer's prefill emits mid-prefill states, so an align-mode
# checkpoint needs no split prefill.
self._checkpoint_alignment: int | None = None
if (
vllm_config.cache_config.mamba_cache_mode == "align"
and self.gdn_prefill_backend == "flashinfer"
):
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
self._vllm_config = vllm_config
self._max_num_batched_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
state_len=self.conv_kernel_size - 1
)
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
)
@@ -666,6 +741,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_prefill_checkpoint_alignment(self) -> int | None:
return self._checkpoint_alignment
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
return spec
# FlashInfer emits states at one stride for the whole batch, the gcd of
# the offsets; keeping every offset on the prefix-cache grid keeps that
# stride a multiple of the grid, which bounds the states emitted.
return replace(
spec,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
)
@functools.cached_property
def _exports_checkpoints(self) -> bool:
# Known only once every layer is built: a Mamba layer that cannot
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
# splitting the prefill instead.
return (
self._checkpoint_alignment is not None
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
)
@functools.cached_property
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
# Offsets on the grid emit at most one state per grid step of a batch.
# Read lazily: the block size is final only after the model is built.
return (
(
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
self.num_v_heads // self.tp_size,
self.head_v_dim,
self.head_k_dim,
),
torch.float32,
)
def _fused_gdn_decode_unsupported_reason(
self, vllm_config: VllmConfig
) -> str | None:
@@ -1228,6 +1343,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
if self._prefill_kernels_warmed_up:
return
self._prefill_kernels_warmed_up = True
if self._exports_checkpoints:
# Size the workspace while profiling; it is locked afterwards.
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
device = qkv_or_qkvz.device
dtype = qkv_or_qkvz.dtype
@@ -1303,6 +1421,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
if self.gdn_prefill_backend == "flashinfer":
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
except Exception:
logger.warning(
"GDN prefill kernel warmup (T=%d) failed for "
@@ -1337,6 +1457,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
torch.accelerator.empty_cache()
def _warmup_flashinfer_prefill_variants(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
state: torch.Tensor,
) -> None:
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
FlashInfer builds separate kernels for its context-parallel route (few
sequences) and its regular route, each with and without checkpoint
output. A single sequence under ``use_cp="auto"`` takes the CP route
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
"""
checkpoint_buffers: list[torch.Tensor | None] = [None]
if self._exports_checkpoints:
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
checkpoint_buffers.append(state_checkpoints[:1])
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
for use_cp, state_checkpoints in itertools.product(
("auto", False), checkpoint_buffers
):
has_checkpoint = state_checkpoints is not None
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=state,
output_final_state=True,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
use_cp=use_cp,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
)
def _forward_core_rocm(
self,
qkvz: torch.Tensor,
@@ -1508,6 +1672,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
# 1.2: Process the remaining part
conv_input_non_spec = mixed_qkv_non_spec
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
@@ -1709,23 +1874,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
assert prefill_has_initial_state is not None
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
plan = attn_metadata.flashinfer_checkpoint_plan
if plan is None:
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
else:
# Only FlashInfer's prefill emits checkpoints.
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
(
core_attn_out_non_spec,
last_recurrent_state,
) = fi_chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=plan.cu_starts,
checkpoint_every_n_tokens=plan.every_n_tokens,
)
assert attn_metadata.checkpoint is not None
assert non_spec_query_start_loc is not None
self._checkpoint_exporter.export(
attn_metadata.checkpoint,
raw_qkv=conv_input_non_spec,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=non_spec_query_start_loc,
recurrent_rows=plan.rows,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
@@ -1,130 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
@dataclass(frozen=True)
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
state_len: int | None = None
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> None:
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
@@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention):
output[:num_actual_tokens], _ = self.out_proj(hidden)
@eager_break_during_capture
def linear_attention(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -7,6 +7,7 @@ import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
@@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode(
)
@eager_break_during_capture
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -8,6 +8,7 @@ from dataclasses import replace
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
@@ -811,10 +812,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - It will read the initial states for every sequence,
# that has "has_initial_states_p" == True,
# from "cache_indices", using "state_indices_tensor_p".
# - It updates the "conv_state" cache in positions pointed
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# - It writes the state at the sequence end to the "conv_state"
# cache in positions pointed to by "state_indices_tensor_p".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -891,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
varlen_states,
ssm_state,
query_start_loc_p,
state_len=self.conv_kernel_size - 1,
)
ssm_state[state_indices_tensor_p] = final_states
if ring_start is not None and self._updates_replayssm_trackers:
@@ -1214,6 +1214,7 @@ def share_replayssm_ring_trackers(
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
@eager_break_during_capture
def mamba_mixer2(
projected_states: torch.Tensor,
output: torch.Tensor,
@@ -238,13 +238,14 @@ class MambaStateShapeCalculator:
"""Append the physical ReplaySSM ring shapes.
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
B_cache uses the un-extended ``n_groups``.
B_cache extends ``n_groups`` for head shards as in ``mamba2_state_shape``.
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
n_groups = n_groups + cls.extra_groups_for_head_shards(n_groups, tp_world_size)
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
@@ -20,17 +20,11 @@ def _causal_conv1d_fwd_kernel( # continuous batching
w_ptr, # (dim, width)
bias_ptr,
initial_states_ptr, # conv_states_ptr
cache_indices_ptr, # (batch, n_blocks + padding) The second dimension contains
# the block indices relevant for each sequence
# plus potential 0-padding at the beginning and at the end
cache_indices_ptr, # (batch,)
has_initial_states_ptr,
query_start_loc_ptr,
batch_ptr,
token_chunk_offset_ptr,
block_idx_first_scheduled_token, # (batch,)
block_idx_last_scheduled_token, # (batch,)
initial_state_idx, # (batch,)
num_computed_tokens, # (batch,)
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
# Matrix dimensions
dim: tl.constexpr,
@@ -46,7 +40,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
stride_cache_indices: tl.constexpr,
stride_o_dim: tl.constexpr,
stride_o_token: tl.int64,
stride_block_m: tl.constexpr, # Stride block to align divided by BLOCK_M
# others
pad_slot_id: tl.constexpr,
null_block_id: tl.constexpr,
@@ -54,7 +47,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
HAS_BIAS: tl.constexpr,
KERNEL_WIDTH: tl.constexpr,
SILU_ACTIVATION: tl.constexpr,
IS_APC_ENABLED: tl.constexpr,
HAS_NULL_BLOCK: tl.constexpr,
NP2_STATELEN: tl.constexpr,
BLOCK_M: tl.constexpr,
@@ -93,40 +85,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
# find the actual sequence length
seqlen = sequence_end_index - sequence_start_index
B_size: tl.constexpr = stride_block_m * BLOCK_M
if IS_APC_ENABLED:
# Handle the case if prefix caching is enabled.
# In particular, if prefix caching is enabled, the program write additional cache states to "cache_indices_ptr"
# Get the length of the completed sequence so far and compute the offset.
current_first_index = tl.load(block_idx_first_scheduled_token + idx_seq)
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
sequence_completed_index = tl.load(num_computed_tokens + idx_seq)
# Compute the offset where the first stride_block_m-aligned first full block is
# Value in "token-space"
sequence_completed_offset_token = sequence_completed_index % B_size
seq_completed_offset = B_size - sequence_completed_offset_token
seq_end_offset = (seqlen - seq_completed_offset) % B_size
last_full_block_token_index = sequence_end_index - seq_end_offset
# If the sequence without the sequence_offset_index is stride_cache_chunk-aligned, then the last full chunk is the second-to-last one
if seq_end_offset == 0:
last_full_block_token_index = last_full_block_token_index - B_size
# Get the number of blocks to be filled for the current sequence
# If n_block_to_fill = 0, then only the state at the sequence end is stored
n_block_to_fill = current_last_index - current_first_index
# Get the index of the init block
conv_state_init_index = tl.load(initial_state_idx + idx_seq)
else:
n_block_to_fill = 0
current_last_index = 0
conv_state_init_index = 0
current_first_index = 0
last_full_block_token_index = 0
token_offset = BLOCK_M * chunk_offset
segment_len = min(BLOCK_M, seqlen - token_offset)
@@ -137,7 +95,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
# cache_idx
conv_states_input_coord = tl.load(
conv_state_indices_ptr + idx_seq * stride_cache_indices + conv_state_init_index
conv_state_indices_ptr + idx_seq * stride_cache_indices
).to(tl.int64)
if HAS_NULL_BLOCK: # noqa
@@ -224,20 +182,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
# Compute the offset where the last block should be written in the conv_states
conv_states_output_coord = tl.load(
conv_state_indices_ptr
+ idx_seq * stride_cache_indices
+ current_last_index
).to(tl.int64)
conv_states_ptrs_target = (
conv_states_ptr
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens_conv * stride_conv_state_tok
)[:, None]
conv_states_base[None, :]
+ (idx_tokens_conv * stride_conv_state_tok)[:, None]
) # [BLOCK_M, BLOCK_N]
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
@@ -346,52 +294,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
conv_states_ptrs = prior_tokens - 3 * stride_x_token # [BLOCK_N]
col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca")
# Store intermediate states aligned with stride_block_m
# The additional states are cached starting from the last stride_block_m.
# For example:
# If n_block_to_fill = 0, then only the state at the sequence end is cached and the process below is not involved.
# If n_block_to_fill > 0, then the states at the sequence end and at the n_block_to_fill-last
# stride_block_m are cached.
# For example chunk_offset = n_block_to_fill stores the state at last_full_block
if (chunk_offset - 1) < n_block_to_fill:
# Store the states at the chunk boundaries from the start of the sequence
idx_tokens_last = (
last_full_block_token_index
- (n_block_to_fill - chunk_offset) * B_size
- state_len
) + tl.arange(0, NP2_STATELEN) # [BLOCK_M]
x_ptrs = (
x_ptr
+ (idx_tokens_last * stride_x_token)[:, None]
+ (idx_feats * stride_x_dim)[None, :]
) # [BLOCK_M,BLOCK_N,]
mask_x = (idx_tokens_last >= 0)[:, None] & (idx_feats < dim)[
None, :
] # token-index # token-index # feature-index
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
# cache_idx
conv_states_output_coord = tl.load(
conv_state_indices_ptr
+ idx_seq * stride_cache_indices
+ current_first_index
+ (chunk_offset - 1)
).to(tl.int64)
conv_states_ptrs_target = (
conv_states_ptr
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens_conv * stride_conv_state_tok
)[:, None]
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
tl.store(conv_states_ptrs_target, loaded_x, mask)
if HAS_BIAS:
bias = bias_ptr + idx_feats
mask_bias = idx_feats < dim
@@ -489,11 +391,6 @@ def causal_conv1d_fn(
activation: str | None = "silu",
pad_slot_id: int = PAD_SLOT_ID,
null_block_id: int = NULL_BLOCK_ID,
block_idx_first_scheduled_token: torch.Tensor | None = None,
block_idx_last_scheduled_token: torch.Tensor | None = None,
initial_state_idx: torch.Tensor | None = None,
num_computed_tokens: torch.Tensor | None = None,
block_size_to_align=0,
metadata=None,
validate_data=False,
):
@@ -536,16 +433,6 @@ def causal_conv1d_fn(
for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
in this case, the kernel will not process entries at
indices 0 and 3
block_idx_first_scheduled_token: (batch,), dtype int32
The pointer into cache_indices, where the first cache block to be filled is located.
block_idx_last_scheduled_token: (batch,), dtype int32
The pointer into cache_indices, where the last cache block to be filled is located.
initial_state_idx: (batch,), dtype int32
The pointer into cache_indices, where the cache block containing the initial state is located.
num_computed_tokens: (batch,), dtype int32
The number of tokens already completed for each sequence
block_size_to_align: int
The block size to align the cached states to
out: same shape as `x`
"""
if isinstance(activation, bool) and activation:
@@ -631,12 +518,6 @@ def causal_conv1d_fn(
assert weight.stride(1) == 1
assert (dim, width) == weight.shape
assert is_channel_last, "Need to run in channel-last layout"
if block_size_to_align is not None and block_size_to_align > 0:
assert (block_size_to_align % BLOCK_M) == 0, (
"The mamba block size needs to be divisible by the BLOCK_M"
)
else:
block_size_to_align = BLOCK_M
if metadata is None:
@@ -720,10 +601,6 @@ def causal_conv1d_fn(
query_start_loc,
batch_ptr,
token_chunk_offset_ptr,
block_idx_first_scheduled_token,
block_idx_last_scheduled_token,
initial_state_idx,
num_computed_tokens,
out,
# Matrix dimensions
dim,
@@ -739,7 +616,6 @@ def causal_conv1d_fn(
stride_cache_indices,
stride_o_dim,
stride_o_token,
block_size_to_align // BLOCK_M,
# others
pad_slot_id,
null_block_id,
@@ -747,7 +623,6 @@ def causal_conv1d_fn(
HAS_BIAS=bias is not None,
KERNEL_WIDTH=width,
SILU_ACTIVATION=activation in ["silu", "swish"],
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
HAS_NULL_BLOCK=null_block_id is not None,
NP2_STATELEN=np2_statelen,
# launch_cooperative_grid=True
@@ -769,8 +644,6 @@ def _causal_conv1d_update_kernel(
conv_state_indices_ptr,
num_accepted_tokens_ptr,
query_start_loc_ptr, # (batch + 1)
block_idx_last_scheduled_token, # (batch,)
initial_state_idx, # (batch,)
o_ptr, # (batch, dim, seqlen)
# Matrix dimensions
batch: int,
@@ -798,7 +671,6 @@ def _causal_conv1d_update_kernel(
KERNEL_WIDTH: tl.constexpr,
SILU_ACTIVATION: tl.constexpr,
IS_VARLEN: tl.constexpr,
IS_APC_ENABLED: tl.constexpr,
IS_SPEC_DECODING: tl.constexpr,
NP2_STATELEN: tl.constexpr,
HAS_NULL_BLOCK: tl.constexpr,
@@ -818,17 +690,9 @@ def _causal_conv1d_update_kernel(
# [BLOCK_N,] elements along the feature-dimension (channel)
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
if IS_APC_ENABLED:
# Get the state from the initial_state_idx
conv_state_init = tl.load(initial_state_idx + idx_seq)
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
else:
conv_state_init = 0
current_last_index = 0
# cache_idx
conv_states_input_coord = tl.load(
conv_state_indices_ptr + idx_seq * stride_state_indices + conv_state_init
conv_state_indices_ptr + idx_seq * stride_state_indices
).to(tl.int64)
if HAS_NULL_BLOCK: # noqa
@@ -941,14 +805,9 @@ def _causal_conv1d_update_kernel(
new_conv_state = tl.where(mask, conv_state, loaded_x)
# Get the state from the initial_state_idx
# cache_idx
conv_states_offset = tl.load(
conv_state_indices_ptr + idx_seq * stride_state_indices + current_last_index
).to(tl.int64)
conv_state_ptrs_target = (
conv_state_ptr
+ (conv_states_offset * stride_conv_state_seq) # Offset from seq
+ (conv_states_input_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens * stride_conv_state_tok
@@ -1104,8 +963,6 @@ def causal_conv1d_update(
query_start_loc: torch.Tensor | None = None,
max_query_len: int = -1,
null_block_id: int = NULL_BLOCK_ID,
block_idx_last_scheduled_token: torch.Tensor | None = None,
initial_state_idx: torch.Tensor | None = None,
validate_data=False,
out: torch.Tensor | None = None,
):
@@ -1123,10 +980,6 @@ def causal_conv1d_update(
If not None, the conv_state is a larger tensor along the batch dim,
and we are selecting the batch coords specified by conv_state_indices.
Useful for a continuous batching scenario.
block_idx_last_scheduled_token: (batch,), dtype int32
The pointer into conv_state_indices, where the last cache block to be filled is located.
initial_state_idx: (batch,), dtype int32
The pointer into conv_state_indices, where the cache block containing the initial state is located.
num_accepted_tokens: (batch,), dtype int32
If not None, it indicates the number of accepted tokens for each
sequence in the batch.
@@ -1237,8 +1090,6 @@ def causal_conv1d_update(
conv_state_indices,
num_accepted_tokens,
query_start_loc,
block_idx_last_scheduled_token,
initial_state_idx,
out,
# Matrix dimensions
batch,
@@ -1266,7 +1117,6 @@ def causal_conv1d_update(
KERNEL_WIDTH=width,
SILU_ACTIVATION=activation in ["silu", "swish"],
IS_VARLEN=query_start_loc is not None,
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
IS_SPEC_DECODING=num_accepted_tokens is not None,
NP2_STATELEN=np2_statelen,
HAS_NULL_BLOCK=null_block_id is not None,
@@ -69,6 +69,8 @@ def store_prefill_checkpoint(
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
*,
state_len: int,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
@@ -78,14 +80,15 @@ def store_prefill_checkpoint(
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
state_len: ``conv_kernel_size - 1``, not the spec-widened conv width.
Readers take the first ``state_len`` slots as the newest inputs.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
@@ -4,6 +4,7 @@
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import ForwardContext, get_forward_context
@@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer):
return MambaAttentionBackendEnum.SHORT_CONV
@eager_break_during_capture
def short_conv(
hidden_states: torch.Tensor,
output: torch.Tensor,