Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -2,11 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
from math import lcm, prod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
"""Token alignment of the checkpoint offsets this layer can export from
|
||||
inside a prefill, if any; in align mode, the platform makes Mamba blocks
|
||||
multiples of it.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
assert mamba_block_size is not None
|
||||
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
"""Get the attention backend class for this Mamba layer."""
|
||||
return get_mamba_attn_backend(self.mamba_type)
|
||||
|
||||
|
||||
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
|
||||
"""LCM of the Mamba layers' prefill checkpoint alignments.
|
||||
|
||||
None unless every Mamba layer declares one: the scheduler takes internal
|
||||
checkpoints only when every Mamba group can export them.
|
||||
"""
|
||||
alignment = 1
|
||||
for layer in get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
MambaBase, # type: ignore[type-abstract]
|
||||
).values():
|
||||
layer_alignment = layer.get_prefill_checkpoint_alignment()
|
||||
if layer_alignment is None:
|
||||
return None
|
||||
alignment = lcm(alignment, layer_alignment)
|
||||
return alignment
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
|
||||
# backends that lay out the checkpoint before any tensor exists (Mamba2's
|
||||
# SSD chunk split, FlashInfer GDN's checkpoint plan).
|
||||
offsets: list[int] | None = None
|
||||
query_lens: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
query_lens=query_lens,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
|
||||
class MambaPrefillCheckpointExporter(ABC):
|
||||
"""Export a mid-prefill checkpoint into backend-specific paged states."""
|
||||
@dataclass(frozen=True)
|
||||
class MambaPrefillCheckpointExporter:
|
||||
"""Store each checkpoint's pre-convolution window and recurrent state.
|
||||
|
||||
The recurrent states are rows of a buffer the prefill kernel emitted;
|
||||
``recurrent_rows`` picks each checkpoint row's state out of it, and by
|
||||
default row ``i`` holds checkpoint ``i``'s.
|
||||
"""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
@abstractmethod
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*args,
|
||||
**kwargs,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
recurrent_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Write checkpoint state into the paged cache."""
|
||||
raise NotImplementedError
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
recurrent_rows is not None,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
recurrent_rows_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
HAS_RECURRENT_ROWS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
if HAS_RECURRENT_ROWS:
|
||||
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
|
||||
else:
|
||||
recurrent_row = seq_idx.to(tl.int64)
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
@@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
|
||||
return weight_loader
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def olmo_hybrid_gdn_full_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3-Next/Qwen3.5 model."""
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
@@ -15,6 +19,7 @@ from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import (
|
||||
get_mamba_prefill_checkpoint_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
|
||||
GDNAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
# Optional ROCm AITER Triton kernels for the GDN decode path.
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
@@ -90,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
|
||||
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
|
||||
def _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
|
||||
) -> list[str]:
|
||||
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
|
||||
model, or an empty list if they can."""
|
||||
reasons = []
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons.append(
|
||||
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
|
||||
"FlyDSL requires 128/128"
|
||||
)
|
||||
dtype = vllm_config.model_config.dtype
|
||||
if dtype != torch.bfloat16:
|
||||
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
|
||||
return reasons
|
||||
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
@@ -108,7 +142,7 @@ def _resolve_gdn_prefill_backend(
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
@@ -136,24 +170,28 @@ def _resolve_gdn_prefill_backend(
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons = _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
if reasons:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
"; ".join(reasons),
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
if backend == "auto":
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
if (
|
||||
on_mi3xx()
|
||||
and not _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
):
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
@@ -241,6 +279,10 @@ def fi_chunk_gated_delta_rule(
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
state_checkpoints: torch.Tensor | None = None,
|
||||
checkpoint_cu_starts: torch.Tensor | None = None,
|
||||
checkpoint_every_n_tokens: int = 0,
|
||||
use_cp: Literal["auto"] | bool = "auto",
|
||||
):
|
||||
from flashinfer.gdn_prefill import (
|
||||
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
|
||||
@@ -271,6 +313,10 @@ def fi_chunk_gated_delta_rule(
|
||||
initial_state=fi_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts,
|
||||
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
backend="flashinfer",
|
||||
)
|
||||
# FlashInfer returns (output, state) when output_final_state=True,
|
||||
@@ -283,6 +329,20 @@ def fi_chunk_gated_delta_rule(
|
||||
return result.unsqueeze(0), None
|
||||
|
||||
|
||||
def _checkpoint_grid(cache_config: CacheConfig) -> int:
|
||||
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
|
||||
|
||||
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
|
||||
alignment; the platform already keeps blocks on it.
|
||||
"""
|
||||
if cache_config.prefix_match_unit is not None:
|
||||
return math.lcm(
|
||||
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
)
|
||||
assert cache_config.mamba_block_size is not None
|
||||
return cache_config.mamba_block_size
|
||||
|
||||
|
||||
@CustomOp.register("chunk_gated_delta_rule")
|
||||
class ChunkGatedDeltaRule(CustomOp):
|
||||
def __init__(self) -> None:
|
||||
@@ -622,6 +682,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
|
||||
self._prefill_kernels_warmed_up = False
|
||||
# FlashInfer's prefill emits mid-prefill states, so an align-mode
|
||||
# checkpoint needs no split prefill.
|
||||
self._checkpoint_alignment: int | None = None
|
||||
if (
|
||||
vllm_config.cache_config.mamba_cache_mode == "align"
|
||||
and self.gdn_prefill_backend == "flashinfer"
|
||||
):
|
||||
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
self._vllm_config = vllm_config
|
||||
self._max_num_batched_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
|
||||
state_len=self.conv_kernel_size - 1
|
||||
)
|
||||
self.enable_packed_recurrent_decode = (
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
@@ -666,6 +741,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
return self._checkpoint_alignment
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# FlashInfer emits states at one stride for the whole batch, the gcd of
|
||||
# the offsets; keeping every offset on the prefix-cache grid keeps that
|
||||
# stride a multiple of the grid, which bounds the states emitted.
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _exports_checkpoints(self) -> bool:
|
||||
# Known only once every layer is built: a Mamba layer that cannot
|
||||
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
|
||||
# splitting the prefill instead.
|
||||
return (
|
||||
self._checkpoint_alignment is not None
|
||||
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
|
||||
# Offsets on the grid emit at most one state per grid step of a batch.
|
||||
# Read lazily: the block size is final only after the model is built.
|
||||
return (
|
||||
(
|
||||
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.head_v_dim,
|
||||
self.head_k_dim,
|
||||
),
|
||||
torch.float32,
|
||||
)
|
||||
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
@@ -1228,6 +1343,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
if self._prefill_kernels_warmed_up:
|
||||
return
|
||||
self._prefill_kernels_warmed_up = True
|
||||
if self._exports_checkpoints:
|
||||
# Size the workspace while profiling; it is locked afterwards.
|
||||
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
|
||||
|
||||
device = qkv_or_qkvz.device
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
@@ -1303,6 +1421,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
if self.gdn_prefill_backend == "flashinfer":
|
||||
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"GDN prefill kernel warmup (T=%d) failed for "
|
||||
@@ -1337,6 +1457,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def _warmup_flashinfer_prefill_variants(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
|
||||
|
||||
FlashInfer builds separate kernels for its context-parallel route (few
|
||||
sequences) and its regular route, each with and without checkpoint
|
||||
output. A single sequence under ``use_cp="auto"`` takes the CP route
|
||||
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
|
||||
"""
|
||||
checkpoint_buffers: list[torch.Tensor | None] = [None]
|
||||
if self._exports_checkpoints:
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
checkpoint_buffers.append(state_checkpoints[:1])
|
||||
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
|
||||
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
|
||||
for use_cp, state_checkpoints in itertools.product(
|
||||
("auto", False), checkpoint_buffers
|
||||
):
|
||||
has_checkpoint = state_checkpoints is not None
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
use_cp=use_cp,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
|
||||
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
|
||||
)
|
||||
|
||||
def _forward_core_rocm(
|
||||
self,
|
||||
qkvz: torch.Tensor,
|
||||
@@ -1508,6 +1672,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
# 1.2: Process the remaining part
|
||||
conv_input_non_spec = mixed_qkv_non_spec
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert mixed_qkv_non_spec is not None
|
||||
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
|
||||
@@ -1709,23 +1874,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
assert prefill_has_initial_state is not None
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~prefill_has_initial_state, ...] = 0
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
plan = attn_metadata.flashinfer_checkpoint_plan
|
||||
if plan is None:
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
else:
|
||||
# Only FlashInfer's prefill emits checkpoints.
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = fi_chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=plan.cu_starts,
|
||||
checkpoint_every_n_tokens=plan.every_n_tokens,
|
||||
)
|
||||
assert attn_metadata.checkpoint is not None
|
||||
assert non_spec_query_start_loc is not None
|
||||
self._checkpoint_exporter.export(
|
||||
attn_metadata.checkpoint,
|
||||
raw_qkv=conv_input_non_spec,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
recurrent_rows=plan.rows,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
|
||||
@@ -1,130 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> None:
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
@@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention):
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode(
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import replace
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
@@ -811,10 +812,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - It will read the initial states for every sequence,
|
||||
# that has "has_initial_states_p" == True,
|
||||
# from "cache_indices", using "state_indices_tensor_p".
|
||||
# - It updates the "conv_state" cache in positions pointed
|
||||
# to by "state_indices_tensor_p".
|
||||
# In particular, it will always write the state at the
|
||||
# sequence end.
|
||||
# - It writes the state at the sequence end to the "conv_state"
|
||||
# cache in positions pointed to by "state_indices_tensor_p".
|
||||
x = hidden_states_B_C_p.transpose(
|
||||
0, 1
|
||||
) # this is the form that causal-conv see
|
||||
@@ -891,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
state_len=self.conv_kernel_size - 1,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
@@ -1214,6 +1214,7 @@ def share_replayssm_ring_trackers(
|
||||
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer2(
|
||||
projected_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -238,13 +238,14 @@ class MambaStateShapeCalculator:
|
||||
"""Append the physical ReplaySSM ring shapes.
|
||||
|
||||
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
|
||||
B_cache uses the un-extended ``n_groups``.
|
||||
B_cache extends ``n_groups`` for head shards as in ``mamba2_state_shape``.
|
||||
"""
|
||||
ring_buffer_len = logical_window
|
||||
if backend == MambaBackendEnum.FLASHINFER:
|
||||
# FlashInfer keeps the live window and current verify window together.
|
||||
ring_buffer_len += 1 + num_speculative_tokens
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
n_groups = n_groups + cls.extra_groups_for_head_shards(n_groups, tp_world_size)
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
|
||||
@@ -20,17 +20,11 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
w_ptr, # (dim, width)
|
||||
bias_ptr,
|
||||
initial_states_ptr, # conv_states_ptr
|
||||
cache_indices_ptr, # (batch, n_blocks + padding) The second dimension contains
|
||||
# the block indices relevant for each sequence
|
||||
# plus potential 0-padding at the beginning and at the end
|
||||
cache_indices_ptr, # (batch,)
|
||||
has_initial_states_ptr,
|
||||
query_start_loc_ptr,
|
||||
batch_ptr,
|
||||
token_chunk_offset_ptr,
|
||||
block_idx_first_scheduled_token, # (batch,)
|
||||
block_idx_last_scheduled_token, # (batch,)
|
||||
initial_state_idx, # (batch,)
|
||||
num_computed_tokens, # (batch,)
|
||||
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
|
||||
# Matrix dimensions
|
||||
dim: tl.constexpr,
|
||||
@@ -46,7 +40,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
stride_cache_indices: tl.constexpr,
|
||||
stride_o_dim: tl.constexpr,
|
||||
stride_o_token: tl.int64,
|
||||
stride_block_m: tl.constexpr, # Stride block to align divided by BLOCK_M
|
||||
# others
|
||||
pad_slot_id: tl.constexpr,
|
||||
null_block_id: tl.constexpr,
|
||||
@@ -54,7 +47,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
HAS_BIAS: tl.constexpr,
|
||||
KERNEL_WIDTH: tl.constexpr,
|
||||
SILU_ACTIVATION: tl.constexpr,
|
||||
IS_APC_ENABLED: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
@@ -93,40 +85,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# find the actual sequence length
|
||||
seqlen = sequence_end_index - sequence_start_index
|
||||
|
||||
B_size: tl.constexpr = stride_block_m * BLOCK_M
|
||||
|
||||
if IS_APC_ENABLED:
|
||||
# Handle the case if prefix caching is enabled.
|
||||
# In particular, if prefix caching is enabled, the program write additional cache states to "cache_indices_ptr"
|
||||
|
||||
# Get the length of the completed sequence so far and compute the offset.
|
||||
current_first_index = tl.load(block_idx_first_scheduled_token + idx_seq)
|
||||
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
|
||||
sequence_completed_index = tl.load(num_computed_tokens + idx_seq)
|
||||
|
||||
# Compute the offset where the first stride_block_m-aligned first full block is
|
||||
# Value in "token-space"
|
||||
sequence_completed_offset_token = sequence_completed_index % B_size
|
||||
seq_completed_offset = B_size - sequence_completed_offset_token
|
||||
seq_end_offset = (seqlen - seq_completed_offset) % B_size
|
||||
last_full_block_token_index = sequence_end_index - seq_end_offset
|
||||
# If the sequence without the sequence_offset_index is stride_cache_chunk-aligned, then the last full chunk is the second-to-last one
|
||||
if seq_end_offset == 0:
|
||||
last_full_block_token_index = last_full_block_token_index - B_size
|
||||
|
||||
# Get the number of blocks to be filled for the current sequence
|
||||
# If n_block_to_fill = 0, then only the state at the sequence end is stored
|
||||
n_block_to_fill = current_last_index - current_first_index
|
||||
|
||||
# Get the index of the init block
|
||||
conv_state_init_index = tl.load(initial_state_idx + idx_seq)
|
||||
else:
|
||||
n_block_to_fill = 0
|
||||
current_last_index = 0
|
||||
conv_state_init_index = 0
|
||||
current_first_index = 0
|
||||
last_full_block_token_index = 0
|
||||
|
||||
token_offset = BLOCK_M * chunk_offset
|
||||
segment_len = min(BLOCK_M, seqlen - token_offset)
|
||||
|
||||
@@ -137,7 +95,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
|
||||
# cache_idx
|
||||
conv_states_input_coord = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_cache_indices + conv_state_init_index
|
||||
conv_state_indices_ptr + idx_seq * stride_cache_indices
|
||||
).to(tl.int64)
|
||||
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
@@ -224,20 +182,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
|
||||
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
|
||||
# Compute the offset where the last block should be written in the conv_states
|
||||
conv_states_output_coord = tl.load(
|
||||
conv_state_indices_ptr
|
||||
+ idx_seq * stride_cache_indices
|
||||
+ current_last_index
|
||||
).to(tl.int64)
|
||||
|
||||
conv_states_ptrs_target = (
|
||||
conv_states_ptr
|
||||
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens_conv * stride_conv_state_tok
|
||||
)[:, None]
|
||||
conv_states_base[None, :]
|
||||
+ (idx_tokens_conv * stride_conv_state_tok)[:, None]
|
||||
) # [BLOCK_M, BLOCK_N]
|
||||
|
||||
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
|
||||
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
|
||||
@@ -346,52 +294,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
conv_states_ptrs = prior_tokens - 3 * stride_x_token # [BLOCK_N]
|
||||
col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca")
|
||||
|
||||
# Store intermediate states aligned with stride_block_m
|
||||
# The additional states are cached starting from the last stride_block_m.
|
||||
# For example:
|
||||
# If n_block_to_fill = 0, then only the state at the sequence end is cached and the process below is not involved.
|
||||
# If n_block_to_fill > 0, then the states at the sequence end and at the n_block_to_fill-last
|
||||
# stride_block_m are cached.
|
||||
# For example chunk_offset = n_block_to_fill stores the state at last_full_block
|
||||
if (chunk_offset - 1) < n_block_to_fill:
|
||||
# Store the states at the chunk boundaries from the start of the sequence
|
||||
idx_tokens_last = (
|
||||
last_full_block_token_index
|
||||
- (n_block_to_fill - chunk_offset) * B_size
|
||||
- state_len
|
||||
) + tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
x_ptrs = (
|
||||
x_ptr
|
||||
+ (idx_tokens_last * stride_x_token)[:, None]
|
||||
+ (idx_feats * stride_x_dim)[None, :]
|
||||
) # [BLOCK_M,BLOCK_N,]
|
||||
|
||||
mask_x = (idx_tokens_last >= 0)[:, None] & (idx_feats < dim)[
|
||||
None, :
|
||||
] # token-index # token-index # feature-index
|
||||
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
|
||||
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
|
||||
# cache_idx
|
||||
conv_states_output_coord = tl.load(
|
||||
conv_state_indices_ptr
|
||||
+ idx_seq * stride_cache_indices
|
||||
+ current_first_index
|
||||
+ (chunk_offset - 1)
|
||||
).to(tl.int64)
|
||||
|
||||
conv_states_ptrs_target = (
|
||||
conv_states_ptr
|
||||
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens_conv * stride_conv_state_tok
|
||||
)[:, None]
|
||||
|
||||
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
|
||||
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
|
||||
tl.store(conv_states_ptrs_target, loaded_x, mask)
|
||||
|
||||
if HAS_BIAS:
|
||||
bias = bias_ptr + idx_feats
|
||||
mask_bias = idx_feats < dim
|
||||
@@ -489,11 +391,6 @@ def causal_conv1d_fn(
|
||||
activation: str | None = "silu",
|
||||
pad_slot_id: int = PAD_SLOT_ID,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
block_idx_first_scheduled_token: torch.Tensor | None = None,
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
block_size_to_align=0,
|
||||
metadata=None,
|
||||
validate_data=False,
|
||||
):
|
||||
@@ -536,16 +433,6 @@ def causal_conv1d_fn(
|
||||
for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
|
||||
in this case, the kernel will not process entries at
|
||||
indices 0 and 3
|
||||
block_idx_first_scheduled_token: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the first cache block to be filled is located.
|
||||
block_idx_last_scheduled_token: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the last cache block to be filled is located.
|
||||
initial_state_idx: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the cache block containing the initial state is located.
|
||||
num_computed_tokens: (batch,), dtype int32
|
||||
The number of tokens already completed for each sequence
|
||||
block_size_to_align: int
|
||||
The block size to align the cached states to
|
||||
out: same shape as `x`
|
||||
"""
|
||||
if isinstance(activation, bool) and activation:
|
||||
@@ -631,12 +518,6 @@ def causal_conv1d_fn(
|
||||
assert weight.stride(1) == 1
|
||||
assert (dim, width) == weight.shape
|
||||
assert is_channel_last, "Need to run in channel-last layout"
|
||||
if block_size_to_align is not None and block_size_to_align > 0:
|
||||
assert (block_size_to_align % BLOCK_M) == 0, (
|
||||
"The mamba block size needs to be divisible by the BLOCK_M"
|
||||
)
|
||||
else:
|
||||
block_size_to_align = BLOCK_M
|
||||
|
||||
if metadata is None:
|
||||
|
||||
@@ -720,10 +601,6 @@ def causal_conv1d_fn(
|
||||
query_start_loc,
|
||||
batch_ptr,
|
||||
token_chunk_offset_ptr,
|
||||
block_idx_first_scheduled_token,
|
||||
block_idx_last_scheduled_token,
|
||||
initial_state_idx,
|
||||
num_computed_tokens,
|
||||
out,
|
||||
# Matrix dimensions
|
||||
dim,
|
||||
@@ -739,7 +616,6 @@ def causal_conv1d_fn(
|
||||
stride_cache_indices,
|
||||
stride_o_dim,
|
||||
stride_o_token,
|
||||
block_size_to_align // BLOCK_M,
|
||||
# others
|
||||
pad_slot_id,
|
||||
null_block_id,
|
||||
@@ -747,7 +623,6 @@ def causal_conv1d_fn(
|
||||
HAS_BIAS=bias is not None,
|
||||
KERNEL_WIDTH=width,
|
||||
SILU_ACTIVATION=activation in ["silu", "swish"],
|
||||
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
NP2_STATELEN=np2_statelen,
|
||||
# launch_cooperative_grid=True
|
||||
@@ -769,8 +644,6 @@ def _causal_conv1d_update_kernel(
|
||||
conv_state_indices_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
query_start_loc_ptr, # (batch + 1)
|
||||
block_idx_last_scheduled_token, # (batch,)
|
||||
initial_state_idx, # (batch,)
|
||||
o_ptr, # (batch, dim, seqlen)
|
||||
# Matrix dimensions
|
||||
batch: int,
|
||||
@@ -798,7 +671,6 @@ def _causal_conv1d_update_kernel(
|
||||
KERNEL_WIDTH: tl.constexpr,
|
||||
SILU_ACTIVATION: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
IS_APC_ENABLED: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
@@ -818,17 +690,9 @@ def _causal_conv1d_update_kernel(
|
||||
# [BLOCK_N,] elements along the feature-dimension (channel)
|
||||
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
if IS_APC_ENABLED:
|
||||
# Get the state from the initial_state_idx
|
||||
conv_state_init = tl.load(initial_state_idx + idx_seq)
|
||||
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
|
||||
else:
|
||||
conv_state_init = 0
|
||||
current_last_index = 0
|
||||
|
||||
# cache_idx
|
||||
conv_states_input_coord = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices + conv_state_init
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices
|
||||
).to(tl.int64)
|
||||
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
@@ -941,14 +805,9 @@ def _causal_conv1d_update_kernel(
|
||||
|
||||
new_conv_state = tl.where(mask, conv_state, loaded_x)
|
||||
|
||||
# Get the state from the initial_state_idx
|
||||
# cache_idx
|
||||
conv_states_offset = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices + current_last_index
|
||||
).to(tl.int64)
|
||||
conv_state_ptrs_target = (
|
||||
conv_state_ptr
|
||||
+ (conv_states_offset * stride_conv_state_seq) # Offset from seq
|
||||
+ (conv_states_input_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens * stride_conv_state_tok
|
||||
@@ -1104,8 +963,6 @@ def causal_conv1d_update(
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
max_query_len: int = -1,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
@@ -1123,10 +980,6 @@ def causal_conv1d_update(
|
||||
If not None, the conv_state is a larger tensor along the batch dim,
|
||||
and we are selecting the batch coords specified by conv_state_indices.
|
||||
Useful for a continuous batching scenario.
|
||||
block_idx_last_scheduled_token: (batch,), dtype int32
|
||||
The pointer into conv_state_indices, where the last cache block to be filled is located.
|
||||
initial_state_idx: (batch,), dtype int32
|
||||
The pointer into conv_state_indices, where the cache block containing the initial state is located.
|
||||
num_accepted_tokens: (batch,), dtype int32
|
||||
If not None, it indicates the number of accepted tokens for each
|
||||
sequence in the batch.
|
||||
@@ -1237,8 +1090,6 @@ def causal_conv1d_update(
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
block_idx_last_scheduled_token,
|
||||
initial_state_idx,
|
||||
out,
|
||||
# Matrix dimensions
|
||||
batch,
|
||||
@@ -1266,7 +1117,6 @@ def causal_conv1d_update(
|
||||
KERNEL_WIDTH=width,
|
||||
SILU_ACTIVATION=activation in ["silu", "swish"],
|
||||
IS_VARLEN=query_start_loc is not None,
|
||||
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
|
||||
IS_SPEC_DECODING=num_accepted_tokens is not None,
|
||||
NP2_STATELEN=np2_statelen,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
|
||||
@@ -69,6 +69,8 @@ def store_prefill_checkpoint(
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
*,
|
||||
state_len: int,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
@@ -78,14 +80,15 @@ def store_prefill_checkpoint(
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
state_len: ``conv_kernel_size - 1``, not the spec-widened conv width.
|
||||
Readers take the first ``state_len`` slots as the newest inputs.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer):
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def short_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user