[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,646 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import TYPE_CHECKING
import torch
from einops import rearrange
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.distributed import divide, get_tensor_model_parallel_rank
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
sharded_weight_loader,
)
from vllm.model_executor.parameter import BasevLLMParameter
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from ...linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from ..mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
from ..ops.gather_initial_states import gather_initial_states
# Empirical lower bound for the KDA gate to avoid numerical underflow.
_KDA_GATE_LOGBOUND_MIN = -5.0
def a_log_weight_loader(
shard_axis: int,
) -> Callable[[torch.Tensor, torch.Tensor], None]:
"""Load KDA A_log stored as either old 4D or current 1D weights."""
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
tp_rank = get_tensor_model_parallel_rank()
shard_size = param.data.shape[shard_axis]
start_idx = tp_rank * shard_size
if loaded_weight.dim() == 4:
assert loaded_weight.shape[:2] == (1, 1), (
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
)
assert loaded_weight.shape[-1] == 1, (
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
)
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
return default_weight_loader(param, loaded_weight)
return loader
def _make_fused_conv1d_weight_loader(
dims: list[int],
tp_size: int,
tp_rank: int,
) -> Callable[..., None]:
sharded_dims = [dim // tp_size for dim in dims]
def weight_loader(
param: torch.Tensor,
loaded_weight: torch.Tensor,
loaded_shard_id: int,
) -> None:
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
shard_size = sharded_dims[loaded_shard_id]
source_start = tp_rank * shard_size
target_start = sum(sharded_dims[:loaded_shard_id])
loaded_shard = loaded_weight[source_start : source_start + shard_size]
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
return weight_loader
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
"""Merged projection with one output replicated across TP ranks.
The replicated shard is represented as ``size * tp_size`` so the merged
parameter reserves ``size`` local rows on every rank. Loading that shard
from rank zero then gives every rank the complete checkpoint weight.
"""
def __init__(
self,
input_size: int,
output_sizes: list[int],
replicated_shard_id: int,
tp_size: int,
**kwargs,
) -> None:
self.replicated_shard_id = replicated_shard_id
output_sizes = output_sizes.copy()
output_sizes[replicated_shard_id] *= tp_size
super().__init__(input_size, output_sizes, **kwargs)
def weight_loader(
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
def weight_loader_v2(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
if loaded_shard_id == self.replicated_shard_id:
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
@PluggableLayer.register("kimi_gated_delta_net_attention")
class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
def get_state_dtype(
self,
) -> tuple[torch.dtype, torch.dtype]:
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
return MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
)
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.kda_state_shape(
self.tp_size,
self.num_heads,
self.head_dim,
conv_kernel_size=self.conv_size,
num_spec=self.num_spec,
)
def __init__(
self,
config: KimiLinearConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix)
kda_config = config.linear_attn_config # type: ignore[attr-defined]
assert kda_config is not None, "linear_attn_config must be set"
self.head_dim = kda_config["head_dim"]
self.num_heads = kda_config["num_heads"]
assert self.num_heads % self.tp_size == 0
self.local_num_heads = divide(self.num_heads, self.tp_size)
self.projection_size = self.head_dim * self.num_heads
self.local_projection_size = divide(self.projection_size, self.tp_size)
self.conv_size = kda_config["short_conv_kernel_size"]
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
if self.use_full_rank_gate:
# Keep f_a before the narrow beta shard, then pad each TP-local row
# to select the aligned BF16 GEMM path. The padding also avoids an
# Inductor correctness issue seen with the row-strided G view.
qkvg_output_sizes = [self.projection_size] * 4
in_proj_output_sizes = qkvg_output_sizes + [
self.head_dim,
self.num_heads,
]
local_output_size = (
4 * self.local_projection_size + self.head_dim + self.local_num_heads
)
self.in_proj_padding = -local_output_size % 16
if self.in_proj_padding:
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
else:
in_proj_output_sizes = [self.projection_size] * 3 + [
self.num_heads,
self.head_dim,
]
self.in_proj_padding = 0
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
self.hidden_size,
in_proj_output_sizes,
replicated_shard_id=4,
tp_size=self.tp_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvgfab",
)
if self.in_proj_padding:
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
self.f_b_proj = ColumnParallelLinear(
self.head_dim,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.f_b_proj",
)
self.dt_bias = nn.Parameter(
torch.empty(self.local_projection_size, dtype=torch.float32)
)
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# One packed parameter and cache let decode run a single conv update.
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
self.conv1d = ColumnParallelLinear(
input_size=self.conv_size,
output_size=3 * self.projection_size,
bias=False,
params_dtype=torch.float32,
prefix=f"{prefix}.conv1d",
)
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
delattr(self.conv1d.weight, "weight_loader")
set_weight_attrs(
self.conv1d.weight,
{
"weight_loader": _make_fused_conv1d_weight_loader(
[self.projection_size] * 3,
self.tp_size,
self.tp_rank,
)
},
)
self.A_log = nn.Parameter(
torch.empty(self.local_num_heads, dtype=torch.float32)
)
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
if self.gate_lower_bound is not None:
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
"KDA gate lower bound must be in "
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
f"Got {self.gate_lower_bound}."
)
self.use_safe_gate = self.gate_lower_bound is not None
additional_config = vllm_config.additional_config
backend = (
additional_config.get("kda_prefill_backend", "auto")
if isinstance(additional_config, dict)
else "auto"
)
backend = "triton" if backend == "auto" else backend
assert backend == "triton", (
"The shared Kimi GDN layer only supports the Triton KDA "
f"prefill backend, got {backend!r}."
)
if not self.use_full_rank_gate:
self.g_a_proj = ReplicatedLinear(
self.hidden_size,
self.head_dim,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_a_proj",
)
self.g_b_proj = ColumnParallelLinear(
self.head_dim,
self.projection_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_b_proj",
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
self.o_proj = RowParallelLinear(
self.projection_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def rearrange_mixed_qkv(
self, mixed_qkv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
seq_len = mixed_qkv.shape[0]
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
# Materialize all three row-strided inputs with one token-major to
# QKV-major permutation. Each unbound tensor is then contiguous.
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.size(0)
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
if self.use_full_rank_gate:
split_sizes = [
3 * self.local_projection_size,
self.local_projection_size,
self.head_dim,
self.local_num_heads,
]
if self.in_proj_padding:
split_sizes.append(self.in_proj_padding)
projected = projected_qkvgfab.split(split_sizes, dim=-1)
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
else:
mixed_qkv, beta, f_a = projected_qkvgfab.split(
[
3 * self.local_projection_size,
self.local_num_heads,
self.head_dim,
],
dim=-1,
)
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
g1 = self.f_b_proj(f_a)[0]
beta = beta.unsqueeze(0)
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._forward(
mixed_qkv=mixed_qkv,
g1=g1,
g2=g2,
beta=beta,
core_attn_out=core_attn_out,
)
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
return self.o_proj(core_attn_out)[0]
@eager_break_during_capture
def _forward(
self,
mixed_qkv: torch.Tensor,
g1: torch.Tensor,
g2: torch.Tensor,
beta: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
return
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
# These copies may have different signatures for Kimi-K3, but they agree
# on the arguments used here.
if TYPE_CHECKING:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
elif current_platform.is_rocm():
from vllm.models.kimi_k3.amd.ops.third_party.kda import ( # type: ignore[assignment]
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
else:
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
if attn_metadata_narrowed is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
m = attn_metadata_narrowed
has_initial_state = m.has_initial_state
non_spec_query_start_loc = m.non_spec_query_start_loc
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
spec_sequence_masks = m.spec_sequence_masks
spec_token_indx = m.spec_token_indx
non_spec_token_indx = m.non_spec_token_indx
spec_state_indices_tensor = m.spec_state_indices_tensor
spec_query_start_loc = m.spec_query_start_loc
num_accepted_tokens = m.num_accepted_tokens
num_actual_tokens = m.num_actual_tokens
mixed_qkv = mixed_qkv[:num_actual_tokens]
g1 = g1[:, :num_actual_tokens]
beta = beta[:, :num_actual_tokens]
constant_caches = self.kv_cache
conv_state, recurrent_state = constant_caches
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
self.local_projection_size, dim=0
)
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
self.local_projection_size, dim=-2
)
# Split tokens into the multi-query spec-decode part and the remaining
# (prefill / plain decode) part.
if spec_sequence_masks is not None:
if m.num_prefills == 0 and m.num_decodes == 0:
mixed_qkv_spec = mixed_qkv
g1_spec, beta_spec = g1, beta
mixed_qkv_ns = g1_ns = beta_ns = None
else:
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
g1_spec = g1.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
g1_ns = g1.index_select(1, non_spec_token_indx)
beta_ns = beta.index_select(1, non_spec_token_indx)
else:
mixed_qkv_spec = g1_spec = beta_spec = None
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
# ---------- spec-decode multi-query path ----------
core_attn_out_spec = None
if spec_sequence_masks is not None:
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
spec_max_query_len = spec_state_indices_tensor.size(-1)
# Sibling beta and, for full-rank gates, output-gate views remain
# live, so write the convolution output separately.
spec_conv_out = torch.empty(
mixed_qkv_spec.shape,
dtype=mixed_qkv_spec.dtype,
device=mixed_qkv_spec.device,
)
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=spec_conv_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=spec_max_query_len,
validate_data=False,
out=spec_conv_out,
)
q_spec, k_spec, v_spec = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
)
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
# Spec-only batches write directly into core_attn_out.
spec_out = (
core_attn_out[:, : q_spec.shape[1]]
if m.num_prefills == 0 and m.num_decodes == 0
else None
)
core_attn_out_spec, _ = fused_recurrent_kda(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
cu_seqlens=spec_cu_seqlens,
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
)
# ---------- non-spec path (prefill or plain decode) ----------
core_attn_out_non_spec = None
if mixed_qkv_ns is not None:
assert g1_ns is not None and beta_ns is not None
if m.num_prefills > 0:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
# Packed prefill conv would require copying V solely to make
# it dense for KDA. Separate calls accept the strided inputs
# and produce dense Q/K/V without that extra traffic.
# TODO: Use packed conv once every KDA prefill backend accepts
# row-strided Q/K/V directly.
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=m,
).transpose(0, 1)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
q_ns, k_ns, v_ns = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in (q_ns, k_ns, v_ns)
)
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
initial_state = gather_initial_states(
recurrent_state,
non_spec_state_indices_tensor,
has_initial_state,
)
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_with_fused_gate(
q=q_ns,
k=k_ns,
v=v_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
g_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=non_spec_query_start_loc,
)
# Init cache
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
else:
# pure-decode non-spec batch
assert non_spec_state_indices_tensor is not None
decode_conv_indices = non_spec_state_indices_tensor[
: mixed_qkv_ns.size(0)
]
# Sibling beta and, for full-rank gates, output-gate views
# remain live, so write the conv output separately.
packed_conv_out = torch.empty(
mixed_qkv_ns.shape,
dtype=mixed_qkv_ns.dtype,
device=mixed_qkv_ns.device,
)
mixed_qkv_ns = causal_conv1d_update(
mixed_qkv_ns,
conv_state,
conv_weights,
self.conv1d.bias,
activation="silu",
conv_state_indices=decode_conv_indices,
validate_data=True,
out=packed_conv_out,
)
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv_ns,
raw_g=g1_ns,
raw_beta=beta_ns,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
state_indices=decode_conv_indices,
)
# ---------- merge spec and non-spec outputs ----------
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
# Mixed batches require indexed placement in the original order.
merged = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_spec.dtype,
device=core_attn_out_spec.device,
)
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
else:
assert core_attn_out_spec is not None
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
@@ -1,626 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from einops import rearrange
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import (
divide,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNormGated
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.model_loader.weight_utils import (
sharded_weight_loader,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
chunk_gated_delta_rule,
fused_recurrent_gated_delta_rule,
)
from vllm.triton_utils import tl, triton
from vllm.triton_utils.allocation import set_triton_allocator
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
This implements the linear attention mechanism that replaces sliding window
attention in the hybrid architecture.
"""
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.gated_delta_net_state_shape(
self.tp_size,
self.num_k_heads,
self.num_v_heads,
self.head_k_dim,
self.head_v_dim,
self.conv_kernel_size,
self.num_spec,
)
def __init__(
self,
config,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix=prefix)
assert getattr(config, "linear_use_gate", True), (
"OlmoHybridGatedDeltaNet requires linear_use_gate=True"
)
self.num_k_heads = config.linear_num_key_heads
self.num_v_heads = config.linear_num_value_heads
self.head_k_dim = config.linear_key_head_dim
self.head_v_dim = config.linear_value_head_dim
self.conv_kernel_size = config.linear_conv_kernel_dim
self.key_dim = self.head_k_dim * self.num_k_heads
self.value_dim = self.head_v_dim * self.num_v_heads
self.allow_neg_eigval = getattr(config, "linear_allow_neg_eigval", False)
# Fused QKVG projection: 1 matmul instead of 4
self.in_proj_qkvg = MergedColumnParallelLinear(
input_size=self.hidden_size,
output_sizes=[self.key_dim, self.key_dim, self.value_dim, self.value_dim],
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvg",
)
# Separate B and A projections to preserve numerical precision.
# Fusing these into one matmul changes FP accumulation order for the
# gating scalars, which compounds through the GDN recurrent state.
self.b_proj = ColumnParallelLinear(
input_size=self.hidden_size,
output_size=self.num_v_heads,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.b_proj",
)
self.a_proj = ColumnParallelLinear(
input_size=self.hidden_size,
output_size=self.num_v_heads,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.a_proj",
)
# Fused conv1d: single parameter instead of 3
self.conv_dim = self.key_dim * 2 + self.value_dim
self.conv1d = ColumnParallelLinear(
input_size=self.conv_kernel_size,
output_size=self.conv_dim,
bias=False,
prefix=f"{prefix}.conv1d",
)
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
delattr(self.conv1d.weight, "weight_loader")
set_weight_attrs(
self.conv1d.weight,
{
"weight_loader": _make_fused_conv1d_weight_loader(
[self.key_dim, self.key_dim, self.value_dim],
self.tp_size,
self.tp_rank,
)
},
)
self.dt_bias = nn.Parameter(
torch.ones(self.num_v_heads // self.tp_size),
)
self.A_log = nn.Parameter(
torch.empty(
divide(self.num_v_heads, self.tp_size),
)
)
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# use eps=1e-5 to match FLA's FusedRMSNormGated
self.o_norm = RMSNormGated(
self.head_v_dim,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=current_platform.current_device(),
dtype=config.torch_dtype if hasattr(config, "torch_dtype") else None,
)
self.o_proj = RowParallelLinear(
self.value_dim,
self.hidden_size,
bias=False,
input_is_parallel=True,
quant_config=self.quant_config,
prefix=f"{prefix}.o_proj",
)
# FLA triton kernels need a PyTorch-backed allocator for scratch
# memory (required by triton >= 3.x autotuner). Set once at init.
set_triton_allocator(current_platform.current_device())
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def rearrange_mixed_qkv(self, mixed_qkv):
if mixed_qkv is None:
return None, None, None
query, key, value = torch.split(
mixed_qkv,
[
self.key_dim // self.tp_size,
self.key_dim // self.tp_size,
self.value_dim // self.tp_size,
],
dim=-1,
)
num_k_heads = self.num_k_heads // self.tp_size
num_v_heads = self.num_v_heads // self.tp_size
query = rearrange(query, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
key = rearrange(key, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
value = rearrange(value, "l (h d) -> 1 l h d", h=num_v_heads, d=self.head_v_dim)
# GQA expansion if needed
if num_v_heads > num_k_heads:
expand_ratio = num_v_heads // num_k_heads
query = query.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
query = query.reshape(1, query.shape[1], num_v_heads, self.head_k_dim)
key = key.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
key = key.reshape(1, key.shape[1], num_v_heads, self.head_k_dim)
return query.contiguous(), key.contiguous(), value.contiguous()
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
# NOTE: We wrap the ENTIRE linear attention forward (projections +
# core recurrence + output norm + output projection) in a single
# custom op, rather than just wrapping the recurrent core like
# other GDN models (e.g. Qwen3Next) do.
#
# Why: torch.compile with inductor generates fused kernels for
# matmuls and pointwise ops. These fused kernels can differ in
# floating-point accumulation order from eager-mode cuBLAS,
# introducing small numerical differences (~1e-7 per op). For
# standard transformer attention this is harmless because each
# position is computed independently. But for the GDN recurrent
# state, these tiny input differences compound at every timestep
# across the full sequence length, causing severe logprob
# divergence (e.g. ~15% top-1 agreement with eager baseline).
#
# By making the full forward opaque to inductor, the projections
# and output norm run with eager-mode kernels (cuBLAS, triton),
# preserving numerical consistency. The tradeoff is reduced
# compilation speedup (~1.5x vs ~3x), but logprob agreement
# improves from ~15% to ~83% top-1 vs eager.
#
# The remaining ~17% divergence comes from inductor compiling
# the MLP and transformer attention layers that are NOT wrapped
# in custom ops -- their small precision differences propagate
# as inputs to the GDN layers from outside.
torch.ops.vllm.olmo_hybrid_gdn_full_forward(
hidden_states,
output,
self.prefix,
)
def _full_forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
num_tokens = hidden_states.size(0)
# ============================================================
# Part 1: Input Projection (2 fused matmuls instead of 6)
# ============================================================
projected_qkvg, _ = self.in_proj_qkvg(hidden_states)
conv_dim_sharded = (self.key_dim * 2 + self.value_dim) // self.tp_size
mixed_qkv = projected_qkvg[..., :conv_dim_sharded]
gate = projected_qkvg[..., conv_dim_sharded:]
b, _ = self.b_proj(hidden_states)
a, _ = self.a_proj(hidden_states)
# ============================================================
# Part 2: Core Attention
# ============================================================
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
self._forward_core(
mixed_qkv=mixed_qkv,
b=b,
a=a,
core_attn_out=core_attn_out,
)
# ============================================================
# Part 3: Output Projection
# ============================================================
gate = gate.view(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim)
core_attn_out_flat = core_attn_out.reshape(-1, core_attn_out.shape[-1])
gate_flat = gate.reshape(-1, gate.shape[-1])
core_attn_out_normed = self.o_norm(core_attn_out_flat, gate_flat)
core_attn_out = core_attn_out_normed.view(
num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim
)
core_attn_out = rearrange(core_attn_out, "l h d -> l (h d)")
output[:num_tokens], _ = self.o_proj(core_attn_out)
def _forward_core(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
):
"""Core attention computation (called by custom op)."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
# V1 profile run
return
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
if attn_metadata is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata, GDNAttentionMetadata)
has_initial_state = attn_metadata.has_initial_state
spec_query_start_loc = attn_metadata.spec_query_start_loc
non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc
spec_sequence_masks = attn_metadata.spec_sequence_masks
spec_token_indx = attn_metadata.spec_token_indx
non_spec_token_indx = attn_metadata.non_spec_token_indx
spec_state_indices_tensor = attn_metadata.spec_state_indices_tensor
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor
self_kv_cache = self.kv_cache
# conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a transpose.
conv_state = (
self_kv_cache[0]
if is_conv_state_dim_first()
else self_kv_cache[0].transpose(-1, -2)
)
ssm_state = self_kv_cache[1]
num_actual_tokens = attn_metadata.num_actual_tokens
num_accepted_tokens = attn_metadata.num_accepted_tokens
mixed_qkv = mixed_qkv[:num_actual_tokens]
b = b[:num_actual_tokens]
a = a[:num_actual_tokens]
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
if spec_sequence_masks is not None:
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
mixed_qkv_spec = mixed_qkv
mixed_qkv_non_spec = None
else:
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
mixed_qkv_non_spec = mixed_qkv.index_select(0, non_spec_token_indx)
else:
mixed_qkv_spec = None
mixed_qkv_non_spec = mixed_qkv
if spec_sequence_masks is not None:
assert spec_query_start_loc is not None
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
conv_state,
conv_weights,
None, # no bias
self.activation,
conv_state_indices=spec_state_indices_tensor[:, 0][
: attn_metadata.num_spec_decodes
],
num_accepted_tokens=num_accepted_tokens,
query_start_loc=spec_query_start_loc,
max_query_len=spec_state_indices_tensor.size(-1),
validate_data=False,
)
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
mixed_qkv_non_spec = causal_conv1d_fn(
mixed_qkv_non_spec_T,
conv_weights,
None,
activation=self.activation,
conv_states=conv_state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=attn_metadata,
).transpose(0, 1)
elif attn_metadata.num_decodes > 0:
assert non_spec_state_indices_tensor is not None
mixed_qkv_non_spec = causal_conv1d_update(
mixed_qkv_non_spec,
conv_state,
conv_weights,
None,
self.activation,
conv_state_indices=non_spec_state_indices_tensor[
: attn_metadata.num_decodes
],
validate_data=True,
)
else:
mixed_qkv_non_spec = None
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
mixed_qkv_non_spec
)
g, beta = fused_olmo_hybrid_gdn_gating(
self.A_log, a, b, self.dt_bias, self.allow_neg_eigval
)
if spec_sequence_masks is not None:
assert spec_token_indx is not None
assert non_spec_token_indx is not None
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
g_spec = g
beta_spec = beta
g_non_spec = None
beta_non_spec = None
else:
g_spec = g.index_select(1, spec_token_indx)
beta_spec = beta.index_select(1, spec_token_indx)
g_non_spec = g.index_select(1, non_spec_token_indx)
beta_non_spec = beta.index_select(1, non_spec_token_indx)
else:
g_spec = None
beta_spec = None
g_non_spec = g
beta_non_spec = beta
if spec_sequence_masks is not None:
assert spec_query_start_loc is not None
assert spec_state_indices_tensor is not None
assert num_accepted_tokens is not None
core_attn_out_spec, last_recurrent_state = fused_recurrent_gated_delta_rule(
q=query_spec,
k=key_spec,
v=value_spec,
g=g_spec,
beta=beta_spec,
initial_state=ssm_state,
inplace_final_state=True,
cu_seqlens=spec_query_start_loc[: attn_metadata.num_spec_decodes + 1],
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
use_qk_l2norm_in_kernel=True,
)
else:
core_attn_out_spec, last_recurrent_state = None, None
if attn_metadata.num_prefills > 0:
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
assert non_spec_query_start_loc is not None
initial_state = ssm_state[non_spec_state_indices_tensor].contiguous()
initial_state[~has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=non_spec_query_start_loc,
use_qk_l2norm_in_kernel=True,
)
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
ssm_state.dtype
)
elif attn_metadata.num_decodes > 0:
assert non_spec_query_start_loc is not None
assert non_spec_state_indices_tensor is not None
core_attn_out_non_spec, last_recurrent_state = (
fused_recurrent_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=ssm_state,
inplace_final_state=True,
cu_seqlens=non_spec_query_start_loc[
: attn_metadata.num_decodes + 1
],
ssm_state_indices=non_spec_state_indices_tensor,
use_qk_l2norm_in_kernel=True,
)
)
else:
core_attn_out_non_spec, last_recurrent_state = None, None
if spec_sequence_masks is not None and core_attn_out_non_spec is not None:
merged_out = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_non_spec.dtype,
device=core_attn_out_non_spec.device,
)
merged_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[:num_actual_tokens] = merged_out.squeeze(0)
elif spec_sequence_masks is not None:
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
else:
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
"""Weight loader for loading separate HF conv weights into a fused conv1d.
dims: list of original (un-sharded) dims per section,
e.g. [key_dim, key_dim, value_dim]
"""
sharded_dims = [d // tp_size for d in dims]
def weight_loader(param, loaded_weight, loaded_shard_id=None):
if loaded_weight.dim() == 2:
loaded_weight = loaded_weight.unsqueeze(1)
dim = dims[loaded_shard_id]
shard_size = dim // tp_size
tp_start = tp_rank * shard_size
sharded_weight = loaded_weight[tp_start : tp_start + shard_size]
offset = sum(sharded_dims[:loaded_shard_id])
param.data[offset : offset + shard_size].copy_(sharded_weight)
return weight_loader
@eager_break_during_capture
def olmo_hybrid_gdn_full_forward(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Full linear attention forward wrapped as a custom op.
Prevents inductor from compiling the projections around the GDN core,
which would introduce numerical divergence that compounds through
the recurrent state.
"""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._full_forward(
hidden_states=hidden_states,
output=output,
)
direct_register_custom_op(
op_name="olmo_hybrid_gdn_full_forward",
op_func=olmo_hybrid_gdn_full_forward,
mutates_args=["output"],
)
@triton.jit
def fused_olmo_hybrid_gdn_gating_kernel(
g,
beta_output,
A_log,
a,
b,
dt_bias,
seq_len,
allow_neg_eigval: tl.constexpr,
NUM_HEADS: tl.constexpr,
beta: tl.constexpr,
threshold: tl.constexpr,
BLK_HEADS: tl.constexpr,
):
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
mask = head_off < NUM_HEADS
blk_A_log = tl.load(A_log + head_off, mask=mask)
blk_a = tl.load(a + off, mask=mask)
blk_b = tl.load(b + off, mask=mask)
blk_bias = tl.load(dt_bias + head_off, mask=mask)
# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
softplus_x = tl.where(
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
)
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
# beta = self.b_proj(hidden_states).sigmoid()
# if self.allow_neg_eigval: beta = beta * 2.0
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
if allow_neg_eigval:
blk_beta_output = blk_beta_output * 2.0
tl.store(
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
)
def fused_olmo_hybrid_gdn_gating(
A_log: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
dt_bias: torch.Tensor,
allow_neg_eigval: bool = False,
beta: float = 1.0,
threshold: float = 20.0,
) -> tuple[torch.Tensor, torch.Tensor]:
batch, num_heads = a.shape
seq_len = 1
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
fused_olmo_hybrid_gdn_gating_kernel[grid](
g,
beta_output,
A_log,
a,
b,
dt_bias,
seq_len,
allow_neg_eigval,
num_heads,
beta,
threshold,
8,
num_warps=1,
)
return g, beta_output
@@ -54,7 +54,12 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.inc import INCConfig
try:
# inc/ loader was removed in the SM86 mini-vLLM cut.
from vllm.model_executor.layers.quantization.inc import INCConfig
except ImportError:
INCConfig = type("AbsentINCConfig", (), {})
from vllm.model_executor.model_loader.weight_utils import (
sharded_weight_loader,
)
@@ -1,247 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import Int32, cute
from quack.compile_utils import make_fake_tensor
from vllm.triton_utils import triton
from .kernel_h import h_cutedsl
from .kernel_kkt_inv_uw import kkt_inv_uw_cutedsl
from .kernel_o import o_cutedsl
class PrepMetaKernel:
def __init__(self, BT: int) -> None:
self.BT = BT
self.num_warps = 8
@cute.jit
def __call__(
self,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
chunk_offsets: cute.Tensor,
stream: CUstream,
):
block = (self.num_warps * 32, 1, 1)
self.kernel(
cu_seqlens,
chunk_indices,
chunk_offsets,
).launch(grid=(1, 1, 1), block=block, stream=stream)
@cute.kernel
def kernel(
self,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
chunk_offsets: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
num_seqs = cu_seqlens.shape[0] - 1
num_warps = self.num_warps
tb_size = num_warps * 32
if tid == 0:
chunk_offsets[0] = 0
coarsen = cute.ceil_div(num_seqs, tb_size)
seq_start = tid * coarsen
num_iters = cutlass.min(seq_start + coarsen, num_seqs) - seq_start
# First pass: compute this thread's total chunk count.
thread_sum = Int32(0)
for i in range(num_iters):
seq_id = seq_start + i
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
thread_sum += cute.ceil_div(seqlen, self.BT)
# warp parallel scan
cu_num_chunks = thread_sum
for i in cutlass.range_constexpr(5):
offset = cutlass.const_expr(1 << i)
lower = cute.arch.shuffle_sync_up(
cu_num_chunks, offset=offset, mask_and_clamp=0
)
if lane_id >= offset:
cu_num_chunks += lower
# cross-warp cumsum (CTA-wide)
smem = cutlass.utils.SmemAllocator()
warp_num_chunks = smem.allocate_array(Int32, num_warps)
if lane_id == 31:
warp_num_chunks[warp_id] = cu_num_chunks
cute.arch.sync_threads()
for i in cutlass.range_constexpr(1, num_warps):
if warp_id >= i:
cu_num_chunks += warp_num_chunks[i - 1]
chunk_start = cu_num_chunks - thread_sum
# Second pass: recompute per-sequence chunk counts and write results.
for i in range(num_iters):
seq_id = seq_start + i
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
num_chunks = cute.ceil_div(seqlen, self.BT)
chunk_end = chunk_start + num_chunks
chunk_offsets[seq_id + 1] = chunk_end
for chunk_id in range(num_chunks):
chunk_indices[chunk_start + chunk_id, 0] = seq_id
chunk_indices[chunk_start + chunk_id, 1] = chunk_id
chunk_start = chunk_end
@cache
@staticmethod
def compile(BT: int):
cu_entries = cute.sym_int()
upper_bound_chunks = cute.sym_int()
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (upper_bound_chunks, 2), divisibility=2)
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
kernel = PrepMetaKernel(BT)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
cu_seqlens,
chunk_indices,
chunk_offsets,
stream,
options="--enable-tvm-ffi",
)
def _upper_bound_chunks(num_seqs: int, total_tokens: int, chunk_size: int) -> int:
return (num_seqs - 1) + triton.cdiv(total_tokens - (num_seqs - 1), chunk_size)
def prepare_metadata_cutedsl(
cu_seqlens: torch.Tensor,
total_tokens: int,
chunk_size: int = 64,
) -> tuple[torch.Tensor, torch.Tensor]:
num_seqs = cu_seqlens.numel() - 1
upper_bound_chunks = _upper_bound_chunks(num_seqs, total_tokens, chunk_size)
chunk_offsets = cu_seqlens.new_empty(num_seqs + 1, dtype=torch.int32)
chunk_indices = cu_seqlens.new_empty((upper_bound_chunks, 2), dtype=torch.int32)
PrepMetaKernel.compile(chunk_size)(cu_seqlens, chunk_indices, chunk_offsets)
return chunk_indices, chunk_offsets
def chunk_gated_delta_rule_cutedsl(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
chunk_offsets: torch.Tensor,
core_attn_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the GDN chunk CuteDSL prefill kernels.
Args:
q: Query tensor with shape ``[1, T, H, K]``.
k: Key tensor with shape ``[1, T, H, K]``.
v: Value tensor with shape ``[1, T, Hv, V]``.
g: Log-space decay tensor with shape ``[1, T, Hv]``.
beta: Delta-rule beta tensor with shape ``[1, T, Hv]``.
initial_state: Recurrent state with shape ``[N, Hv, V, K]``.
cu_seqlens: Cumulative sequence lengths with shape ``[N + 1]``.
chunk_indices: Chunk index metadata with shape ``[NT, 2]``.
chunk_offsets: Cumulative chunk offsets with shape ``[N + 1]``.
core_attn_out: Optional output buffer with shape ``[T, Hv, V]``.
Returns:
A tuple ``(output, final_state)`` where ``output`` has shape
``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``.
When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of
that buffer.
"""
q = q.squeeze(0)
k = k.squeeze(0)
v = v.squeeze(0)
g = g.squeeze(0)
beta = beta.squeeze(0)
_, _, K_dim = k.shape
_, num_v_heads, V_dim = v.shape
chunk_size = 64
upper_bound_chunks = chunk_indices.shape[0]
pad_t = upper_bound_chunks * chunk_size
total_chunks_ptr = chunk_offsets[-1:]
g_cu = torch.empty_like(g, dtype=torch.float32)
u = q.new_empty(pad_t, num_v_heads, V_dim)
w = q.new_empty(pad_t, num_v_heads, K_dim)
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
kkt_inv_uw_cutedsl(
k,
v,
u,
w,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks_ptr,
num_sms=num_sms,
)
h = k.new_empty(upper_bound_chunks, num_v_heads, V_dim, K_dim)
v_new = q.new_empty(pad_t, num_v_heads, V_dim)
final_state = torch.empty_like(initial_state)
h_cutedsl(
k,
u,
w,
v_new,
g_cu,
h,
initial_state,
final_state,
cu_seqlens,
chunk_offsets,
)
output = core_attn_out if core_attn_out is not None else torch.empty_like(v)
scale = K_dim**-0.5
o_cutedsl(
q,
k,
v_new,
h,
g_cu,
output,
cu_seqlens,
chunk_indices,
total_chunks_ptr,
scale,
num_sms=num_sms,
)
return output.unsqueeze(0), final_state
__all__ = [
"chunk_gated_delta_rule_cutedsl",
"prepare_metadata_cutedsl",
]
@@ -1,751 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_tcgen05,
cvt,
fence_before_tma_store,
simple_tma_copy,
)
class Sm100ChunkHKernel:
"""For each sequence, compute the chunk recurrent update.
The input V tile is the U output from the KKT/UW kernel. For each chunk:
V_new = U - W @ H.T
(we actually do V_new.T = U.T - H @ W.T instead)
H_scaled = H * exp(g_last)
V_scaled = V_new * exp(g_last - g)
H_new = H_scaled + V_scaled.T @ K
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
h_dtype: cutlass.Numeric = Float32,
BT: int = 64,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == V_dim == 128
assert BT == 64
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.h_dtype = h_dtype
self.BT = BT
self.num_stages = num_stages
self.num_warps = 10
@cute.jit
def _make_bf16_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def _make_h_tma_args(self, tensor: cute.Tensor, op: cpasync.TmaCopyOp):
# number of elements to fill 128B
num_elems = 128 // (tensor.element_type.width // 8)
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems)),
stride=(0, 0, num_elems, (1, self.V_dim * num_elems)),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, None, num_elems)),
slayout,
cta_tiler=(1, 1, self.V_dim, self.K_dim),
)
@cute.jit
def __call__(
self,
K: cute.Tensor,
V: cute.Tensor,
W: cute.Tensor,
V_new: cute.Tensor,
g_cu: cute.Tensor,
h: cute.Tensor,
h0: cute.Tensor,
ht: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_offsets: cute.Tensor,
stream: CUstream,
):
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
K_tma = self._make_bf16_tma_args(K, self.K_dim, tma_g2s, self.num_stages)
V_tma = self._make_bf16_tma_args(V, self.V_dim, tma_g2s, self.num_stages)
W_tma = self._make_bf16_tma_args(W, self.K_dim, tma_g2s, self.num_stages)
V_new_tma = self._make_bf16_tma_args(V_new, self.V_dim, tma_s2g, 1)
H0_tma = self._make_h_tma_args(h0, tma_g2s)
HT_tma = self._make_h_tma_args(ht, tma_s2g)
H_tma = self._make_h_tma_args(h, tma_s2g)
grid = (self.Hv, h0.shape[0], 1)
block = (self.num_warps * 32, 1, 1)
self.kernel(
K_tma,
V_tma,
W_tma,
V_new_tma,
H0_tma,
HT_tma,
H_tma,
g_cu,
cu_seqlens,
chunk_offsets,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
W_tma: cpasync.TmaInfo,
V_new_tma: cpasync.TmaInfo,
H0_tma: cpasync.TmaInfo,
HT_tma: cpasync.TmaInfo,
H_tma: cpasync.TmaInfo,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_offsets: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
head_id, seq_id, _ = cute.arch.block_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
BT = self.BT
V_dim = self.V_dim
K_dim = self.K_dim
num_stages = self.num_stages
is_f32 = self.h_dtype == Float32
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
# remove size=1 modes
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sH0 = allocate_tensor(smem, self.h_dtype, H0_tma.smem_layout)[0, 0, None, None]
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None]
sV_new = allocate_tensor(smem, BFloat16, V_new_tma.smem_layout)[
None, 0, None, 0
]
s_v_scale = smem.allocate_array(Float32, BT)
tma_mbar = smem.allocate_array(Int64, num_stages)
wh_in_mbar = smem.allocate_array(Int64, num_stages)
wh_done_mbar = smem.allocate_array(Int64, num_stages)
vk_in_mbar = smem.allocate_array(Int64, num_stages)
vk_done_mbar = smem.allocate_array(Int64, num_stages)
h0_mbar = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
wh_tmem = 0
vk_tmem = wh_tmem + BT
h_tmem_base = vk_tmem + K_dim
v_tmem_base = h_tmem_base + K_dim // 2
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(tma_mbar + i, 1)
cute.arch.mbarrier_init(wh_in_mbar + i, 256)
cute.arch.mbarrier_init(wh_done_mbar + i, 1)
cute.arch.mbarrier_init(vk_in_mbar + i, 256)
cute.arch.mbarrier_init(vk_done_mbar + i, 1)
cute.arch.mbarrier_init(h0_mbar, 1)
cute.arch.mbarrier_init_fence()
elif warp_id == 1:
cpasync.prefetch_descriptor(H0_tma.atom)
cpasync.prefetch_descriptor(W_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(HT_tma.atom)
cpasync.prefetch_descriptor(H_tma.atom)
cpasync.prefetch_descriptor(V_new_tma.atom)
cute.arch.sync_threads()
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
seqlen = eos - bos
num_chunks = cute.ceil_div(seqlen, BT)
if warp_id == 9:
# TMA warp
stage_id = 0
parity = 1
k_head_id = head_id // (self.Hv // self.H)
chunk_offset = chunk_offsets[seq_id]
# load H0
with cute.arch.elect_one():
H0_size = V_dim * K_dim * self.h_dtype.width // 8
cute.arch.mbarrier_arrive_and_expect_tx(h0_mbar, H0_size)
simple_tma_copy(
H0_tma.atom,
H0_tma.tma_tensor[seq_id, head_id, None, None],
sH0,
h0_mbar,
)
# shape: ((BT, num_BT_tiles), (64, 2))
gW_tiles = cute.logical_divide(
W_tma.tma_tensor[None, head_id, None], (BT, None)
)
gV_tiles = cute.logical_divide(
V_tma.tma_tensor[None, head_id, None], (BT, None)
)
gK_tiles = cute.logical_divide(
cute.domain_offset((bos, 0), K_tma.tma_tensor[None, k_head_id, None]),
(BT, None),
)
for chunk_id in range(num_chunks):
mbar = tma_mbar + stage_id
gW = gW_tiles[(None, chunk_offset + chunk_id), None]
gV = gV_tiles[(None, chunk_offset + chunk_id), None]
gK = gK_tiles[(None, chunk_id), None]
# wait for MMA to release the buffer
cute.arch.mbarrier_wait(vk_done_mbar + stage_id, parity)
# load W, V (i.e. U), and K
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + V_dim + K_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(
W_tma.atom, gW, sW[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 8:
# MMA warp
_tcgen05.alloc(taddr)
stage_id = 0
parity = 0
wh_idesc = _tcgen05.make_bf16_idesc(V_dim, BT, negate_A=True)
vk_idesc = _tcgen05.make_bf16_idesc(V_dim, K_dim, transpose_B=True)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
# when using BF16 state, H is read from smem for the 1st iteration
# variable names in this conditional branch can't be the same as those
# in the mainloop below due to CuteDSL restrictions.
if cutlass.const_expr(not is_f32):
##### 1st MMA: V_new.T = V.T - H @ W.T #####
Haddr0 = sH0[None, None].iterator.toint()
Waddr0 = sW[None, None, stage_id].iterator.toint()
hdesc0_base = sdesc_template | (Haddr0 >> 4)
wdesc0_base = sdesc_template | (Waddr0 >> 4)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
hdesc0 = hdesc0_base | ((i * V_dim * 128 + j * 32) >> 4)
wdesc0 = wdesc0_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_f16(wh_tmem, hdesc0, wdesc0, wh_idesc, True)
_tcgen05.commit(wh_done_mbar + stage_id)
##### 2nd MMA: H_new = H + V_new.T @ K #####
Kaddr0 = sK[None, None, stage_id].iterator.toint()
kdesc0_base = sdesc_template | (Kaddr0 >> 4)
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for k in cutlass.range_constexpr(BT // 16):
vtmem0 = v_tmem_base + k * 8
kdesc0 = kdesc0_base | ((k * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(vk_tmem, vtmem0, kdesc0, vk_idesc, True)
_tcgen05.commit(vk_done_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
num_iters = num_chunks - int(not is_f32)
for _ in range(num_iters):
##### 1st MMA: V_new.T = V.T - H @ W.T #####
Waddr = sW[None, None, stage_id].iterator.toint()
wdesc_base = sdesc_template | (Waddr >> 4)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
htmem = h_tmem_base + i * 32 + j * 8
wdesc = wdesc_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_ts_f16(wh_tmem, htmem, wdesc, wh_idesc, True)
_tcgen05.commit(wh_done_mbar + stage_id)
##### 2nd MMA: H_new = H + V_new.T @ K #####
Kaddr = sK[None, None, stage_id].iterator.toint()
kdesc_base = sdesc_template | (Kaddr >> 4)
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for k in cutlass.range_constexpr(BT // 16):
vtmem = v_tmem_base + k * 8
kdesc = kdesc_base | ((k * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(vk_tmem, vtmem, kdesc, vk_idesc, True)
_tcgen05.commit(vk_done_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id >= 4:
# H warps
tid_ = tid % 128
warp_id_ = warp_id % 4
chunk_offset = chunk_offsets[seq_id]
stage_id = 0
vk_stage_id = 0
vk_parity = 0
op = cute.nvgpu.CopyUniversalOp()
cp_16B = cute.make_copy_atom(op, Float32, num_bits_per_copy=128)
##### chunk_id = 0 #####
if True:
chunk_id = 0
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
# for 1st chunk, wait for H0 transfer from gmem
if warp_id_ == 0:
cute.arch.mbarrier_wait(h0_mbar, 0)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
# when H0 is FP32, we need to pack it to BF16
# also store to smem for TMA store later.
if cutlass.const_expr(is_f32):
for i in cutlass.range_constexpr(K_dim // 32):
# H0 smem layout: (V_dim, (32, K_dim/32))
h_f32 = cute.make_rmem_tensor(32, Float32)
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.load().to(BFloat16))
_tcgen05.st(
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
)
# H smem layout: (V_dim, (64, K_dim/64))
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, dst)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# scale H for 2nd MMA
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
if cutlass.const_expr(is_f32):
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
else:
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
sH_src = cute.local_tile(sH0[tid_, None], (32,), (i,))
cute.copy(cp_16B, sH_src, h_bf16)
h_f32.store(
cvt.bf16x2_to_fp32x2(
cute.recast_tensor(h_bf16, Uint32)
).load()
)
for j in cutlass.range(32, vectorize=True):
h_f32[j] *= h_scale
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# for BF16 H0, we issue TMA store from H0 smem
# for FP32 H0, we issue TMA store from H smem (after packing)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
fence_before_tma_store()
if warp_id_ == 3:
h_src = sH if cutlass.const_expr(is_f32) else sH0
h_dst = H_tma.tma_tensor[
chunk_offset + chunk_id, head_id, None, None
]
simple_tma_copy(H_tma.atom, h_src, h_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
# When H0 is BF16, and there is only 1 chunk, storing
# the final state to sH0 can race before this store
# has finished. hence, we need to wait here.
if cutlass.const_expr(not is_f32):
cute.arch.cp_async_bulk_wait_group(0, read=True)
stage_id = (stage_id + 1) % num_stages
##### subsequent chunks #####
for chunk_id in range(1, num_chunks):
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
# wait for H from previous vk MMA
if warp_id_ == 0:
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
vk_stage_id = (vk_stage_id + 1) % num_stages
if vk_stage_id == 0:
vk_parity ^= 1
elif warp_id_ == 3:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
# load FP32 H from tmem, convert to BF16, store to tmem for 1st MMA,
# store to smem for TMA store later.
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.to(BFloat16))
_tcgen05.st(
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
)
# H smem layout: (V_dim, (64, K_dim/64))
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, dst)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# scale H for 2nd MMA
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
h_f32.store(
_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
)
for j in cutlass.range(32, vectorize=True):
h_f32[j] *= h_scale
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# issue TMA store for O kernel
cute.arch.barrier(barrier_id=1, number_of_threads=128)
fence_before_tma_store()
if warp_id_ == 3:
h_dst = H_tma.tma_tensor[
chunk_offset + chunk_id, head_id, None, None
]
simple_tma_copy(H_tma.atom, sH, h_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
# handle final state. reuse H0 smem.
if warp_id_ == 0:
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 32):
h_f32 = cute.make_rmem_tensor(32, Float32)
h_f32.store(_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32))
if cutlass.const_expr(is_f32):
cute.copy(cp_16B, h_f32, sH0[tid_, (None, i)])
else:
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
h_bf16.store(h_f32.load().to(BFloat16))
sH0_dst = cute.local_tile(sH0[tid_, None], (32,), (i,))
cute.copy(cp_16B, h_bf16, sH0_dst)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
if warp_id_ == 0:
ht_dst = HT_tma.tma_tensor[seq_id, head_id, None, None]
simple_tma_copy(HT_tma.atom, sH0, ht_dst)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
if warp_id_ == 1:
_tcgen05.dealloc()
else:
# V warps
stage_id = 0
parity = 0
chunk_offset = chunk_offsets[seq_id]
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
stsm_trans_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=True)
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
stsm_trans_atom = cute.make_copy_atom(stsm_trans_op, BFloat16)
# ((BT, num_BT_tiles), V_dim)
gV_new_tiles = cute.logical_divide(
V_new_tma.tma_tensor[None, head_id, None], (BT, None)
)
# sV shape: [BT, (64, V_dim/64), num_stages]
# sV_view shape: [BT, (8, (8,2)), num_stages]
sV_view = cute.logical_divide(sV, (None, 8, None))
sV_new_view = cute.logical_divide(sV_new, (None, 8))
# [BT, 8, num_stages]
s_col = warp_id * 4 + (lane_id // 8)
sV_view = sV_view[None, (None, s_col), None]
sV_new_view = sV_new_view[None, (None, s_col)]
for chunk_id in range(num_chunks):
# wait for V to arrive
if warp_id == 0:
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
# unpack V BF16->FP32, then store to tmem for 1st MMA
# V smem layout: [BT, (64, V_dim/64)] / [BT, V_dim]
# each iteration, CTA loads [8, V_dim] tile
# (warp loads [8, 32] tile)
for i in cutlass.range_constexpr(BT // 8):
s_row = i * 8 + (lane_id % 8)
v_bf16 = cute.make_rmem_tensor(8, BFloat16)
cute.copy(ldsm_trans_atom, sV_view[s_row, None, stage_id], v_bf16)
v_fp32 = cvt.bf16x2_to_fp32x2(cute.recast_tensor(v_bf16, Uint32))
v_fp32 = cute.logical_divide(v_fp32, 4) # (4, 2)
tcol = wh_tmem + i * 8
_tcgen05.st(warp_id * 32 + 0, tcol, "16x256b", 1, v_fp32[None, 0])
_tcgen05.st(warp_id * 32 + 16, tcol, "16x256b", 1, v_fp32[None, 1])
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
# load g_cu for scaling
if tid < BT:
end_t = min(bos + (chunk_id + 1) * BT, eos)
last_idx = end_t - 1
t = bos + chunk_id * BT + tid
val = Float32(0.0)
if t < eos:
val = cute.math.exp(
g_cu[last_idx, head_id] - g_cu[t, head_id],
fastmath=True,
)
s_v_scale[tid] = val
# wait for 1st MMA to finish
if warp_id == 2:
cute.arch.mbarrier_wait(wh_done_mbar + stage_id, parity)
elif warp_id == 3:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 8):
v_new = cute.make_rmem_tensor((4, 2), Float32)
tcol = wh_tmem + i * 8
v_new[None, 0].store(
_tcgen05.ld(warp_id * 32 + 0, tcol, "16x256b", 1)
)
v_new[None, 1].store(
_tcgen05.ld(warp_id * 32 + 16, tcol, "16x256b", 1)
)
v_new_bf16 = cute.make_rmem_tensor(8, BFloat16)
v_new_bf16.store(v_new.load().to(BFloat16))
# scale V_new for 2nd MMA
scale0 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 0]
scale1 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 1]
v_scaled = cute.make_rmem_tensor(8, Float32)
for k in cutlass.range_constexpr(4):
v_scaled[k * 2] = v_new[k * 2] * scale0
v_scaled[k * 2 + 1] = v_new[k * 2 + 1] * scale1
v_scaled_bf16 = v_scaled.load().to(BFloat16).reshape((4, 2))
# store V_new BF16 for O kernel
s_row = i * 8 + (lane_id % 8)
cute.copy(stsm_trans_atom, v_new_bf16, sV_new_view[s_row, None])
# store to tmem
tcol = v_tmem_base + i * 4
_tcgen05.st(
warp_id * 32 + 0, tcol, "16x128b", 1, v_scaled_bf16[None, 0]
)
_tcgen05.st(
warp_id * 32 + 16, tcol, "16x128b", 1, v_scaled_bf16[None, 1]
)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
# issue TMA store for V_new
cute.arch.barrier(barrier_id=2, number_of_threads=128)
fence_before_tma_store()
if warp_id == 3:
gV = gV_new_tiles[(None, chunk_offset + chunk_id), None]
simple_tma_copy(V_new_tma.atom, sV_new, gV)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
@cache
@staticmethod
def compile(
H: int,
Hv: int,
K_dim: int,
V_dim: int,
h_dtype: cutlass.Numeric = Float32,
BT: int = 64,
num_stages: int = 2,
):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
num_sequences = cute.sym_int()
cu_entries = cute.sym_int()
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
V = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
V_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
h = make_fake_tensor(
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
)
h0 = make_fake_tensor(
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
)
ht = make_fake_tensor(
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
)
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
kernel = Sm100ChunkHKernel(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
K,
V,
W,
V_new,
g_cu,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
stream,
options="--enable-tvm-ffi",
)
def h_cutedsl(
K: torch.Tensor,
V: torch.Tensor,
W: torch.Tensor,
V_new: torch.Tensor,
g_cu: torch.Tensor,
h: torch.Tensor,
h0: torch.Tensor,
ht: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_offsets: torch.Tensor,
BT: int = 64,
num_stages: int = 2,
) -> None:
"""Compute H/V_new with the same argument order as the CUDA wrapper."""
_, H, K_dim = K.shape
_, Hv, V_dim = V.shape
h_dtype = {
torch.bfloat16: BFloat16,
torch.float32: Float32,
}[h0.dtype]
Sm100ChunkHKernel.compile(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)(
K,
V,
W,
V_new,
g_cu,
h,
h0,
ht,
cu_seqlens,
chunk_offsets,
)
h_v2b_cutedsl = h_cutedsl
@@ -1,925 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_bf16x2_neg,
_bf16x2_sub,
_tcgen05,
cvt,
fence_before_tma_store,
mma_sync,
simple_tma_copy,
)
class Sm100ChunkUWKernel:
"""Compute per-chunk KKT inverse preprocessing and U/W tiles.
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
A = strictLower(beta * (K @ K.T) * Gamma)
Ai = inverse(I + A)
U = (Ai * beta) @ V
W = (Ai * beta * exp(g_cu)) @ K
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == V_dim == 128
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.num_stages = num_stages
# hard-code
self.BT = 64
self.num_warps = 4 + 4 + 4
@cute.jit
def _make_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
num_stages: int,
op: cpasync.TmaCopyOp,
):
# logical layout: [BT, dim]
# permute for TMA: [dim/64, BT, 64] with swizzling
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), num_stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
# we need to convert gmem layout to (T, H, (64, D/64)) for make_tiled_tma_atom()
# to emit a single 4D TMA. otherwise, it will emit (D/64)x 3D TMA.
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def __call__(
self,
K: cute.Tensor,
V: cute.Tensor,
U: cute.Tensor,
W: cute.Tensor,
g: cute.Tensor,
beta: cute.Tensor,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
num_sms: Int32,
stream: CUstream,
):
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
K_tma = self._make_tma_args(K, self.K_dim, self.num_stages, tma_g2s)
V_tma = self._make_tma_args(V, self.V_dim, self.num_stages, tma_g2s)
U_tma = self._make_tma_args(U, self.V_dim, 1, tma_s2g)
W_tma = self._make_tma_args(W, self.K_dim, 1, tma_s2g)
grid = (num_sms // self.Hv, self.Hv, 1)
block = (self.num_warps * 32, 1, 1)
self.kernel(
K_tma,
V_tma,
U_tma,
W_tma,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
U_tma: cpasync.TmaInfo,
W_tma: cpasync.TmaInfo,
g: cute.Tensor,
beta: cute.Tensor,
g_cu: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
):
tid, _, _ = cute.arch.thread_idx()
bid, head_id, _ = cute.arch.block_idx()
grid_x, _, _ = cute.arch.grid_dim()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
k_head_id = head_id // (self.Hv // self.H)
BT = self.BT
K_dim = self.K_dim
V_dim = self.V_dim
num_stages = self.num_stages
INV_BAR = 1
EPI_BAR = 2
SCAN_BAR = 3
TMEM_ALLOC_BAR = 4
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sU = allocate_tensor(smem, BFloat16, U_tma.smem_layout)[None, 0, None, 0]
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, 0]
sA_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
sAi_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
s_beta = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
s_g_cu = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
s_beta_g = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
tma_mbar = smem.allocate_array(Int64, num_stages)
prep_mbar = smem.allocate_array(Int64, num_stages)
mma_kkt_mbar = smem.allocate_array(Int64, num_stages)
inv_mbar = smem.allocate_array(Int64, num_stages)
mma_u_mbar = smem.allocate_array(Int64, num_stages)
mma_w_mbar = smem.allocate_array(Int64, num_stages)
epi_mbar = smem.allocate_array(Int64, num_stages)
taddr = smem.allocate(Int32, 4)
kkt_tmem = 0
U_tmem_base = kkt_tmem + BT
Ab_tmem_base = U_tmem_base + V_dim * num_stages
assert Ab_tmem_base + (BT // 2) <= 512
# prepare ldmatrix/stmatrix ops
ldsm_op = warp.LdMatrix8x8x16bOp(num_matrices=4)
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4)
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
ldsm_atom = cute.make_copy_atom(ldsm_op, BFloat16)
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(tma_mbar + i, 1)
cute.arch.mbarrier_init(prep_mbar + i, 64)
cute.arch.mbarrier_init(mma_kkt_mbar + i, 1)
cute.arch.mbarrier_init(inv_mbar + i, 128)
cute.arch.mbarrier_init(mma_u_mbar + i, 1)
cute.arch.mbarrier_init(mma_w_mbar + i, 1)
cute.arch.mbarrier_init(epi_mbar + i, 128)
cute.arch.mbarrier_init_fence()
elif warp_id == 1:
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(U_tma.atom)
cpasync.prefetch_descriptor(W_tma.atom)
cute.arch.sync_threads()
num_global_chunks = total_chunks[0]
if warp_id == 11:
# TMA warp
stage_id = 0
parity = 1
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
# since off_t is not a multiple of BT, we need to use
# domain_offset() to shift the pointer first.
mbar = tma_mbar + stage_id
gK = cute.local_tile(
cute.domain_offset(
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
gV = cute.local_tile(
cute.domain_offset((bos, 0), V_tma.tma_tensor[None, head_id, None]),
tiler=(BT, V_dim),
coord=(chunk_id, 0),
)
# when UW MMA is done, K and V TMA buffers are released
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + V_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 10:
# MMA warp
_tcgen05.alloc(taddr)
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
stage_id = 0
parity = 0
kkt_idesc = _tcgen05.make_bf16_idesc(BT, BT)
u_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
w_idesc = _tcgen05.make_bf16_idesc(BT, K_dim, transpose_B=True)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
for global_chunk_id in range(bid, num_global_chunks, grid_x):
U_tmem = U_tmem_base + V_dim * stage_id
W_tmem = U_tmem | (16 << 16)
Ab_tmem = Ab_tmem_base
Abg_tmem = Ab_tmem_base | (16 << 16)
##### KKT MMA: KKT = K @ K.T #####
kaddr = sK[None, None, stage_id].iterator.toint()
kdesc_base = sdesc_template | (kaddr >> 4)
# wait for TMA data to arrive
# kkt tmem is guaranteed to be free as this is issued
# after the previous kkt's consumer (inv warps)
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
enable_d = (i > 0) or (j > 0)
_tcgen05.mma_f16(kkt_tmem, kdesc, kdesc, kkt_idesc, enable_d)
_tcgen05.commit(mma_kkt_mbar + stage_id)
##### U/W MMA: U = Ab @ V, W = Abg @ K #####
vaddr = sV[None, None, stage_id].iterator.toint()
vdesc = sdesc_template | (vaddr >> 4)
kdesc = sdesc_template | (kaddr >> 4)
# wait for epilogue to release tmem buffer
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 16):
_tcgen05.mma_ts_f16(W_tmem, Abg_tmem + i * 8, kdesc, w_idesc, i > 0)
kdesc += (16 * 128) >> 4
_tcgen05.commit(mma_w_mbar + stage_id)
for i in cutlass.range_constexpr(BT // 16):
_tcgen05.mma_ts_f16(U_tmem, Ab_tmem + i * 8, vdesc, u_idesc, i > 0)
vdesc += (16 * 128) >> 4
_tcgen05.commit(mma_u_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
_tcgen05.dealloc()
elif warp_id >= 8:
# dedicated prep warps for beta and gate, consumed by INV warps
stage_id = 0
parity = 0
tid_ = tid % 128
warp_id_ = warp_id % 4
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
off_t = bos + chunk_id * BT
t = off_t + tid_
in_bounds = t < eos
beta_val = beta[t, head_id] if in_bounds else Float32(0.0)
g_val = g[t, head_id] if in_bounds else Float32(0.0)
# warp-local prefix scan
for i in cutlass.range_constexpr(5):
offset = cutlass.const_expr(1 << i)
lower = cute.arch.shuffle_sync_up(g_val, offset, mask_and_clamp=0)
if lane_id >= offset:
g_val += lower
# Delay the stage-reuse wait until just before touching smem:
# global loads and the warp-local scan do not use staged buffers.
if warp_id_ == 0:
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity ^ 1)
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
# Store beta and the per-warp scan totals for the cross-warp fixup.
s_beta[tid_, stage_id] = beta_val
if lane_id == 31:
s_g_cu[warp_id_, stage_id] = g_val
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
# Add the sum from the lower prep warp.
if warp_id_ == 1:
g_val += s_g_cu[0, stage_id]
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
if in_bounds:
g_cu[t, head_id] = g_val
s_g_cu[tid_, stage_id] = g_val
s_beta_g[tid_, stage_id] = beta_val * cute.math.exp(g_val)
cute.arch.mbarrier_arrive(prep_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id >= 4:
# inv warps
tid_ = tid % 128
warp_id_ = warp_id % 4
def store_ab_abg(
Ai_f32, s_beta, s_beta_g, warp_id_, lane_id, tile_col, Ab_tmem_base
):
# compute Ab and Abg from Ai, then store to tmem
beta_col = cute.make_rmem_tensor((2, 2), Float32)
beta_g_col = cute.make_rmem_tensor((2, 2), Float32)
for i in cutlass.range_constexpr(2):
base = tile_col * 16 + i * 8 + (lane_id % 4) * 2
for j in cutlass.range_constexpr(2):
beta_col[j, i] = s_beta[base + j]
beta_g_col[j, i] = s_beta_g[base + j]
# without conversion to TensorSSA, cutlass.range(vectorize=True) fails
beta_col = beta_col.load()
beta_g_col = beta_g_col.load()
Ab_f32 = cute.make_rmem_tensor(8, Float32)
Abg_f32 = cute.make_rmem_tensor(8, Float32)
for i in cutlass.range(8, vectorize=True):
scale_idx = (i // 4) * 2 + (i % 2)
Ab_f32[i] = Ai_f32[i] * beta_col[scale_idx]
Abg_f32[i] = Ai_f32[i] * beta_g_col[scale_idx]
Ab = Ab_f32.load().to(BFloat16)
Abg = Abg_f32.load().to(BFloat16)
Ab_tmem = Ab_tmem_base + tile_col * 8
_tcgen05.st(warp_id_ * 32, Ab_tmem, "16x128b", 2, Ab)
_tcgen05.st(warp_id_ * 32 + 16, Ab_tmem, "16x128b", 2, Abg)
# clear the Ab/Abg tmem buffer once before mainloop
# this is to keep the upper triangular tiles zeros
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
zeros_ = cute.make_rmem_tensor(BT // 2, Float32)
zeros_.fill(0.0)
_tcgen05.st(warp_id_ * 32, Ab_tmem_base, "32x32b", BT // 2, zeros_)
stage_id = 0
parity = 0
# for sA, we can avoid bank conflict without using swizzling because
# this is only used as tmp buffer between rmem<->smem, no gmem interactions.
# to do so, we can put (8,8) tile contiguous in memory. logically, we are
# partitioning (64,64) tile into 4x (16,16) tiles. to make indexing easier
# later, we view (16,16) tile as (32,8) tile here.
# (4,4) is (row_tile, col_tile).
sA_layout = cute.make_layout(((8, 32), (4, 4)))
sA = cute.make_tensor(sA_ptr, sA_layout)
sAi = cute.make_tensor(sAi_ptr, sA_layout)
# pre-compute ldmatrix addresses
sA_ldsm = sA[(None, lane_id), None]
sAi_ldsm = sAi[(None, lane_id), None]
# init Ai smem buffer with zeros (only the first 48 rows)
zeros_bf16 = cute.make_rmem_tensor(8, BFloat16)
zeros_bf16.fill(0.0)
for i in cutlass.range_constexpr(3):
cute.copy(stsm_atom, zeros_bf16, sAi_ldsm[None, (i, warp_id_)])
# indices for ldmatrix layout later
row_indices = cute.make_rmem_tensor((1, 2, 1), Int32)
row_indices[0, 0, 0] = warp_id_ * 16 + (lane_id // 4)
row_indices[0, 1, 0] = warp_id_ * 16 + (lane_id // 4) + 8
row_indices = row_indices.load()
col_indices = cute.make_rmem_tensor((2, 1, 2), Int32)
col_indices[0, 0, 0] = (lane_id % 4) * 2 + 0
col_indices[1, 0, 0] = (lane_id % 4) * 2 + 1
col_indices[0, 0, 1] = (lane_id % 4) * 2 + 8
col_indices[1, 0, 1] = (lane_id % 4) * 2 + 9
col_indices = col_indices.load()
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
off_t = bos + chunk_id * BT
##### Phase 1: A = strictLower(beta * kkt * Gamma) #####
# Ab/Abg share one tmem slot across stages. The MMA warp commits
# tcgen05 groups in program order: KKT_i, W_i, U_i, KKT_{i+1}.
# Waiting for KKT_i means the previous W/U commits have completed,
# so the INV warps can safely overwrite Ab/Abg for this iteration.
# Wait for prep warps to publish beta/gate and for KKT MMA.
if warp_id_ == 0:
cute.arch.mbarrier_wait(prep_mbar + stage_id, parity)
cute.arch.mbarrier_wait(mma_kkt_mbar + stage_id, parity)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
# tmem 16x256b layout / ldmatrix layout
beta_row = cute.make_rmem_tensor(2, Float32)
g_cu_row = cute.make_rmem_tensor(2, Float32)
for i in cutlass.range_constexpr(2):
idx = warp_id_ * 16 + i * 8 + (lane_id // 4)
beta_row[i] = s_beta[idx, stage_id]
g_cu_row[i] = s_g_cu[idx, stage_id]
# mode0 is 8 rows together
# mode1 is top and bottom 8 rows
# mode2 is groups of 16 rows
beta_row = beta_row.load().reshape((1, 2, 1))
g_cu_row = g_cu_row.load().reshape((1, 2, 1))
# mode0 is 2 consecutive elems
# mode1 is top and bottom 8 rows
# mode2 is next 8 columns
# mode3 is repeating that 16x16 tile pattern
kkt = _tcgen05.ld(kkt_tmem, 0, "16x256b", BT // 8)
kkt = kkt.reshape((2, 2, 2, BT // 16))
for i in cutlass.range_constexpr(BT // 16):
# mode0 is 2 elems next to each other
# mode1 is 4 pairs of elems on 1 row
# mode2 is top and bottom 8 rows
# mode3 is next 16 columns
col_coord = (None, lane_id % 4, None, i)
s_g_cu_view = cute.make_tensor(
s_g_cu[None, stage_id].iterator, (2, 4, 2, BT // 16)
)
g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2))
Gamma = cute.math.exp(g_cu_row - g_cu_col, fastmath=True)
A = kkt[None, None, None, i] * beta_row * Gamma
# strict lower mask
# NOTE: for OOB t position, s_beta is filled with zeros.
# hence, we don't need to apply bounds check for columns.
A_masked = cute.where(row_indices > col_indices + i * 16, A, 0.0)
# pack to BF16
# CuteDSL doesn't generate cvt.bf16x2.f32 here for some reasons
packed = cute.make_rmem_tensor(4, Uint32)
for j in cutlass.range_constexpr(4):
packed[j] = cvt.fp32x2_to_bf16x2(
A_masked[j * 2], A_masked[j * 2 + 1]
)
# store to smem
cute.copy(
stsm_atom,
cute.recast_tensor(packed, BFloat16),
sA_ldsm[None, (warp_id_, i)],
)
# use sync warp instead of bar.sync because for block-diagonal inverse,
# each warp reads its own private smem memory.
cute.arch.sync_warp()
##### Phase 2: matrix inverse #####
# we use Newton-Schulz iterations to compute the inverse
# of the four 16x16 diagonal blocks.
# Ai_new = 2 Ai - Ai @ M @ Ai
# where M = I + A
#
# we do this with 2 MMAs:
# 1. -AiM = Ai @ (-M)
# 2. Ai_new = 2 Ai + (-AiM) @ Ai
zeros_f32 = cute.make_rmem_tensor(4, Float32)
zeros_f32.fill(0.0)
Ai_bf16 = cute.make_rmem_tensor(8, BFloat16)
mma_B_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
M_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
acc = cute.make_rmem_tensor((4, 2), Float32)
# ldmatrix copies require rank-1 destination views.
mma_B_bf16_ldsm = cute.group_modes(mma_B_bf16, 0, 2)
# Packed aliases for BF16x2 arithmetic.
Ai = cute.recast_tensor(Ai_bf16, Uint32)
M = cute.recast_tensor(M_bf16, Uint32)
# construct rmem-backed identity matrix
eye = cute.make_rmem_tensor(4, Uint32)
eye[0] = Uint32(lane_id % 9 == 0) * Uint32(0x00003F80) + Uint32(
lane_id % 9 == 4
) * Uint32(0x3F800000)
eye[1] = 0
eye[2] = 0
eye[3] = eye[0]
# initial guess: Ai = I-A
cute.copy(ldsm_atom, sA_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_sub(eye[i], Ai[i])
# (4, 2)
Ai_f32 = cute.logical_divide(cvt.bf16x2_to_fp32x2(Ai), 4)
# M is holding -(I+A), stay constant throughout the iterations
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_)],
cute.group_modes(M_bf16, 0, 2),
)
for i in cutlass.range_constexpr(4):
M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i])
# The fourth round keeps correlated-key KKT blocks stable in BF16.
for _ in cutlass.range_constexpr(4):
# First MMA: -AiM = Ai @ (-M)
cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)])
cute.arch.sync_warp()
acc[None, 0] = mma_sync(Ai_bf16, M_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, M_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
# Second MMA: Ai_new = 2Ai + (-AiM) @ Ai
for j in cutlass.range(8, vectorize=True):
Ai_f32[j] *= 2.0
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_)],
mma_B_bf16_ldsm,
)
Ai_f32[None, 0] = mma_sync(
Ai_bf16, mma_B_bf16[None, 0], Ai_f32[None, 0]
)
Ai_f32[None, 1] = mma_sync(
Ai_bf16, mma_B_bf16[None, 1], Ai_f32[None, 1]
)
Ai_bf16.store(Ai_f32.load().to(BFloat16))
cute.copy(stsm_atom, Ai_bf16, sAi_ldsm[None, (warp_id_, warp_id_)])
store_ab_abg(
Ai_f32,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
warp_id_,
Ab_tmem_base,
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 1
# given
# [ Ai00 ]
# [ A10 Ai11 ]
# [ A20 A21 Ai22 ]
# [ A30 A31 A32 Ai33]
# warp1: Ai10 = -Ai11 @ A10 @ Ai00
# warp2: Ai21 = -Ai22 @ A21 @ Ai11
# warp3: Ai32 = -Ai33 @ A32 @ Ai22
if warp_id_ >= 1:
neg_Ai = cute.make_rmem_tensor(4, Uint32)
for i in cutlass.range_constexpr(4):
neg_Ai[i] = _bf16x2_neg(Ai[i])
neg_Ai_bf16 = cute.recast_tensor(neg_Ai, BFloat16)
cute.copy(
ldsm_trans_atom,
sA_ldsm[None, (warp_id_, warp_id_ - 1)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (warp_id_ - 1, warp_id_ - 1)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
Ai_bf16.store(acc.load().to(BFloat16))
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
warp_id_ - 1,
Ab_tmem_base,
)
cute.copy(
stsm_atom,
Ai_bf16,
sAi_ldsm[None, (warp_id_, warp_id_ - 1)],
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 2
# warp2: Ai20 = -Ai22 @ (A20 @ Ai00 + A21 @ Ai10)
# warp3: Ai31 = -Ai33 @ (A31 @ Ai11 + A32 @ Ai21)
if warp_id_ >= 2:
tile_col = warp_id_ - 2
cute.copy(
ldsm_atom,
sA_ldsm[None, (warp_id_, tile_col)],
Ai_bf16,
)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (tile_col, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
cute.copy(
ldsm_atom, sA_ldsm[None, (warp_id_, tile_col + 1)], Ai_bf16
)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (tile_col + 1, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], acc[None, 0])
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], acc[None, 1])
tmp = cute.make_rmem_tensor(8, BFloat16)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
cute.arch.sync_warp()
cute.copy(ldsm_atom, sAi_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_neg(Ai[i])
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (warp_id_, tile_col)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
tile_col,
Ab_tmem_base,
)
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
# off-diagonal by 3
# warp3: Ai30 = -Ai33 @ (A30 @ Ai00 + A31 @ Ai10 + A32 @ Ai20)
if warp_id_ == 3:
cute.copy(ldsm_atom, sA_ldsm[None, (3, 0)], Ai_bf16)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (0, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
for i in cutlass.range_constexpr(1, 3):
cute.copy(ldsm_atom, sA_ldsm[None, (3, i)], Ai_bf16)
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (i, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(
Ai_bf16, mma_B_bf16[None, 0], acc[None, 0]
)
acc[None, 1] = mma_sync(
Ai_bf16, mma_B_bf16[None, 1], acc[None, 1]
)
tmp = cute.make_rmem_tensor(8, BFloat16)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
cute.arch.sync_warp()
cute.copy(ldsm_atom, sAi_ldsm[None, (3, 3)], Ai_bf16)
for i in cutlass.range_constexpr(4):
Ai[i] = _bf16x2_neg(Ai[i])
cute.copy(
ldsm_trans_atom,
sAi_ldsm[None, (3, 0)],
mma_B_bf16_ldsm,
)
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
tmp.store(acc.load().to(BFloat16))
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
store_ab_abg(
acc,
s_beta[None, stage_id],
s_beta_g[None, stage_id],
warp_id_,
lane_id,
0,
Ab_tmem_base,
)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(inv_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
else:
# epi warps
stage_id = 0
parity = 0
# ((BT, num_global_chunks), V_dim)
gU_tiles = cute.logical_divide(
U_tma.tma_tensor[None, head_id, None], (BT, None)
)
gW_tiles = cute.logical_divide(
W_tma.tma_tensor[None, head_id, None], (BT, None)
)
# sW shape: [BT, (64, K_dim/64)]
# sW_view shape: [(8, 2), (4, K_dim/64)]
s_row = warp_id * 16 + lane_id % 16 # select the rows of [16,16] tile
sW_view = cute.zipped_divide(
sW[s_row, None],
tiler=cute.make_layout((8, 2)),
)
sU_view = cute.zipped_divide(
sU[s_row, None],
tiler=cute.make_layout((8, 2)),
)
# select the 8 columns within [16,16] tile
sW_view = sW_view[(None, lane_id // 16), None]
sU_view = sU_view[(None, lane_id // 16), None]
for global_chunk_id in range(bid, num_global_chunks, grid_x):
# wait for W MMA + previous TMA store to finish
U_tmem = U_tmem_base + V_dim * stage_id
if warp_id == 0:
cute.arch.mbarrier_wait(mma_w_mbar + stage_id, parity)
elif warp_id == 1:
with cute.arch.elect_one():
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
w_f32 = _tcgen05.ld(warp_id * 32 + 16, U_tmem, "16x256b", K_dim // 8)
_tcgen05.wait_ld()
w_bf16 = cute.make_rmem_tensor((8, K_dim // 16), BFloat16)
w_bf16.store(w_f32.to(BFloat16))
cute.copy(stsm_atom, w_bf16, sW_view)
# wait for U MMA + issue W TMA store
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
fence_before_tma_store()
if warp_id == 0:
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
elif warp_id == 1:
# don't need to commit
simple_tma_copy(
W_tma.atom, sW, gW_tiles[(None, global_chunk_id), None]
)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
u_f32 = _tcgen05.ld(warp_id * 32, U_tmem, "16x256b", V_dim // 8)
_tcgen05.wait_ld()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar + stage_id)
u_bf16 = cute.make_rmem_tensor((8, V_dim // 16), BFloat16)
u_bf16.store(u_f32.to(BFloat16))
cute.copy(stsm_atom, u_bf16, sU_view)
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
fence_before_tma_store()
if warp_id == 1:
simple_tma_copy(
U_tma.atom, sU, gU_tiles[(None, global_chunk_id), None]
)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
@cache
@staticmethod
def compile(H: int, Hv: int, K_dim: int, V_dim: int, num_stages: int = 2):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
num_sequences = cute.sym_int()
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
V = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
U = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
g = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
beta = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
cu_seqlens = make_fake_tensor(Int32, (num_sequences,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
kernel = Sm100ChunkUWKernel(H, Hv, K_dim, V_dim, num_stages)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
K,
V,
U,
W,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
Int32(148),
stream,
options="--enable-tvm-ffi",
)
def kkt_inv_uw_cutedsl(
K: torch.Tensor,
V: torch.Tensor,
U: torch.Tensor,
W: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
g_cu: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
total_chunks: torch.Tensor,
num_sms: int = 148,
) -> None:
_, Hv, V_dim = V.shape
_, H, K_dim = K.shape
Sm100ChunkUWKernel.compile(H, Hv, K_dim, V_dim)(
K,
V,
U,
W,
g,
beta,
g_cu,
cu_seqlens,
chunk_indices,
total_chunks,
num_sms,
)
@@ -1,632 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
from cutlass.cute.nvgpu import cpasync, warp
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import (
EVICT_FIRST,
_tcgen05,
cvt,
fence_before_tma_store,
simple_tma_copy,
)
class Sm100ChunkOKernel:
"""Compute per-token output from recurrent and intra-chunk terms.
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
P = mask((Q @ K.T) * Gamma)
O = scale * (exp(g_cu) * (Q @ H.T) + P @ V)
"""
def __init__(
self,
H: int,
Hv: int,
K_dim: int,
V_dim: int,
BT: int = 64,
num_stages: int = 2,
) -> None:
assert Hv % H == 0
assert K_dim == 128
assert V_dim == 128
assert BT == 64
self.H = H
self.Hv = Hv
self.K_dim = K_dim
self.V_dim = V_dim
self.BT = BT
self.num_stages = num_stages
self.num_warps = 10
@cute.jit
def _make_bf16_tma_args(
self,
tensor: cute.Tensor,
dim: cutlass.Constexpr[int],
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(self.BT, 1, (64, dim // 64), stages),
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, 64)),
slayout,
cta_tiler=(self.BT, 1, dim),
)
@cute.jit
def _make_h_tma_args(
self,
tensor: cute.Tensor,
op: cpasync.TmaCopyOp,
stages: cutlass.Constexpr[int],
):
num_elems = 128 // (tensor.element_type.width // 8)
swizzle_128B = cute.make_swizzle(3, 4, 3)
slayout = cute.make_layout(
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems), stages),
stride=(
0,
0,
num_elems,
(1, self.V_dim * num_elems),
self.V_dim * self.K_dim,
),
)
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
return cpasync.make_tiled_tma_atom(
op,
cute.logical_divide(tensor, (None, None, None, num_elems)),
slayout,
cta_tiler=(1, 1, self.V_dim, self.K_dim),
)
@cute.jit
def __call__(
self,
q: cute.Tensor,
k: cute.Tensor,
v_new_chunks: cute.Tensor,
h: cute.Tensor,
g_cu: cute.Tensor,
o: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
scale: Float32,
num_sms: Int32,
stream: CUstream,
):
grid = (num_sms // self.Hv, self.Hv, 1)
block = (self.num_warps * 32, 1, 1)
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
Q_tma = self._make_bf16_tma_args(q, self.K_dim, tma_g2s, self.num_stages)
K_tma = self._make_bf16_tma_args(k, self.K_dim, tma_g2s, self.num_stages)
V_tma = self._make_bf16_tma_args(
v_new_chunks, self.V_dim, tma_g2s, self.num_stages
)
H_tma = self._make_h_tma_args(h, tma_g2s, self.num_stages)
O_tma = self._make_bf16_tma_args(o, self.V_dim, tma_s2g, 1)
self.kernel(
Q_tma,
K_tma,
V_tma,
H_tma,
O_tma,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
scale,
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
@cute.kernel
def kernel(
self,
Q_tma: cpasync.TmaInfo,
K_tma: cpasync.TmaInfo,
V_tma: cpasync.TmaInfo,
H_tma: cpasync.TmaInfo,
O_tma: cpasync.TmaInfo,
g_cu: cute.Tensor,
o: cute.Tensor,
cu_seqlens: cute.Tensor,
chunk_indices: cute.Tensor,
total_chunks: cute.Tensor,
scale: Float32,
):
tid, _, _ = cute.arch.thread_idx()
bid, v_head_id, _ = cute.arch.block_idx()
grid_x, _, _ = cute.arch.grid_dim()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
BT = self.BT
K_dim = self.K_dim
V_dim = self.V_dim
num_stages = self.num_stages
heads_per_qk = self.Hv // self.H
k_head_id = v_head_id // heads_per_qk
num_global_chunks = total_chunks[0]
def allocate_tensor(smem, dtype, layout):
return smem.allocate_tensor(
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
)
smem = cutlass.utils.SmemAllocator()
sQ = allocate_tensor(smem, BFloat16, Q_tma.smem_layout)[None, 0, None, None]
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None, None]
sO = allocate_tensor(smem, BFloat16, O_tma.smem_layout)[None, 0, None, 0]
s_g_cu = smem.allocate_array(Float32, BT)
qk_full_mbar = smem.allocate_array(Int64, num_stages)
hv_full_mbar = smem.allocate_array(Int64, num_stages)
qk_empty_mbar = smem.allocate_array(Int64, num_stages)
pv_mma_mbar = smem.allocate_array(Int64, num_stages)
qk_mbar = smem.allocate_array(Int64, 1)
mask_mbar = smem.allocate_array(Int64, 1)
epi_mbar = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
qk_tmem = 0
p_tmem = 64
out_tmem = 128
qh_tmem = 256
if warp_id == 0:
with cute.arch.elect_one():
for i in cutlass.range_constexpr(num_stages):
cute.arch.mbarrier_init(qk_full_mbar + i, 1)
cute.arch.mbarrier_init(qk_empty_mbar + i, 1)
cute.arch.mbarrier_init(hv_full_mbar + i, 1)
cute.arch.mbarrier_init(pv_mma_mbar + i, 1)
cute.arch.mbarrier_init(qk_mbar, 1)
cute.arch.mbarrier_init(mask_mbar, 128)
cute.arch.mbarrier_init(epi_mbar, 128)
cute.arch.mbarrier_init_fence()
elif warp_id == 9:
cpasync.prefetch_descriptor(Q_tma.atom)
cpasync.prefetch_descriptor(K_tma.atom)
cpasync.prefetch_descriptor(V_tma.atom)
cpasync.prefetch_descriptor(H_tma.atom)
cute.arch.sync_threads()
if warp_id == 9:
# TMA warp
stage_id = 0
parity = 1
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
# copy Q and K
q_tile = cute.local_tile(
cute.domain_offset(
(bos, 0), Q_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
k_tile = cute.local_tile(
cute.domain_offset(
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
),
tiler=(BT, K_dim),
coord=(chunk_id, 0),
)
mbar = qk_full_mbar + stage_id
cute.arch.mbarrier_wait(qk_empty_mbar + stage_id, parity)
with cute.arch.elect_one():
STAGE_SIZE = BT * (K_dim + K_dim) * 2
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
simple_tma_copy(Q_tma.atom, q_tile, sQ[None, None, stage_id], mbar)
simple_tma_copy(K_tma.atom, k_tile, sK[None, None, stage_id], mbar)
# copy H and V
gH = H_tma.tma_tensor[global_chunk_id, v_head_id, None, None]
gV = cute.local_tile(
V_tma.tma_tensor[None, v_head_id, None],
tiler=(BT, V_dim),
coord=(global_chunk_id, 0),
)
mbar = hv_full_mbar + stage_id
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, parity)
with cute.arch.elect_one():
H_STAGE_SIZE = V_dim * K_dim * 2
V_STAGE_SIZE = BT * V_dim * 2
cute.arch.mbarrier_arrive_and_expect_tx(
mbar, H_STAGE_SIZE + V_STAGE_SIZE
)
simple_tma_copy(
H_tma.atom, gH, sH[None, None, stage_id], mbar, EVICT_FIRST
)
simple_tma_copy(
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
parity ^= 1
elif warp_id == 8:
# MMA warp
_tcgen05.alloc(taddr)
# LBO=BT*128 is ignored for K-major
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
qk_idesc = _tcgen05.make_bf16_idesc(BT, BT)
qh_idesc = _tcgen05.make_bf16_idesc(BT, V_dim)
pv_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
stage_id = 0
tma_parity = 0
mask_parity = 0
for global_chunk_id in range(bid, num_global_chunks, grid_x):
qaddr = sQ[None, None, stage_id].iterator.toint()
kaddr = sK[None, None, stage_id].iterator.toint()
haddr = sH[None, None, stage_id].iterator.toint()
vaddr = sV[None, None, stage_id].iterator.toint()
qdesc_base = sdesc_template | (qaddr >> 4)
kdesc_base = sdesc_template | (kaddr >> 4)
hdesc_base = sdesc_template | (haddr >> 4)
vdesc_base = sdesc_template | (vaddr >> 4)
##### 1st MMA: Q @ K.T #####
# do this first to unblock mask(QK)
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
cute.arch.mbarrier_wait(qk_full_mbar + stage_id, tma_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
_tcgen05.mma_f16(
qk_tmem, qdesc, kdesc, qk_idesc, (i > 0) or (j > 0)
)
_tcgen05.commit(qk_mbar)
##### 2nd MMA: Q @ H.T #####
cute.arch.mbarrier_wait(hv_full_mbar + stage_id, tma_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(K_dim // 64):
for j in cutlass.range_constexpr(64 // 16):
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
hdesc = hdesc_base | ((i * V_dim * 128 + j * 32) >> 4)
_tcgen05.mma_f16(
qh_tmem, qdesc, hdesc, qh_idesc, (i > 0) or (j > 0)
)
_tcgen05.commit(qk_empty_mbar + stage_id)
##### 3rd MMA: P @ V #####
# stalled by mask(QK)
cute.arch.mbarrier_wait(mask_mbar, mask_parity)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BT // 16):
vdesc = vdesc_base | ((i * 16 * 128) >> 4)
_tcgen05.mma_ts_f16(
out_tmem, p_tmem + i * 8, vdesc, pv_idesc, i > 0
)
_tcgen05.commit(pv_mma_mbar + stage_id)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
tma_parity ^= 1
mask_parity ^= 1
# wait for epilogue to finish for deallocation
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
_tcgen05.dealloc()
elif warp_id >= 4:
# masking warps
warp_id_ = warp_id % 4
tid_ = tid % 128
row0 = warp_id_ * 16 + lane_id // 4
row1 = row0 + 8
parity = 0
# for ldmatrix layout later
row_indices = cute.make_rmem_tensor(2, Int32)
row_indices[0] = warp_id_ * 16 + lane_id // 4
row_indices[1] = warp_id_ * 16 + lane_id // 4 + 8
row_indices = row_indices.load().reshape((1, 2))
col_indices = cute.make_rmem_tensor(2, Int32)
col_indices[0] = (lane_id % 4) * 2
col_indices[1] = (lane_id % 4) * 2 + 1
col_indices = col_indices.load().reshape((2, 1))
for global_chunk_id in range(bid, num_global_chunks, grid_x):
if tid_ < BT:
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
t_ = bos + chunk_id * BT + tid_
s_g_cu[tid_] = g_cu[t_, v_head_id] if t_ < eos else Float32(0.0)
# wait for QK MMA
if warp_id_ == 0:
cute.arch.mbarrier_wait(qk_mbar, parity)
cute.arch.barrier(barrier_id=1, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
qk = _tcgen05.ld(warp_id_ * 32, qk_tmem, "16x256b", BT // 8)
qk = qk.reshape((2, 2, BT // 8))
_tcgen05.wait_ld()
g_cu_rows = cute.make_rmem_tensor(2, Float32)
g_cu_rows[0] = s_g_cu[row0]
g_cu_rows[1] = s_g_cu[row1]
g_cu_rows = g_cu_rows.load().reshape((1, 2))
for i in cutlass.range_constexpr(BT // 8):
col = i * 8 + (lane_id % 4) * 2
g_cu_cols = cute.make_rmem_tensor(2, Float32)
g_cu_cols[0] = s_g_cu[col]
g_cu_cols[1] = s_g_cu[col + 1]
g_cu_cols = g_cu_cols.load().reshape((2, 1))
# apply gamma and causal mask
Gamma = cute.math.exp(g_cu_rows - g_cu_cols, fastmath=True)
tmp = qk[None, None, i] * Gamma
tmp = cute.where(row_indices >= col_indices + i * 8, tmp, 0.0)
# CuteDSL can't emit cvt.bf16x2.f32 here
attn_lo = cute.make_rmem_tensor(2, Uint32)
attn_lo[0] = cvt.fp32x2_to_bf16x2(tmp[0, 0], tmp[1, 0])
attn_lo[1] = cvt.fp32x2_to_bf16x2(tmp[0, 1], tmp[1, 1])
_tcgen05.st(warp_id_ * 32, p_tmem + i * 4, "16x128b", 1, attn_lo)
_tcgen05.wait_st()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(mask_mbar)
parity ^= 1
else:
# epilogue warps
# for ldmatrix layout later
row0 = warp_id * 16 + lane_id // 4
row1 = row0 + 8
stage_id = 0
mma_parity = 0
op = cute.nvgpu.CopyUniversalOp()
cp_4B = cute.make_copy_atom(op, BFloat16, num_bits_per_copy=32)
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=False)
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
# ldmatrix layout
# [total_seq_len, ((2, 4, WIDTH/8), V_DIM/WIDTH)]
WIDTH = 64
o_view = cute.logical_divide(
o[None, v_head_id, None],
(None, cute.make_layout((2, 4, WIDTH // 8))),
)
# select lane: [total_seq_len, 2, WIDTH/8, V_DIM/WIDTH]
o_view = o_view[None, ((None, lane_id % 4, None), None)]
for global_chunk_id in range(bid, num_global_chunks, grid_x):
seq_id = chunk_indices[global_chunk_id, 0]
chunk_id = chunk_indices[global_chunk_id, 1]
bos = cu_seqlens[seq_id]
eos = cu_seqlens[seq_id + 1]
chunk_start = bos + chunk_id * BT
full_chunk = chunk_start + BT <= eos
g_cu_rows = cute.make_rmem_tensor(2, Float32)
g_cu_rows.fill(0.0)
# load g_cu
if chunk_start + row0 < eos:
g_cu_rows[0] = cute.math.exp(
g_cu[chunk_start + row0, v_head_id], fastmath=True
)
if chunk_start + row1 < eos:
g_cu_rows[1] = cute.math.exp(
g_cu[chunk_start + row1, v_head_id], fastmath=True
)
g_cu_rows = g_cu_rows.load().reshape((1, 2, 1))
if warp_id == 0:
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, mma_parity)
elif warp_id == 3 and full_chunk:
cute.arch.cp_async_bulk_wait_group(0, read=True)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
if full_chunk:
# use TMA store: tmem->rmem->smem->gmem
for i in cutlass.range_constexpr(V_dim // WIDTH):
qh = _tcgen05.ld(
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
pv = _tcgen05.ld(
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
_tcgen05.wait_ld()
if i == V_dim // WIDTH - 1:
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar)
qh = qh.reshape((2, 2, WIDTH // 8))
pv = pv.reshape((2, 2, WIDTH // 8))
out_f32 = scale * (g_cu_rows * qh + pv)
out_bf16 = cute.make_rmem_tensor((8, WIDTH // 16), BFloat16)
out_bf16.store(out_f32.to(BFloat16).reshape((8, WIDTH // 16)))
# TODO: issue single cute.copy()
for j in cutlass.range_constexpr(WIDTH // 16):
s_row = warp_id * 16 + lane_id % 16
s_col = i * (WIDTH // 8) + j * 2 + lane_id // 16
sO_tile = cute.local_tile(sO[s_row, None], (8,), (s_col,))
cute.copy(stsm_atom, out_bf16[None, j], sO_tile)
cute.arch.barrier(barrier_id=2, number_of_threads=128)
fence_before_tma_store()
if warp_id == 3:
gO = cute.local_tile(
cute.domain_offset(
(bos, 0), O_tma.tma_tensor[None, v_head_id, None]
),
tiler=(BT, V_dim),
coord=(chunk_id, 0),
)
simple_tma_copy(O_tma.atom, sO, gO)
with cute.arch.elect_one():
cute.arch.cp_async_bulk_commit_group()
else:
# direct gmem store
# TODO: explore doing multiple 1D TMAs
for i in cutlass.range_constexpr(V_dim // WIDTH):
qh = _tcgen05.ld(
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
pv = _tcgen05.ld(
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
)
_tcgen05.wait_ld()
if i == V_dim // WIDTH - 1:
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(epi_mbar)
qh = qh.reshape((2, 2, WIDTH // 8))
pv = pv.reshape((2, 2, WIDTH // 8))
out_f32 = scale * (g_cu_rows * qh + pv)
out_bf16 = cute.make_rmem_tensor((2, 2, WIDTH // 8), BFloat16)
out_bf16.store(out_f32.to(BFloat16))
if chunk_start + row0 < eos:
cute.copy(
cp_4B,
out_bf16[None, 0, None],
o_view[chunk_start + row0, None, None, i],
)
if chunk_start + row1 < eos:
cute.copy(
cp_4B,
out_bf16[None, 1, None],
o_view[chunk_start + row1, None, None, i],
)
stage_id = (stage_id + 1) % num_stages
if stage_id == 0:
mma_parity ^= 1
@cache
@staticmethod
def compile(
H: int,
Hv: int,
K_dim: int,
V_dim: int,
BT: int = 64,
num_stages: int = 2,
):
total_t = cute.sym_int()
pad_t = cute.sym_int()
total_chunks_n = cute.sym_int()
cu_entries = cute.sym_int()
q = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
k = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
v_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
h = make_fake_tensor(
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
)
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
o = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
kernel = Sm100ChunkOKernel(
H,
Hv,
K_dim,
V_dim,
BT,
num_stages,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
q,
k,
v_new,
h,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
Float32(1.0),
Int32(148),
stream,
options="--enable-tvm-ffi",
)
def o_cutedsl(
q: torch.Tensor,
k: torch.Tensor,
v_new: torch.Tensor,
h: torch.Tensor,
g_cu: torch.Tensor,
o: torch.Tensor,
cu_seqlens: torch.Tensor,
chunk_indices: torch.Tensor,
total_chunks: torch.Tensor,
scale: float,
num_sms: int = 148,
) -> None:
_, H, K_dim = q.shape
_, Hv, V_dim = o.shape
Sm100ChunkOKernel.compile(H, Hv, K_dim, V_dim)(
q,
k,
v_new,
h,
g_cu,
o,
cu_seqlens,
chunk_indices,
total_chunks,
float(scale),
num_sms,
)