[Aiter][ROCm] gdn_linear_attn kernel fusion (#40711)
Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Chuan Li <chuali@amd.com> Co-authored-by: hellozhuo <zhuo.su@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -32,6 +32,7 @@ def chunk_gated_delta_rule_fwd(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
g = chunk_local_cumsum(
|
||||
g, chunk_size=FLA_CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
|
||||
@@ -77,6 +78,7 @@ def chunk_gated_delta_rule_fwd(
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
if SUPPRESS_LEVEL < 3:
|
||||
return g, o, A, final_state, None, None, None
|
||||
@@ -102,6 +104,7 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q)
|
||||
@@ -119,9 +122,15 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
ctx.scale = scale
|
||||
ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel
|
||||
if core_attn_out is not None:
|
||||
assert not torch.is_grad_enabled(), (
|
||||
"core_attn_out buffer reuse is only supported for inference"
|
||||
)
|
||||
assert q.dtype == o.dtype, "Incompatible dtype for inplace computation"
|
||||
return o.to(q.dtype), final_state
|
||||
|
||||
|
||||
@@ -139,6 +148,7 @@ def chunk_gated_delta_rule(
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
r"""
|
||||
Args:
|
||||
@@ -230,5 +240,6 @@ def chunk_gated_delta_rule(
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
use_qk_l2norm_in_kernel,
|
||||
core_attn_out,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
@@ -148,6 +148,7 @@ def chunk_fwd_o(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
B, T, Hg, K, V = *q.shape, v.shape[-1]
|
||||
H = v.shape[-2]
|
||||
@@ -158,7 +159,13 @@ def chunk_fwd_o(
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
o = torch.empty_like(v)
|
||||
if core_attn_out is not None:
|
||||
assert core_attn_out.numel() >= v.numel(), (
|
||||
f"core_attn_out too small: {core_attn_out.numel()} < {v.numel()}"
|
||||
)
|
||||
o = core_attn_out[: v.numel()].view(*v.shape)
|
||||
else:
|
||||
o = torch.empty_like(v)
|
||||
|
||||
def grid(meta):
|
||||
return (triton.cdiv(V, meta["BV"]), NT, B * H)
|
||||
|
||||
@@ -8,6 +8,7 @@ from torch import nn
|
||||
from transformers.activations import ACT2FN
|
||||
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
@@ -64,6 +65,20 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
# Optional ROCm AITER Triton kernels for the GDN decode fast-path.
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
# references are imported here so that they can be called without per-call
|
||||
# import overhead.
|
||||
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
from aiter.ops.triton.causal_conv1d_update_single_token import (
|
||||
fused_reshape_causal_conv1d_update_single_token as gdn_aiter_fused_reshape_causal_conv1d_update_single_token, # noqa: E501
|
||||
)
|
||||
from aiter.ops.triton.gated_delta_net.fused_rearrange_sigmoid_gdr import (
|
||||
fused_rearrange_sigmoid_gated_delta_rule as gdn_aiter_fused_rearrange_sigmoid_gated_delta_rule, # noqa: E501
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -169,8 +184,9 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
return fi_chunk_gated_delta_rule(
|
||||
o, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
@@ -181,6 +197,11 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
if core_attn_out is not None:
|
||||
o_flat = o.squeeze(0).reshape(-1)
|
||||
co_flat = core_attn_out.reshape(-1)
|
||||
co_flat[: o_flat.numel()].copy_(o_flat)
|
||||
return o, final_state
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -195,6 +216,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
return fla_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -208,6 +230,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
|
||||
@@ -271,7 +294,6 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
else 0
|
||||
)
|
||||
self.gqa_interleaved_layout = gqa_interleaved_layout
|
||||
self._forward_method = self.forward_cuda
|
||||
if current_platform.is_xpu():
|
||||
self._forward_method = self.forward_xpu
|
||||
elif current_platform.is_cpu():
|
||||
@@ -281,6 +303,10 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
|
||||
register_cpu_gdn_attention_ops()
|
||||
self._forward_method = self.forward_cpu
|
||||
elif current_platform.is_rocm():
|
||||
self._forward_method = self.forward_hip
|
||||
else:
|
||||
self._forward_method = self.forward_cuda
|
||||
|
||||
# QKV
|
||||
self.conv_dim = self.key_dim * 2 + self.value_dim
|
||||
@@ -297,6 +323,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
# we need to create qkvz_proj adaptively here.
|
||||
# When create_in_proj_qkvz is False (e.g. LoRA enabled in Qwen3.5),
|
||||
# in_proj_qkv and in_proj_z are created separately instead.
|
||||
self.has_lora_projections = not create_in_proj_qkvz
|
||||
if create_in_proj_qkvz:
|
||||
self.in_proj_qkvz = self.create_qkvz_proj(
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -497,24 +524,155 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
|
||||
return query, key, value, z, b, a
|
||||
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
if mixed_qkv is None:
|
||||
return None, None, None
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
@torch.compile(fullgraph=True)
|
||||
def prepare_gdn_attention_core_inputs(
|
||||
self,
|
||||
mixed_qkvz: torch.Tensor,
|
||||
mixed_ba: torch.Tensor,
|
||||
num_tokens: int,
|
||||
):
|
||||
"""
|
||||
Derives mixed_qkv, z, b, a from projected qkvz/ba for the GDN custom op.
|
||||
|
||||
For gqa_interleaved_layout (Qwen3-Next): unpack the interleaved
|
||||
[ng, (hk + hk + np/ng*hv + np/ng*hv)] layout into contiguous qkv.
|
||||
For non-interleaved layout (Qwen3.5): simple split along last dim.
|
||||
"""
|
||||
if not self.gqa_interleaved_layout:
|
||||
# Qwen3.5: weights are in [q, k, v, z] order
|
||||
assert num_tokens == mixed_qkvz.shape[0]
|
||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
z_size = self.value_dim // self.tp_size
|
||||
mixed_qkv, z_flat = mixed_qkvz.split([qkv_size, z_size], dim=-1)
|
||||
n = mixed_qkvz.shape[0]
|
||||
z_out = z_flat.reshape(n, -1, self.head_v_dim)
|
||||
b, a = mixed_ba.chunk(2, dim=-1)
|
||||
return mixed_qkv, z_out, b, a
|
||||
|
||||
# Qwen3-Next: interleaved GQA layout
|
||||
base_shape_qkvz = mixed_qkvz.size()[:-1]
|
||||
base_shape_ba = mixed_ba.size()[:-1]
|
||||
ng = self.num_k_heads // self.tp_size
|
||||
|
||||
new_tensor_shape_qkvz = base_shape_qkvz + (
|
||||
ng,
|
||||
(
|
||||
self.head_k_dim
|
||||
+ self.head_k_dim
|
||||
+ (self.head_v_dim + self.head_v_dim)
|
||||
* self.num_v_heads
|
||||
// self.num_k_heads
|
||||
),
|
||||
)
|
||||
new_tensor_shape_ba = base_shape_ba + (
|
||||
ng,
|
||||
2 * self.num_v_heads // self.num_k_heads,
|
||||
)
|
||||
|
||||
mixed_qkvz = mixed_qkvz.view(*new_tensor_shape_qkvz)
|
||||
mixed_ba = mixed_ba.view(*new_tensor_shape_ba)
|
||||
|
||||
split_arg_list_qkvz = [
|
||||
self.head_k_dim,
|
||||
self.head_k_dim,
|
||||
(self.num_v_heads // self.num_k_heads * self.head_v_dim),
|
||||
(self.num_v_heads // self.num_k_heads * self.head_v_dim),
|
||||
]
|
||||
split_arg_list_ba = [
|
||||
self.num_v_heads // self.num_k_heads,
|
||||
self.num_v_heads // self.num_k_heads,
|
||||
]
|
||||
|
||||
(query, key, value, z) = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=-1)
|
||||
(b, a) = torch.split(mixed_ba, split_arg_list_ba, dim=-1)
|
||||
|
||||
mixed_qkv_logical = torch.cat(
|
||||
[
|
||||
self.key_dim // self.tp_size,
|
||||
self.key_dim // self.tp_size,
|
||||
self.value_dim // self.tp_size,
|
||||
query.reshape(num_tokens, -1),
|
||||
key.reshape(num_tokens, -1),
|
||||
value.reshape(num_tokens, -1),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
query, key = map(
|
||||
lambda x: rearrange(x, "l (h d) -> 1 l h d", d=self.head_k_dim),
|
||||
(query, key),
|
||||
|
||||
# The split above produces non-contiguous views into the interleaved
|
||||
# buffer. Concatenating everything into a single flat tensor forces a
|
||||
# contiguous copy, then slicing back out gives contiguous q/k/v/z/b/a
|
||||
# tensors that downstream kernels require. Doing this in one cat+slice
|
||||
# keeps torch.compile in a single Triton graph instead of emitting
|
||||
# separate copy kernels per tensor. The original code used
|
||||
# rearrange(...).contiguous() on each tensor individually.
|
||||
fused = torch.cat(
|
||||
[
|
||||
mixed_qkv_logical.reshape(-1),
|
||||
z.reshape(-1),
|
||||
b.reshape(-1),
|
||||
a.reshape(-1),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
value = rearrange(value, "l (h d) -> 1 l h d", d=self.head_v_dim)
|
||||
return query.contiguous(), key.contiguous(), value.contiguous()
|
||||
|
||||
curr = 0
|
||||
qkv_numel = mixed_qkv_logical.numel()
|
||||
z_numel = z.numel()
|
||||
b_numel = b.numel()
|
||||
a_numel = a.numel()
|
||||
|
||||
mixed_qkv_out = fused[curr : curr + qkv_numel].view(num_tokens, -1)
|
||||
curr += qkv_numel
|
||||
|
||||
z_out = fused[curr : curr + z_numel].view(
|
||||
num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim
|
||||
)
|
||||
curr += z_numel
|
||||
|
||||
b_out = fused[curr : curr + b_numel].view(
|
||||
num_tokens, self.num_v_heads // self.tp_size
|
||||
)
|
||||
curr += b_numel
|
||||
|
||||
a_out = fused[curr : curr + a_numel].view(
|
||||
num_tokens, self.num_v_heads // self.tp_size
|
||||
)
|
||||
|
||||
return mixed_qkv_out, z_out, b_out, a_out
|
||||
|
||||
@torch.compile(fullgraph=True)
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
"""Split packed qkv into contiguous (1, seq, heads, dim) tensors.
|
||||
|
||||
The original code used ``rearrange(x, "l (h d) -> 1 l h d", d=...)``
|
||||
followed by ``.contiguous()`` on each tensor. This version flattens
|
||||
all three splits into a single buffer via ``torch.cat`` so that
|
||||
torch.compile emits one Triton copy kernel instead of three separate
|
||||
contiguous() calls.
|
||||
"""
|
||||
if mixed_qkv is None:
|
||||
return None, None, None
|
||||
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
q_dim = self.key_dim // self.tp_size
|
||||
k_dim = self.key_dim // self.tp_size
|
||||
v_dim = self.value_dim // self.tp_size
|
||||
|
||||
query, key, value = torch.split(mixed_qkv, [q_dim, k_dim, v_dim], dim=-1)
|
||||
|
||||
fused = torch.cat(
|
||||
[query.reshape(-1), key.reshape(-1), value.reshape(-1)], dim=0
|
||||
)
|
||||
|
||||
q_size = seq_len * q_dim
|
||||
k_size = seq_len * k_dim
|
||||
|
||||
q_contig = fused[0:q_size]
|
||||
k_contig = fused[q_size : q_size + k_size]
|
||||
v_contig = fused[q_size + k_size :]
|
||||
|
||||
query = q_contig.view(1, seq_len, -1, self.head_k_dim)
|
||||
key = k_contig.view(1, seq_len, -1, self.head_k_dim)
|
||||
value = v_contig.view(1, seq_len, -1, self.head_v_dim)
|
||||
|
||||
return query, key, value
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -523,6 +681,63 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
):
|
||||
self._forward_method(hidden_states, output)
|
||||
|
||||
def _output_projection(
|
||||
self,
|
||||
core_attn_out: torch.Tensor,
|
||||
z: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
num_tokens: int,
|
||||
):
|
||||
"""Part 3: RMSNormGated + output linear projection.
|
||||
|
||||
The RMSNormGated + quant sequence is eligible for fusion
|
||||
by the compilation pass when fuse_norm_quant is enabled.
|
||||
"""
|
||||
z_shape_og = z.shape
|
||||
core_attn_out = core_attn_out.reshape(-1, core_attn_out.shape[-1])
|
||||
z = z.reshape(-1, z.shape[-1])
|
||||
core_attn_out = self.norm(core_attn_out, z)
|
||||
core_attn_out = core_attn_out.reshape(z_shape_og)
|
||||
core_attn_out = rearrange(core_attn_out, "... h d -> ... (h d)")
|
||||
output[:num_tokens], _ = self.out_proj(core_attn_out)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
if not self.has_lora_projections and GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
projected_states_ba, _ = self.in_proj_ba(hidden_states)
|
||||
projected_states_qkvz = projected_states_qkvz.view(num_tokens, -1)
|
||||
projected_states_ba = projected_states_ba.view(num_tokens, -1)
|
||||
core_attn_out = torch.empty(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
z = torch.empty(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=projected_states_qkvz.dtype,
|
||||
device=projected_states_qkvz.device,
|
||||
)
|
||||
|
||||
torch.ops.vllm.gdn_attention_core(
|
||||
projected_states_qkvz,
|
||||
projected_states_ba,
|
||||
z,
|
||||
core_attn_out,
|
||||
fast_kernel=True,
|
||||
layer_name=_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
self._output_projection(core_attn_out, z, output, num_tokens)
|
||||
else:
|
||||
self.forward_cuda(hidden_states, output)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -538,7 +753,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
# ============================================================
|
||||
# Part 1: Input Projection
|
||||
# ============================================================
|
||||
if hasattr(self, "in_proj_qkv"):
|
||||
if self.has_lora_projections:
|
||||
# LoRA path (Qwen3.5 only): separate in_proj_qkv and in_proj_z
|
||||
mixed_qkv, _ = self.in_proj_qkv(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
@@ -586,20 +801,14 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
b,
|
||||
a,
|
||||
core_attn_out,
|
||||
_encode_layer_name(self.prefix),
|
||||
fast_kernel=False,
|
||||
layer_name=_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Part 3: Output Projection
|
||||
# ============================================================
|
||||
z_shape_og = z.shape
|
||||
# Reshape input data into 2D tensor
|
||||
core_attn_out = core_attn_out.reshape(-1, core_attn_out.shape[-1])
|
||||
z = z.reshape(-1, z.shape[-1])
|
||||
core_attn_out = self.norm(core_attn_out, z)
|
||||
core_attn_out = core_attn_out.reshape(z_shape_og)
|
||||
core_attn_out = rearrange(core_attn_out, "... h d -> ... (h d)")
|
||||
output[:num_tokens], _ = self.out_proj(core_attn_out)
|
||||
self._output_projection(core_attn_out, z, output, num_tokens)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
@@ -614,7 +823,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
"""
|
||||
num_tokens = hidden_states.size(0)
|
||||
|
||||
assert not hasattr(self, "in_proj_qkv"), "lora isn't supported on XPU."
|
||||
assert not self.has_lora_projections, "lora isn't supported on XPU."
|
||||
|
||||
# ============================================================
|
||||
# Part 1: Input Projection
|
||||
@@ -702,7 +911,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
core_attn_out = rearrange(core_attn_out, "... h d -> ... (h d)")
|
||||
output[:num_tokens], _ = self.out_proj(core_attn_out)
|
||||
|
||||
def _warmup_prefill_kernels(self, mixed_qkv: torch.Tensor) -> None:
|
||||
def _warmup_prefill_kernels(self, qkv_or_qkvz: torch.Tensor, v_dim: int) -> None:
|
||||
"""Warm up GDN prefill kernels during V1 profiling.
|
||||
|
||||
During V1 profile runs, ``_forward_core`` returns early because
|
||||
@@ -723,7 +932,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
``BT = chunk_size`` (64). A single warmup pass with T = 64
|
||||
is sufficient to populate the autotuner cache.
|
||||
|
||||
The decode path uses ``fused_sigmoid_gating_delta_rule_update``
|
||||
The decode path uses ``gdn_aiter_fused_rearrange_sigmoid_gated_delta_rule``
|
||||
which has fixed kernel parameters (no autotuning), so only the
|
||||
prefill (chunked) path needs warming up.
|
||||
"""
|
||||
@@ -731,8 +940,8 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
return
|
||||
self._prefill_kernels_warmed_up = True
|
||||
|
||||
device = mixed_qkv.device
|
||||
dtype = mixed_qkv.dtype
|
||||
device = qkv_or_qkvz.device
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
num_k_heads = self.num_k_heads // self.tp_size
|
||||
num_v_heads = self.num_v_heads // self.tp_size
|
||||
_, state_dtype = self.get_state_dtype()
|
||||
@@ -743,7 +952,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
# then run chunk_gated_delta_rule with in-kernel L2 norm disabled.
|
||||
T = FLA_CHUNK_SIZE
|
||||
dummy_mixed_qkv = torch.randn(
|
||||
T, mixed_qkv.shape[-1], device=device, dtype=dtype
|
||||
T, qkv_or_qkvz.shape[-1] - v_dim, device=device, dtype=dtype
|
||||
)
|
||||
dummy_a = torch.randn(T, num_v_heads, device=device, dtype=dtype)
|
||||
dummy_b = torch.randn(T, num_v_heads, device=device, dtype=dtype)
|
||||
@@ -806,6 +1015,66 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def _forward_core_rocm(
|
||||
self,
|
||||
qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
z_out: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""ROCm AITER fast path: conv1d + recurrent attention from packed
|
||||
qkvz/ba layout.
|
||||
|
||||
For decode-only (no spec, no prefill), dispatches directly to
|
||||
``_forward_core_decode_fast``. Otherwise unpacks the packed
|
||||
layout and falls through to ``_forward_core``.
|
||||
|
||||
Args:
|
||||
qkvz: packed [q, k, v, z] projection (num_tokens, qkvz_dim)
|
||||
ba: packed [b, a] gating vectors (num_tokens, 2*num_heads)
|
||||
z_out: **output** buffer for z (num_tokens, num_heads,
|
||||
head_dim); mutated in-place.
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
|
||||
self._warmup_prefill_kernels(qkvz, v_dim)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
|
||||
if (
|
||||
attn_metadata.spec_sequence_masks is None
|
||||
and attn_metadata.num_prefills == 0
|
||||
and attn_metadata.num_decodes > 0
|
||||
):
|
||||
return self._forward_core_decode_fast(
|
||||
qkvz=qkvz,
|
||||
ba=ba,
|
||||
z_out=z_out,
|
||||
core_attn_out=core_attn_out,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
core_attn_out.zero_()
|
||||
z_out.zero_()
|
||||
num_tokens_all = qkvz.shape[0]
|
||||
mixed_qkv, z, b, a = self.prepare_gdn_attention_core_inputs(
|
||||
qkvz, ba, num_tokens_all
|
||||
)
|
||||
z_out[:] = z
|
||||
self._forward_core(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
def _forward_core(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
@@ -813,13 +1082,19 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""Core conv1d + recurrent attention (standard path).
|
||||
|
||||
Args:
|
||||
mixed_qkv: packed [q, k, v] projection (num_tokens, qkv_dim)
|
||||
b: beta gating vector (num_tokens, num_heads)
|
||||
a: alpha gating vector (num_tokens, num_heads)
|
||||
core_attn_out: Pre-allocated output buffer for attention results.
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
# V1 profile run — warm up prefill kernels so that
|
||||
# autotuning completes before KV cache allocation.
|
||||
self._warmup_prefill_kernels(mixed_qkv)
|
||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
@@ -1065,6 +1340,72 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
else:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||
|
||||
def _forward_core_decode_fast(
|
||||
self,
|
||||
qkvz: torch.Tensor,
|
||||
ba: torch.Tensor,
|
||||
z_out: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
):
|
||||
non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501
|
||||
self_kv_cache = self.kv_cache
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
conv_state = (
|
||||
self_kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self_kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self_kv_cache[1]
|
||||
|
||||
# 1. Convolution sequence transformation
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
mixed_qkv_non_spec, b, a = (
|
||||
gdn_aiter_fused_reshape_causal_conv1d_update_single_token(
|
||||
qkvz,
|
||||
attn_metadata.num_actual_tokens,
|
||||
self.num_k_heads // self.tp_size,
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.head_k_dim,
|
||||
self.head_v_dim,
|
||||
ba,
|
||||
z_out,
|
||||
core_attn_out,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=non_spec_state_indices_tensor[ # type: ignore[index]
|
||||
: attn_metadata.num_actual_tokens
|
||||
],
|
||||
validate_data=True,
|
||||
)
|
||||
)
|
||||
|
||||
# 2. Recurrent attention
|
||||
gdn_aiter_fused_rearrange_sigmoid_gated_delta_rule(
|
||||
A_log=self.A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=self.dt_bias,
|
||||
qkv=mixed_qkv_non_spec,
|
||||
key_dim=self.key_dim // self.tp_size,
|
||||
value_dim=self.value_dim // self.tp_size,
|
||||
head_k_dim=self.head_k_dim,
|
||||
head_v_dim=self.head_v_dim,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc[: attn_metadata.num_decodes + 1], # type: ignore[index]
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
core_attn_out=core_attn_out.reshape(-1),
|
||||
)
|
||||
|
||||
def _forward_core_decode_non_spec(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
@@ -1121,33 +1462,51 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
|
||||
|
||||
def gdn_attention_core(
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
qkv_or_qkvz: torch.Tensor,
|
||||
b_or_ba: torch.Tensor,
|
||||
a_or_z_out: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
fast_kernel: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""
|
||||
Custom op for the core attention computation.
|
||||
Only handles the convolution + recurrent attention part.
|
||||
Input/output projections are handled outside this op.
|
||||
"""Custom op dispatching to _forward_core or _forward_core_rocm.
|
||||
|
||||
Handles conv1d + recurrent attention only; input/output projections
|
||||
are performed by the caller.
|
||||
|
||||
When ``fast_kernel=False`` (standard path):
|
||||
qkv_or_qkvz is [q, k, v], b_or_ba is b, a_or_z_out is a (read-only).
|
||||
When ``fast_kernel=True`` (AITER Triton fast path, ROCm only):
|
||||
qkv_or_qkvz is [q, k, v, z], b_or_ba is [b, a], a_or_z_out is the
|
||||
z output buffer (mutated in-place).
|
||||
|
||||
``core_attn_out`` is always mutated in-place.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward_core(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
if fast_kernel:
|
||||
self._forward_core_rocm(
|
||||
qkvz=qkv_or_qkvz,
|
||||
ba=b_or_ba,
|
||||
z_out=a_or_z_out,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
else:
|
||||
self._forward_core(
|
||||
mixed_qkv=qkv_or_qkvz,
|
||||
b=b_or_ba,
|
||||
a=a_or_z_out,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
|
||||
def gdn_attention_core_fake(
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
qkv_or_qkvz: torch.Tensor,
|
||||
b_or_ba: torch.Tensor,
|
||||
a_or_z_out: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
fast_kernel: bool,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Fake implementation for torch.compile."""
|
||||
@@ -1157,7 +1516,7 @@ def gdn_attention_core_fake(
|
||||
direct_register_custom_op(
|
||||
op_name="gdn_attention_core",
|
||||
op_func=gdn_attention_core,
|
||||
mutates_args=["core_attn_out"],
|
||||
mutates_args=["a_or_z_out", "core_attn_out"],
|
||||
fake_impl=gdn_attention_core_fake,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user