[Aiter][ROCm] gdn_linear_attn kernel fusion (#40711)

Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Chuan Li <chuali@amd.com>
Co-authored-by: hellozhuo <zhuo.su@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Tres
2026-05-07 23:11:37 -07:00
committed by GitHub
co-authored by hellozhuo Cursor
parent 1acd67a795
commit ed582b6a4c
4 changed files with 448 additions and 52 deletions
@@ -32,6 +32,7 @@ def chunk_gated_delta_rule_fwd(
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
core_attn_out: torch.Tensor | None = None,
):
g = chunk_local_cumsum(
g, chunk_size=FLA_CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
@@ -77,6 +78,7 @@ def chunk_gated_delta_rule_fwd(
scale=scale,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
core_attn_out=core_attn_out,
)
if SUPPRESS_LEVEL < 3:
return g, o, A, final_state, None, None, None
@@ -102,6 +104,7 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = False,
core_attn_out: torch.Tensor | None = None,
):
if use_qk_l2norm_in_kernel:
q = l2norm_fwd(q)
@@ -119,9 +122,15 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
core_attn_out=core_attn_out,
)
ctx.scale = scale
ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel
if core_attn_out is not None:
assert not torch.is_grad_enabled(), (
"core_attn_out buffer reuse is only supported for inference"
)
assert q.dtype == o.dtype, "Incompatible dtype for inplace computation"
return o.to(q.dtype), final_state
@@ -139,6 +148,7 @@ def chunk_gated_delta_rule(
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = False,
core_attn_out: torch.Tensor | None = None,
):
r"""
Args:
@@ -230,5 +240,6 @@ def chunk_gated_delta_rule(
chunk_indices,
chunk_offsets,
use_qk_l2norm_in_kernel,
core_attn_out,
)
return o, final_state
@@ -148,6 +148,7 @@ def chunk_fwd_o(
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_size: int = FLA_CHUNK_SIZE,
core_attn_out: torch.Tensor | None = None,
) -> torch.Tensor:
B, T, Hg, K, V = *q.shape, v.shape[-1]
H = v.shape[-2]
@@ -158,7 +159,13 @@ def chunk_fwd_o(
if scale is None:
scale = k.shape[-1] ** -0.5
o = torch.empty_like(v)
if core_attn_out is not None:
assert core_attn_out.numel() >= v.numel(), (
f"core_attn_out too small: {core_attn_out.numel()} < {v.numel()}"
)
o = core_attn_out[: v.numel()].view(*v.shape)
else:
o = torch.empty_like(v)
def grid(meta):
return (triton.cdiv(V, meta["BV"]), NT, B * H)