[Aiter][ROCm] gdn_linear_attn kernel fusion (#40711)
Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Chuan Li <chuali@amd.com> Co-authored-by: hellozhuo <zhuo.su@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -32,6 +32,7 @@ def chunk_gated_delta_rule_fwd(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
g = chunk_local_cumsum(
|
||||
g, chunk_size=FLA_CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
|
||||
@@ -77,6 +78,7 @@ def chunk_gated_delta_rule_fwd(
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
if SUPPRESS_LEVEL < 3:
|
||||
return g, o, A, final_state, None, None, None
|
||||
@@ -102,6 +104,7 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
if use_qk_l2norm_in_kernel:
|
||||
q = l2norm_fwd(q)
|
||||
@@ -119,9 +122,15 @@ class ChunkGatedDeltaRuleFunction(torch.autograd.Function):
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
ctx.scale = scale
|
||||
ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel
|
||||
if core_attn_out is not None:
|
||||
assert not torch.is_grad_enabled(), (
|
||||
"core_attn_out buffer reuse is only supported for inference"
|
||||
)
|
||||
assert q.dtype == o.dtype, "Incompatible dtype for inplace computation"
|
||||
return o.to(q.dtype), final_state
|
||||
|
||||
|
||||
@@ -139,6 +148,7 @@ def chunk_gated_delta_rule(
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
):
|
||||
r"""
|
||||
Args:
|
||||
@@ -230,5 +240,6 @@ def chunk_gated_delta_rule(
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
use_qk_l2norm_in_kernel,
|
||||
core_attn_out,
|
||||
)
|
||||
return o, final_state
|
||||
|
||||
@@ -148,6 +148,7 @@ def chunk_fwd_o(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_size: int = FLA_CHUNK_SIZE,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
B, T, Hg, K, V = *q.shape, v.shape[-1]
|
||||
H = v.shape[-2]
|
||||
@@ -158,7 +159,13 @@ def chunk_fwd_o(
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
o = torch.empty_like(v)
|
||||
if core_attn_out is not None:
|
||||
assert core_attn_out.numel() >= v.numel(), (
|
||||
f"core_attn_out too small: {core_attn_out.numel()} < {v.numel()}"
|
||||
)
|
||||
o = core_attn_out[: v.numel()].view(*v.shape)
|
||||
else:
|
||||
o = torch.empty_like(v)
|
||||
|
||||
def grid(meta):
|
||||
return (triton.cdiv(V, meta["BV"]), NT, B * H)
|
||||
|
||||
Reference in New Issue
Block a user