[ROCm][Perf] Enable the AITER GDN decode fast path for flat qkvz layouts (#53623)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
Mikko Tukiainen
2026-09-19 06:05:04 -07:00
committed by GitHub
co-authored by Cursor TJian
parent 211e252d0b
commit fbe8a157fb
4 changed files with 132 additions and 8 deletions
@@ -399,6 +399,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.key_dim = self.head_k_dim * self.num_k_heads
self.value_dim = self.head_v_dim * self.num_v_heads
self.gqa_interleaved_layout = gqa_interleaved_layout
self.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat"
if current_platform.is_xpu():
self._forward_method = self.forward_xpu
elif current_platform.is_cpu():
@@ -1206,9 +1207,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
"""ROCm AITER fast path: conv1d + recurrent attention from packed
qkvz/ba layout.
For decode-only (no spec, no prefill) interleaved-GQA layouts,
dispatches directly to ``_forward_core_decode_aiter``. Otherwise unpacks
the packed layout and falls through to ``_forward_core``.
For decode-only (no spec, no prefill) batches, dispatches directly to
``_forward_core_decode_aiter``. Otherwise unpacks the packed layout and
falls through to ``_forward_core``.
Args:
qkvz: packed [q, k, v, z] projection (num_tokens, qkvz_dim)
@@ -1231,12 +1232,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
assert isinstance(attn_metadata, GDNAttentionMetadata)
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
# GQA layout. Qwen3.5 uses a non-interleaved q/k/v/z layout and must use
# the generic path below to split/rearrange inputs correctly.
if (
self.gqa_interleaved_layout
and attn_metadata.spec_sequence_masks is None
attn_metadata.spec_sequence_masks is None
and attn_metadata.num_prefills == 0
and attn_metadata.num_decodes > 0
):
@@ -1625,6 +1622,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
: attn_metadata.num_actual_tokens
],
validate_data=True,
qkvz_layout=self.qkvz_layout,
)
)