[ROCm][Perf] Enable the AITER GDN decode fast path for flat qkvz layouts (#53623)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Cursor
TJian
parent
211e252d0b
commit
fbe8a157fb
@@ -399,6 +399,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.key_dim = self.head_k_dim * self.num_k_heads
|
||||
self.value_dim = self.head_v_dim * self.num_v_heads
|
||||
self.gqa_interleaved_layout = gqa_interleaved_layout
|
||||
self.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat"
|
||||
if current_platform.is_xpu():
|
||||
self._forward_method = self.forward_xpu
|
||||
elif current_platform.is_cpu():
|
||||
@@ -1206,9 +1207,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"""ROCm AITER fast path: conv1d + recurrent attention from packed
|
||||
qkvz/ba layout.
|
||||
|
||||
For decode-only (no spec, no prefill) interleaved-GQA layouts,
|
||||
dispatches directly to ``_forward_core_decode_aiter``. Otherwise unpacks
|
||||
the packed layout and falls through to ``_forward_core``.
|
||||
For decode-only (no spec, no prefill) batches, dispatches directly to
|
||||
``_forward_core_decode_aiter``. Otherwise unpacks the packed layout and
|
||||
falls through to ``_forward_core``.
|
||||
|
||||
Args:
|
||||
qkvz: packed [q, k, v, z] projection (num_tokens, qkvz_dim)
|
||||
@@ -1231,12 +1232,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
|
||||
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
|
||||
# GQA layout. Qwen3.5 uses a non-interleaved q/k/v/z layout and must use
|
||||
# the generic path below to split/rearrange inputs correctly.
|
||||
if (
|
||||
self.gqa_interleaved_layout
|
||||
and attn_metadata.spec_sequence_masks is None
|
||||
attn_metadata.spec_sequence_masks is None
|
||||
and attn_metadata.num_prefills == 0
|
||||
and attn_metadata.num_decodes > 0
|
||||
):
|
||||
@@ -1625,6 +1622,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
: attn_metadata.num_actual_tokens
|
||||
],
|
||||
validate_data=True,
|
||||
qkvz_layout=self.qkvz_layout,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user