[ROCm][Perf] Add AITER FlyDSL GDN prefill backend (#57560)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-07 09:44:22 -05:00
committed by GitHub
co-authored by Cursor
parent f6e3944e31
commit 548597367e
5 changed files with 462 additions and 12 deletions
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
"""Resolve GDN prefill backend.
FlashInfer's GDN prefill kernel is chosen when:
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
In-tree CuteDSL GDN prefill kernel is chosen when:
* "cutedsl" is requested; (opt-in only)
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
additional_config = vllm_config.additional_config
backend_cfg = (
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
)
backend = str(backend_cfg).strip().lower()
if not current_platform.is_cuda():
return backend, "triton"
head_k_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
)
head_v_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
)
if current_platform.is_rocm():
if backend == "aiter_flydsl":
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but the "
"AITER FlyDSL prefill kernels are not importable. Check "
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
)
return backend, "triton"
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
f"AITER backend and the current platform is {current_platform.device_name}."
)
if not current_platform.is_cuda():
return backend, "triton"
supports_flashinfer = False
supports_cutedsl = False
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
chosen = {
"flashinfer": "FlashInfer",
"cutedsl": "CuteDSL",
"aiter_flydsl": "AITER FlyDSL",
"triton": "Triton/FLA",
}[active_backend]
logger.info_once(
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
self._forward_method = self.forward_cuda
elif active_backend == "cutedsl":
self._forward_method = self.forward_cutedsl
elif active_backend == "aiter_flydsl":
self._forward_method = self.forward_aiter_flydsl
else:
self._forward_method = self.forward_native
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = fi_chunk_gated_delta_rule(
q=q,
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
return fla_chunk_gated_delta_rule(
q=q,
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
chunk_gated_delta_rule_cutedsl,
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
final_state = None
return o, final_state
def forward_aiter_flydsl(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
aiter_prefill_metadata=aiter_prefill_metadata,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
)
if core_attn_out is not None:
o_flat = o.reshape(-1)
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
return o, final_state
@PluggableLayer.register("qwen_gated_delta_net_attention")
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
@@ -1132,12 +1214,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
# CuteDSL kernels require metadata
chunk_indices = None
chunk_offsets = None
aiter_prefill_metadata = None
if self.gdn_prefill_backend == "cutedsl":
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
prepare_metadata_cutedsl,
)
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
elif self.gdn_prefill_backend == "aiter_flydsl":
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
[T],
cu_seqlens=cu_seqlens,
)
try:
self.chunk_gated_delta_rule(
@@ -1152,6 +1240,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
except Exception:
logger.warning(
@@ -1182,6 +1271,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
cu_seqlens,
chunk_indices,
chunk_offsets,
aiter_prefill_metadata,
)
torch.accelerator.empty_cache()
@@ -1523,6 +1613,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)