[ROCm][Perf] Add AITER FlyDSL GDN prefill backend (#57560)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
f6e3944e31
commit
548597367e
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
"""Resolve GDN prefill backend.
|
||||
|
||||
FlashInfer's GDN prefill kernel is chosen when:
|
||||
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
|
||||
In-tree CuteDSL GDN prefill kernel is chosen when:
|
||||
* "cutedsl" is requested; (opt-in only)
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
additional_config = vllm_config.additional_config
|
||||
backend_cfg = (
|
||||
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
|
||||
)
|
||||
backend = str(backend_cfg).strip().lower()
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
head_v_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if backend == "aiter_flydsl":
|
||||
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but the "
|
||||
"AITER FlyDSL prefill kernels are not importable. Check "
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
|
||||
f"AITER backend and the current platform is {current_platform.device_name}."
|
||||
)
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
supports_flashinfer = False
|
||||
supports_cutedsl = False
|
||||
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
"aiter_flydsl": "AITER FlyDSL",
|
||||
"triton": "Triton/FLA",
|
||||
}[active_backend]
|
||||
logger.info_once(
|
||||
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
self._forward_method = self.forward_cuda
|
||||
elif active_backend == "cutedsl":
|
||||
self._forward_method = self.forward_cutedsl
|
||||
elif active_backend == "aiter_flydsl":
|
||||
self._forward_method = self.forward_aiter_flydsl
|
||||
else:
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
return fla_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
chunk_gated_delta_rule_cutedsl,
|
||||
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
final_state = None
|
||||
return o, final_state
|
||||
|
||||
def forward_aiter_flydsl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
if core_attn_out is not None:
|
||||
o_flat = o.reshape(-1)
|
||||
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
@@ -1132,12 +1214,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# CuteDSL kernels require metadata
|
||||
chunk_indices = None
|
||||
chunk_offsets = None
|
||||
aiter_prefill_metadata = None
|
||||
if self.gdn_prefill_backend == "cutedsl":
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
|
||||
elif self.gdn_prefill_backend == "aiter_flydsl":
|
||||
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
[T],
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
|
||||
try:
|
||||
self.chunk_gated_delta_rule(
|
||||
@@ -1152,6 +1240,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
@@ -1182,6 +1271,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
aiter_prefill_metadata,
|
||||
)
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1523,6 +1613,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
Reference in New Issue
Block a user