From f9b65c30c7b518600b622d4b683c9f2f1eb04370 Mon Sep 17 00:00:00 2001 From: Mikko Tukiainen Date: Wed, 14 Oct 2026 13:11:57 +0300 Subject: [PATCH] [ROCm][Perf] Make AITER FlyDSL GDN prefill default on gfx942/gfx950 (#60645) Signed-off-by: Mikko Tukiainen Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../mamba/test_gdn_rocm_layout_dispatch.py | 4 +- tests/v1/attention/test_gdn_aiter_flydsl.py | 42 ++++++++++++---- vllm/_aiter_ops.py | 7 ++- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 49 +++++++++++++------ 4 files changed, 74 insertions(+), 28 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py index 851d6fb1d3..267f86b66e 100644 --- a/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py +++ b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py @@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated): model="Qwen/Qwen3.5-0.8B", revision="2fc06364715b967f1860aea9cf38778875588b17", max_model_len=1024, - ) + ), + # auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path. + additional_config={"gdn_prefill_backend": "triton"}, ) with set_current_vllm_config(config): prefill = ChunkGatedDeltaRule() diff --git a/tests/v1/attention/test_gdn_aiter_flydsl.py b/tests/v1/attention/test_gdn_aiter_flydsl.py index 3de625e936..c81058bb85 100644 --- a/tests/v1/attention/test_gdn_aiter_flydsl.py +++ b/tests/v1/attention/test_gdn_aiter_flydsl.py @@ -35,7 +35,7 @@ def _make_config( ) -def _resolve_on_rocm(config, *, kernels_available: bool = True): +def _resolve_on_rocm(config, *, kernels_available: bool = True, mi3xx: bool = True): with ( patch.object( qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True @@ -45,6 +45,7 @@ def _resolve_on_rocm(config, *, kernels_available: bool = True): "is_gdn_flydsl_prefill_available", return_value=kernels_available, ), + patch("vllm.platforms.rocm.on_mi3xx", return_value=mi3xx), ): return _resolve_gdn_prefill_backend(config) @@ -72,16 +73,38 @@ def test_resolve_aiter_flydsl_gdn_prefill_backend( assert active == expected -def test_kda_style_model_cannot_select_flydsl(): +@pytest.mark.parametrize( + "mi3xx,kernels_available,head_k_dim,expected", + [ + (True, True, 128, "aiter_flydsl"), + (False, True, 128, "triton"), + (True, False, 128, "triton"), + (True, True, 64, "triton"), + ], +) +def test_auto_defaults_to_aiter_flydsl( + mi3xx: bool, kernels_available: bool, head_k_dim: int, expected: str +): + """Under auto, FlyDSL is the default only where it was validated and the + model and build can serve it; everything else stays on Triton.""" + config = _make_config(backend="auto", head_k_dim=head_k_dim) + + assert _resolve_on_rocm( + config, kernels_available=kernels_available, mi3xx=mi3xx + ) == ("auto", expected) + + +@pytest.mark.parametrize("backend", ["aiter_flydsl", "auto"]) +def test_kda_style_model_cannot_select_flydsl(backend: str): """Kimi K3 KDA reports head dims through linear_attn_config, not these. Its builder overrides _build_chunk_metadata, which the AITER path skips, so this pins the reason the two never meet: the resolver turns the model down before the builder is ever constructed. """ - config = _make_config(head_k_dim=None, head_v_dim=None) + config = _make_config(backend=backend, head_k_dim=None, head_v_dim=None) - assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton") + assert _resolve_on_rocm(config) == (backend, "triton") @pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"]) @@ -109,15 +132,16 @@ def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool): def _flydsl_prefill_unavailable() -> bool: - return ( - not current_platform.is_rocm() - or not rocm_aiter_ops.is_gdn_flydsl_prefill_available() - ) + if not current_platform.is_rocm(): + return True + from vllm.platforms.rocm import on_mi3xx + + return not on_mi3xx() or not rocm_aiter_ops.is_gdn_flydsl_prefill_available() @pytest.mark.skipif( _flydsl_prefill_unavailable(), - reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed", + reason="needs gfx942/gfx950 with the AITER FlyDSL GDN prefill kernels installed", ) @pytest.mark.parametrize( "state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"] diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 0817c651a0..9e01735441 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -2544,11 +2544,10 @@ class rocm_aiter_ops: @classmethod @functools.cache def is_gdn_flydsl_prefill_available(cls) -> bool: - """Whether the opt-in AITER FlyDSL GDN prefill path can be used. + """Whether the AITER FlyDSL GDN prefill path can be used. - Selecting the backend is an explicit opt-in in itself, but it still - runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that - turns all of them off. + It runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch + that turns all of them off. """ return ( cls._AITER_ENABLED diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 1a3f8183c4..774e6cde60 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -107,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8 FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16) +def _aiter_flydsl_unsupported_reasons( + vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None +) -> list[str]: + """Every reason the AITER FlyDSL GDN prefill kernels cannot serve this + model, or an empty list if they can.""" + reasons = [] + if head_k_dim != 128 or head_v_dim != 128: + reasons.append( + f"linear head dims are K={head_k_dim} V={head_v_dim}; " + "FlyDSL requires 128/128" + ) + dtype = vllm_config.model_config.dtype + if dtype != torch.bfloat16: + reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16") + return reasons + + def _resolve_gdn_prefill_backend( vllm_config: VllmConfig, ) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]: @@ -125,7 +142,7 @@ def _resolve_gdn_prefill_backend( * Blackwell (SM10.x) with ``head_k_dim == 128``; AITER FlyDSL GDN prefill kernels are chosen when: - * "aiter_flydsl" is requested; (opt-in only) + * "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950; * ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels; * the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``. """ @@ -153,24 +170,28 @@ def _resolve_gdn_prefill_backend( "that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or " "newer GPU, and that the installed AITER exports them." ) - if head_k_dim != 128 or head_v_dim != 128: + reasons = _aiter_flydsl_unsupported_reasons( + vllm_config, head_k_dim, head_v_dim + ) + if reasons: logger.warning_once( - "GDN prefill backend 'aiter_flydsl' was requested but " - "linear head dims are K=%s V=%s; FlyDSL requires 128/128. " + "GDN prefill backend 'aiter_flydsl' was requested but %s. " "Falling back to Triton/FLA.", - head_k_dim, - head_v_dim, - ) - return backend, "triton" - if vllm_config.model_config.dtype != torch.bfloat16: - logger.warning_once( - "GDN prefill backend 'aiter_flydsl' was requested but " - "model dtype is %s; FlyDSL requires bfloat16. " - "Falling back to Triton/FLA.", - vllm_config.model_config.dtype, + "; ".join(reasons), ) return backend, "triton" return backend, "aiter_flydsl" + if backend == "auto": + from vllm.platforms.rocm import on_mi3xx + + if ( + on_mi3xx() + and not _aiter_flydsl_unsupported_reasons( + vllm_config, head_k_dim, head_v_dim + ) + and rocm_aiter_ops.is_gdn_flydsl_prefill_available() + ): + return backend, "aiter_flydsl" return backend, "triton" if backend == "aiter_flydsl":