[ROCm][Perf] Make AITER FlyDSL GDN prefill default on gfx942/gfx950 (#60645)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Mikko Tukiainen
2026-10-14 10:11:57 +00:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3f466c852c
commit f9b65c30c7
4 changed files with 74 additions and 28 deletions
@@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
model="Qwen/Qwen3.5-0.8B",
revision="2fc06364715b967f1860aea9cf38778875588b17",
max_model_len=1024,
)
),
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
additional_config={"gdn_prefill_backend": "triton"},
)
with set_current_vllm_config(config):
prefill = ChunkGatedDeltaRule()
+33 -9
View File
@@ -35,7 +35,7 @@ def _make_config(
)
def _resolve_on_rocm(config, *, kernels_available: bool = True):
def _resolve_on_rocm(config, *, kernels_available: bool = True, mi3xx: bool = True):
with (
patch.object(
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
@@ -45,6 +45,7 @@ def _resolve_on_rocm(config, *, kernels_available: bool = True):
"is_gdn_flydsl_prefill_available",
return_value=kernels_available,
),
patch("vllm.platforms.rocm.on_mi3xx", return_value=mi3xx),
):
return _resolve_gdn_prefill_backend(config)
@@ -72,16 +73,38 @@ def test_resolve_aiter_flydsl_gdn_prefill_backend(
assert active == expected
def test_kda_style_model_cannot_select_flydsl():
@pytest.mark.parametrize(
"mi3xx,kernels_available,head_k_dim,expected",
[
(True, True, 128, "aiter_flydsl"),
(False, True, 128, "triton"),
(True, False, 128, "triton"),
(True, True, 64, "triton"),
],
)
def test_auto_defaults_to_aiter_flydsl(
mi3xx: bool, kernels_available: bool, head_k_dim: int, expected: str
):
"""Under auto, FlyDSL is the default only where it was validated and the
model and build can serve it; everything else stays on Triton."""
config = _make_config(backend="auto", head_k_dim=head_k_dim)
assert _resolve_on_rocm(
config, kernels_available=kernels_available, mi3xx=mi3xx
) == ("auto", expected)
@pytest.mark.parametrize("backend", ["aiter_flydsl", "auto"])
def test_kda_style_model_cannot_select_flydsl(backend: str):
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
Its builder overrides _build_chunk_metadata, which the AITER path skips,
so this pins the reason the two never meet: the resolver turns the model
down before the builder is ever constructed.
"""
config = _make_config(head_k_dim=None, head_v_dim=None)
config = _make_config(backend=backend, head_k_dim=None, head_v_dim=None)
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
assert _resolve_on_rocm(config) == (backend, "triton")
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
@@ -109,15 +132,16 @@ def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
def _flydsl_prefill_unavailable() -> bool:
return (
not current_platform.is_rocm()
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
)
if not current_platform.is_rocm():
return True
from vllm.platforms.rocm import on_mi3xx
return not on_mi3xx() or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
@pytest.mark.skipif(
_flydsl_prefill_unavailable(),
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
reason="needs gfx942/gfx950 with the AITER FlyDSL GDN prefill kernels installed",
)
@pytest.mark.parametrize(
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
+3 -4
View File
@@ -2544,11 +2544,10 @@ class rocm_aiter_ops:
@classmethod
@functools.cache
def is_gdn_flydsl_prefill_available(cls) -> bool:
"""Whether the opt-in AITER FlyDSL GDN prefill path can be used.
"""Whether the AITER FlyDSL GDN prefill path can be used.
Selecting the backend is an explicit opt-in in itself, but it still
runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that
turns all of them off.
It runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch
that turns all of them off.
"""
return (
cls._AITER_ENABLED
@@ -107,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _aiter_flydsl_unsupported_reasons(
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
) -> list[str]:
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
model, or an empty list if they can."""
reasons = []
if head_k_dim != 128 or head_v_dim != 128:
reasons.append(
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
"FlyDSL requires 128/128"
)
dtype = vllm_config.model_config.dtype
if dtype != torch.bfloat16:
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
return reasons
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
@@ -125,7 +142,7 @@ def _resolve_gdn_prefill_backend(
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
@@ -153,24 +170,28 @@ def _resolve_gdn_prefill_backend(
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
reasons = _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
if reasons:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
"; ".join(reasons),
)
return backend, "triton"
return backend, "aiter_flydsl"
if backend == "auto":
from vllm.platforms.rocm import on_mi3xx
if (
on_mi3xx()
and not _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
):
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":