[ROCm][Perf] Make AITER FlyDSL GDN prefill default on gfx942/gfx950 (#60645)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3f466c852c
commit
f9b65c30c7
@@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
revision="2fc06364715b967f1860aea9cf38778875588b17",
|
||||
max_model_len=1024,
|
||||
)
|
||||
),
|
||||
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
|
||||
additional_config={"gdn_prefill_backend": "triton"},
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
prefill = ChunkGatedDeltaRule()
|
||||
|
||||
@@ -35,7 +35,7 @@ def _make_config(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_on_rocm(config, *, kernels_available: bool = True):
|
||||
def _resolve_on_rocm(config, *, kernels_available: bool = True, mi3xx: bool = True):
|
||||
with (
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
|
||||
@@ -45,6 +45,7 @@ def _resolve_on_rocm(config, *, kernels_available: bool = True):
|
||||
"is_gdn_flydsl_prefill_available",
|
||||
return_value=kernels_available,
|
||||
),
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=mi3xx),
|
||||
):
|
||||
return _resolve_gdn_prefill_backend(config)
|
||||
|
||||
@@ -72,16 +73,38 @@ def test_resolve_aiter_flydsl_gdn_prefill_backend(
|
||||
assert active == expected
|
||||
|
||||
|
||||
def test_kda_style_model_cannot_select_flydsl():
|
||||
@pytest.mark.parametrize(
|
||||
"mi3xx,kernels_available,head_k_dim,expected",
|
||||
[
|
||||
(True, True, 128, "aiter_flydsl"),
|
||||
(False, True, 128, "triton"),
|
||||
(True, False, 128, "triton"),
|
||||
(True, True, 64, "triton"),
|
||||
],
|
||||
)
|
||||
def test_auto_defaults_to_aiter_flydsl(
|
||||
mi3xx: bool, kernels_available: bool, head_k_dim: int, expected: str
|
||||
):
|
||||
"""Under auto, FlyDSL is the default only where it was validated and the
|
||||
model and build can serve it; everything else stays on Triton."""
|
||||
config = _make_config(backend="auto", head_k_dim=head_k_dim)
|
||||
|
||||
assert _resolve_on_rocm(
|
||||
config, kernels_available=kernels_available, mi3xx=mi3xx
|
||||
) == ("auto", expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["aiter_flydsl", "auto"])
|
||||
def test_kda_style_model_cannot_select_flydsl(backend: str):
|
||||
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
|
||||
|
||||
Its builder overrides _build_chunk_metadata, which the AITER path skips,
|
||||
so this pins the reason the two never meet: the resolver turns the model
|
||||
down before the builder is ever constructed.
|
||||
"""
|
||||
config = _make_config(head_k_dim=None, head_v_dim=None)
|
||||
config = _make_config(backend=backend, head_k_dim=None, head_v_dim=None)
|
||||
|
||||
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
|
||||
assert _resolve_on_rocm(config) == (backend, "triton")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
|
||||
@@ -109,15 +132,16 @@ def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
|
||||
|
||||
|
||||
def _flydsl_prefill_unavailable() -> bool:
|
||||
return (
|
||||
not current_platform.is_rocm()
|
||||
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
)
|
||||
if not current_platform.is_rocm():
|
||||
return True
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return not on_mi3xx() or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_flydsl_prefill_unavailable(),
|
||||
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
|
||||
reason="needs gfx942/gfx950 with the AITER FlyDSL GDN prefill kernels installed",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
|
||||
|
||||
+3
-4
@@ -2544,11 +2544,10 @@ class rocm_aiter_ops:
|
||||
@classmethod
|
||||
@functools.cache
|
||||
def is_gdn_flydsl_prefill_available(cls) -> bool:
|
||||
"""Whether the opt-in AITER FlyDSL GDN prefill path can be used.
|
||||
"""Whether the AITER FlyDSL GDN prefill path can be used.
|
||||
|
||||
Selecting the backend is an explicit opt-in in itself, but it still
|
||||
runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch that
|
||||
turns all of them off.
|
||||
It runs AITER kernels, so VLLM_ROCM_USE_AITER remains the one switch
|
||||
that turns all of them off.
|
||||
"""
|
||||
return (
|
||||
cls._AITER_ENABLED
|
||||
|
||||
@@ -107,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
|
||||
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
|
||||
def _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
|
||||
) -> list[str]:
|
||||
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
|
||||
model, or an empty list if they can."""
|
||||
reasons = []
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons.append(
|
||||
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
|
||||
"FlyDSL requires 128/128"
|
||||
)
|
||||
dtype = vllm_config.model_config.dtype
|
||||
if dtype != torch.bfloat16:
|
||||
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
|
||||
return reasons
|
||||
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
@@ -125,7 +142,7 @@ def _resolve_gdn_prefill_backend(
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
@@ -153,24 +170,28 @@ def _resolve_gdn_prefill_backend(
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons = _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
if reasons:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
"; ".join(reasons),
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
if backend == "auto":
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
if (
|
||||
on_mi3xx()
|
||||
and not _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
):
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
|
||||
Reference in New Issue
Block a user