[ROCm][Perf] Use AITER tuned configs to select a4w4 gemm kernel instead of env var (#61278)

Signed-off-by: Andy Friedrich <afriedri@amd.com>
Signed-off-by: afriedri <afriedri@amd.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
afriedri
2026-10-17 04:00:28 -05:00
committed by GitHub
co-authored by Simon Danielsson Kevin H. Luu Andreas Karatzas
parent 3bf25c5097
commit 2067297b2f
4 changed files with 66 additions and 43 deletions
+41 -30
View File
@@ -147,26 +147,15 @@ def test_fp4_env_defaults():
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
@pytest.mark.parametrize(
(
"use_aiter",
"use_fp4_asm_gemm",
"use_fp4bmm",
),
[
(True, True, True),
(True, True, False),
(True, False, True),
(False, True, True),
],
("use_aiter", "use_fp4bmm"),
[(True, True), (True, False), (False, True)],
)
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
use_aiter,
use_fp4_asm_gemm,
use_fp4bmm,
monkeypatch,
):
@@ -182,22 +171,16 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
_assert_aiter_supported()
on_gfx950_value = on_gfx950()
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
expected_asm_gemm = use_aiter and on_gfx950_value
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv(
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
"1" if use_fp4_asm_gemm else "0",
)
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert (
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
)
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
@@ -695,24 +678,52 @@ def test_asm_fp4_scale_swizzle_rejects_non_2d():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
"""A misaligned weight_scale must disable ASM and take the Triton path."""
@pytest.mark.parametrize(
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
[
(True, True, (1280, 4096), (1280, 256), True),
(True, False, (1280, 4096), (1280, 256), False),
(False, True, (1280, 4096), (1280, 256), False),
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
(True, True, (2560, 160), (2560, 10), False),
],
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
)
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
):
"""A layer takes the ASM layout only when aiter has tuned configs for both
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
Otherwise it stays on the Triton layout."""
from torch.nn.parameter import Parameter
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
monkeypatch.setattr(
rocm_aiter_ops,
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
lambda n, k: preshuffle_tuned,
)
kernel = object.__new__(AiterMxfp4LinearKernel)
kernel.use_asm_gemm = True
kernel.asm_gemm_supported = True
kernel.use_asm_gemm = False
kernel.out_dtype = torch.bfloat16
layer = torch.nn.Module()
layer.weight = Parameter(
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
layer.weight_scale = Parameter(
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
kernel.process_weights_after_loading(layer)
assert kernel.use_asm_gemm is False
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == (10, 2560)
assert layer.weight_scale.is_contiguous()
assert kernel.use_asm_gemm is expected
if not expected:
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
assert layer.weight_scale.is_contiguous()