From 4282fe52d5353084a5773e924a508e9346dc2718 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Wed, 5 Aug 2026 16:27:33 -0500 Subject: [PATCH] [ROCm][CI] Add More AITER quantization/MoE kernel tests (#49375) Signed-off-by: Micah Williamson Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 22 +- tests/kernels/moe/test_rocm_aiter_moe.py | 1079 +++++++++++++++++ tests/kernels/quantization/test_rocm_fp8.py | 541 +++++++++ tests/kernels/quantization/test_rocm_mxfp4.py | 638 ++++++++++ tests/kernels/utils.py | 57 + 5 files changed, 2335 insertions(+), 2 deletions(-) create mode 100644 tests/kernels/moe/test_rocm_aiter_moe.py create mode 100644 tests/kernels/quantization/test_rocm_fp8.py create mode 100644 tests/kernels/quantization/test_rocm_mxfp4.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index d9b8d88942..caea1f7666 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1613,8 +1613,12 @@ steps: source_file_dependencies: - csrc/quantization/cutlass_w8a8/moe/ - csrc/moe/ + - csrc/rocm/ - tests/kernels/moe + - tests/kernels/utils.py - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/mxfp4.py + - vllm/model_executor/layers/quantization/utils/quant_utils.py - vllm/distributed/device_communicators/ - vllm/envs.py - vllm/config @@ -1634,12 +1638,17 @@ steps: working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/quantization/ + - csrc/rocm/ - vllm/model_executor/layers/quantization - tests/kernels/quantization - - tests/kernels/quantization/test_rocm_skinny_gemms.py + - tests/kernels/quant_utils.py + - tests/kernels/utils.py - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/envs.py - vllm/platforms/rocm.py - vllm/model_executor/kernels/ + - vllm/v1/attention/backends/rocm_aiter_fa.py commands: - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT @@ -3406,8 +3415,12 @@ steps: source_file_dependencies: - csrc/quantization/cutlass_w8a8/moe/ - csrc/moe/ + - csrc/rocm/ - tests/kernels/moe + - tests/kernels/utils.py - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/mxfp4.py + - vllm/model_executor/layers/quantization/utils/quant_utils.py - vllm/distributed/device_communicators/ - vllm/envs.py - vllm/config @@ -3426,12 +3439,17 @@ steps: working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/quantization/ + - csrc/rocm/ - vllm/model_executor/layers/quantization - tests/kernels/quantization - - tests/kernels/quantization/test_rocm_skinny_gemms.py + - tests/kernels/quant_utils.py + - tests/kernels/utils.py - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/envs.py - vllm/platforms/rocm.py - vllm/model_executor/kernels/ + - vllm/v1/attention/backends/rocm_aiter_fa.py commands: - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py new file mode 100644 index 0000000000..cbb527a701 --- /dev/null +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -0,0 +1,1079 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Kernel tests for ROCm AITER fused MoE. + +This file owns the ROCm-specific fused-MoE custom-op path: +- custom-op registration and fake-tensor support +- enablement gating for ``VLLM_ROCM_USE_AITER_MOE`` and shared experts +- BF16 accuracy for the fused MoE kernel on representative shapes +- gfx950-only AITER MXFP4 W4A16 MoE support, accuracy, and determinism +- MoE-facing FP8 group-quant activation quality +- deterministic routing and representative gfx942 / gfx950 coverage + +Generic fused-MoE backend selection and non-ROCm kernel coverage live in the +generic MoE test files under ``tests/kernels/moe``. + +Raw ROCm AITER helper-op coverage such as ``group_fp8_quant`` and +``per_tensor_quant`` lives in ``tests/kernels/core/test_rocm_aiter_ops.py``. +This file only keeps the MoE-shaped integration angle for those helpers. +""" + +import importlib +import math +import warnings +from typing import Any, NamedTuple + +import pytest +import torch +import torch.nn.functional as F + +from tests.kernels.utils import _assert_deterministic +from vllm.platforms import current_platform +from vllm.platforms.rocm import on_gfx942, on_gfx950 +from vllm.utils.torch_utils import set_random_seed + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + + +# Helpers ----------------------------------------------------------------- + + +def _reload_envs(): + import vllm.envs as envs + + return importlib.reload(envs) + + +@pytest.fixture(autouse=True) +def _restore_rocm_env_state(): + """Restore global env + AITER flag state after every test.""" + yield + _reload_envs() + from vllm._aiter_ops import rocm_aiter_ops + + rocm_aiter_ops.refresh_env_variables() + + +def _assert_aiter_supported() -> None: + from vllm._aiter_ops import is_aiter_found_and_supported + + assert is_aiter_found_and_supported(), ( + "aiter is required on supported ROCm hardware for this test" + ) + + +def _format_observed_rate(count: int, total: int) -> str: + return f"{count / total:.4%} ({count}/{total})" + + +def _format_allowed_rate(rate: float, total: int) -> str: + allowed_count = int(rate * total) + return f"{rate:.4%} (<= {allowed_count}/{total})" + + +def _quantile(values: torch.Tensor, q: float) -> float: + if values.numel() == 0: + return 0.0 + return torch.quantile(values, q).item() + + +def _assert_close_budget( + actual: torch.Tensor, + expected: torch.Tensor, + *, + label: str, + atol: float, + rtol: float = 0.0, + pass_rate: float = 0.99999, + max_violation_factor: float = 3.0, +) -> None: + actual_f = actual.detach().float().flatten() + expected_f = expected.detach().float().flatten() + abs_diff = (actual_f - expected_f).abs() + allowed = atol + rtol * expected_f.abs() + + total = abs_diff.numel() + within = abs_diff <= allowed + passed = int(within.sum().item()) + failed = total - passed + allowed_fail_rate = 1.0 - pass_rate + + max_abs = abs_diff.max().item() + mean_abs = abs_diff.mean().item() + p99_abs = _quantile(abs_diff, 0.99) + p999_abs = _quantile(abs_diff, 0.999) + worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item() + max_atol = max_violation_factor * atol + above_max_count = int((abs_diff > max_atol).sum().item()) + + msg = ( + "[rocm_aiter_moe] " + f"{label}: " + f"pass={passed / total:.4%} ({passed}/{total}) " + f"fail={_format_observed_rate(failed, total)} " + f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} " + f"atol={atol:g} " + f"rtol={rtol:g} " + f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} " + f"allowed_above_max={_format_allowed_rate(0.0, total)} " + f"max_abs={max_abs:.6g} " + f"mean_abs={mean_abs:.6g} " + f"p99_abs={p99_abs:.6g} " + f"p999_abs={p999_abs:.6g} " + f"worst_ratio={worst_ratio:.6g}" + ) + print(msg) + if failed > 0: + warnings.warn(msg, stacklevel=2) + + assert passed / total >= pass_rate, msg + assert max_abs <= max_atol, msg + assert mean_abs <= atol * 0.25, msg + + +def _assert_group_quant_quality( + actual: torch.Tensor, + expected: torch.Tensor, + *, + label: str, + preferred_rel: float = 0.06, + pass_rate: float = 0.9999, + max_rel: float = 0.55, + max_fail_rate: float = 0.000005, + mean_limit: float = 0.03, +) -> None: + rel = ( + (actual.float() - expected.float()).abs() + / expected.float().abs().clamp_min(1e-5) + ).flatten() + total = rel.numel() + within_preferred_count = int((rel <= preferred_rel).sum().item()) + fail_count = total - within_preferred_count + allowed_fail_rate = 1.0 - pass_rate + above_max_count = int((rel > max_rel).sum().item()) + mean_rel = rel.mean().item() + max_rel_err = rel.max().item() + p99 = _quantile(rel, 0.99) + p999 = _quantile(rel, 0.999) + + msg = ( + "[rocm_aiter_moe] " + f"{label}: " + f"rel<={preferred_rel:g} pass={within_preferred_count / total:.4%} " + f"({within_preferred_count}/{total}) " + f"fail={_format_observed_rate(fail_count, total)} " + f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} " + f"rel>{max_rel:g}={_format_observed_rate(above_max_count, total)} " + f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} " + f"mean_rel={mean_rel:.6g} " + f"max_rel={max_rel_err:.6g} " + f"p99={p99:.6g} " + f"p999={p999:.6g}" + ) + print(msg) + if fail_count > 0: + warnings.warn(msg, stacklevel=2) + + assert within_preferred_count / total >= pass_rate, msg + assert above_max_count / total <= max_fail_rate, msg + assert mean_rel < mean_limit, msg + + +def ref_moe_forward( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str = "silu", +) -> torch.Tensor: + """Float32 mask-based MoE reference for the ROCm fused-MoE kernel.""" + num_tokens, hidden_dim = hidden_states.shape + num_experts = w1.shape[0] + intermediate_dim = w1.shape[1] // 2 + topk = topk_ids.shape[1] + device = hidden_states.device + + hidden_states_f = hidden_states.float() + w1_f = w1.float() + w2_f = w2.float() + + expanded = hidden_states_f.view(num_tokens, 1, hidden_dim).expand( + num_tokens, topk, hidden_dim + ) + expanded = expanded.reshape(num_tokens * topk, hidden_dim) + output = torch.zeros( + num_tokens * topk, hidden_dim, dtype=torch.float32, device=device + ) + flat_topk_ids = topk_ids.view(-1).long() + + for expert_idx in range(num_experts): + expert_mask = flat_topk_ids == expert_idx + if expert_mask.sum() == 0: + continue + gate_up = expanded[expert_mask] @ w1_f[expert_idx].T + gate = gate_up[:, :intermediate_dim] + up = gate_up[:, intermediate_dim:] + if activation == "silu": + act = F.silu(gate) * up + elif activation == "gelu": + act = F.gelu(gate) * up + else: + raise ValueError(f"Unknown activation: {activation}") + output[expert_mask] = act @ w2_f[expert_idx].T + + output = output.view(num_tokens, topk, hidden_dim) + weights = topk_weights.float().view(num_tokens, topk, 1) + return (output * weights).sum(dim=1) + + +class AiterMxfp4MoeCase(NamedTuple): + hidden_states: torch.Tensor + w1_kernel: torch.Tensor + w2_kernel: torch.Tensor + w1_ref: torch.Tensor + w2_ref: torch.Tensor + topk_weights: torch.Tensor + topk_ids: torch.Tensor + moe_config: Any + quant_config: Any + + +def _make_topk_ids( + num_tokens: int, + num_experts: int, + topk: int, + *, + device: str = "cuda", +) -> torch.Tensor: + """Generate distinct expert IDs per token with the same top-k shape as + production routing.""" + router_logits = torch.randn(num_tokens, num_experts, device=device) + _, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1) + return topk_ids.to(torch.int32) + + +def _shuffle_moe_weights( + w1: torch.Tensor, + w2: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Shuffle MoE weights into the AITER CK layout used in production.""" + from vllm._aiter_ops import rocm_aiter_ops + + w1_shuffled, w2_shuffled = rocm_aiter_ops.shuffle_weights(w1, w2) + w1_shuffled.is_shuffled = True + w2_shuffled.is_shuffled = True + return w1_shuffled, w2_shuffled + + +def _make_moe_case( + *, + num_tokens: int, + hidden_dim: int, + intermediate_dim: int, + num_experts: int, + topk: int, + seed: int, +) -> dict[str, torch.Tensor]: + torch.set_default_device("cuda") + set_random_seed(seed) + + hidden_states = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16) + w1 = torch.randn( + num_experts, + intermediate_dim * 2, + hidden_dim, + dtype=torch.bfloat16, + ) / math.sqrt(hidden_dim) + w2 = torch.randn( + num_experts, + hidden_dim, + intermediate_dim, + dtype=torch.bfloat16, + ) / math.sqrt(intermediate_dim) + topk_weights = torch.rand(num_tokens, topk, dtype=torch.float32) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_ids = _make_topk_ids(num_tokens, num_experts, topk) + return { + "hidden_states": hidden_states, + "w1": w1, + "w2": w2, + "topk_weights": topk_weights, + "topk_ids": topk_ids, + } + + +def _interleave_gate_up_rows(t: torch.Tensor) -> torch.Tensor: + """Reorder contiguous ``[gate; up]`` rows into gpt-oss interleaved order.""" + e, two_i = t.shape[0], t.shape[1] + i, rest = two_i // 2, t.shape[2:] + perm = (0, 2, 1, *range(3, 3 + len(rest))) + return t.view(e, 2, i, *rest).permute(*perm).contiguous().view(e, two_i, *rest) + + +def _make_aiter_mxfp4_moe_case( + *, + num_tokens: int, + hidden_dim: int, + intermediate_dim: int, + num_experts: int, + topk: int, + seed: int, +) -> AiterMxfp4MoeCase: + from triton_kernels.numerics_details.mxfp import ( + downcast_to_mxfp, + upcast_from_mxfp, + ) + + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, + make_mxfp4_moe_quant_config, + ) + + torch.set_default_device("cuda") + set_random_seed(seed) + + hidden_states = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16) + w1 = torch.randn( + num_experts, + intermediate_dim * 2, + hidden_dim, + dtype=torch.bfloat16, + ) / math.sqrt(hidden_dim) + w2 = torch.randn( + num_experts, + hidden_dim, + intermediate_dim, + dtype=torch.bfloat16, + ) / math.sqrt(intermediate_dim) + topk_weights = torch.rand(num_tokens, topk, dtype=torch.float32) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_ids = _make_topk_ids(num_tokens, num_experts, topk) + + w1_q, w1_scale = downcast_to_mxfp(w1, torch.uint8, axis=-1) + w2_q, w2_scale = downcast_to_mxfp(w2, torch.uint8, axis=-1) + + w1_ref = upcast_from_mxfp(w1_q, w1_scale, torch.bfloat16, axis=-1) + w2_ref = upcast_from_mxfp(w2_q, w2_scale, torch.bfloat16, axis=-1) + + ( + w1_kernel, + w2_kernel, + w1_scale_kernel, + w2_scale_kernel, + _, + _, + ) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( + mxfp4_backend=Mxfp4MoeBackend.AITER, + layer=torch.nn.Module(), + # w13 rows must be gpt-oss-interleaved; the converter de-interleaves them + # back to the contiguous gate/up blocks that ``w1_ref`` / ``ref_moe_forward`` + # use. w2 has no gate/up split, so it stays as-is. + w13_weight=_interleave_gate_up_rows(w1_q), + w2_weight=w2_q.clone(), + w13_weight_scale=_interleave_gate_up_rows(w1_scale), + w2_weight_scale=w2_scale.clone(), + ) + + moe_config = make_dummy_moe_config( + num_experts=num_experts, + experts_per_token=topk, + hidden_dim=hidden_dim, + intermediate_size=intermediate_dim, + in_dtype=torch.bfloat16, + ) + quant_config = make_mxfp4_moe_quant_config( + Mxfp4MoeBackend.AITER, + w1_scale=w1_scale_kernel, + w2_scale=w2_scale_kernel, + ) + + return AiterMxfp4MoeCase( + hidden_states=hidden_states, + w1_kernel=w1_kernel, + w2_kernel=w2_kernel, + w1_ref=w1_ref, + w2_ref=w2_ref, + topk_weights=topk_weights, + topk_ids=topk_ids, + moe_config=moe_config, + quant_config=quant_config, + ) + + +def _run_fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + *, + activation_method: int, + quant_method: int, +) -> torch.Tensor: + w1_shuffled, w2_shuffled = _shuffle_moe_weights(w1, w2) + return torch.ops.vllm.rocm_aiter_fused_moe( + hidden_states, + w1_shuffled, + w2_shuffled, + topk_weights, + topk_ids, + expert_mask=None, + activation_method=activation_method, + quant_method=quant_method, + doweight_stage1=False, + ) + + +# Custom op tests --------------------------------------------------------- + + +def test_aiter_fused_moe_custom_op_registered(): + """The main fused-MoE custom op should stay registered for runtime use.""" + _assert_aiter_supported() + import vllm._aiter_ops as aiter_ops # noqa: F401 + + assert hasattr(torch.ops.vllm, "rocm_aiter_fused_moe") + assert callable(torch.ops.vllm.rocm_aiter_fused_moe) + + +def test_aiter_asm_moe_tkw1_custom_op_registered(): + """The tkw1 custom op should stay registered for FP8 apply-router-weight + paths.""" + _assert_aiter_supported() + import vllm._aiter_ops as aiter_ops # noqa: F401 + + assert hasattr(torch.ops.vllm, "rocm_aiter_asm_moe_tkw1") + assert callable(torch.ops.vllm.rocm_aiter_asm_moe_tkw1) + + +def test_aiter_fused_moe_fake_tensor_support(): + """The fused-MoE op should preserve fake-tensor compatibility for + torch.compile-style tracing.""" + _assert_aiter_supported() + import vllm._aiter_ops # noqa: F401 + + num_tokens = 16 + hidden_dim = 1024 + intermediate_dim = 2048 + num_experts = 8 + topk = 2 + + hidden_states = torch.randn( + num_tokens, hidden_dim, dtype=torch.bfloat16, device="cuda" + ) + w1 = torch.randn( + num_experts, + intermediate_dim * 2, + hidden_dim, + dtype=torch.bfloat16, + device="cuda", + ) + w2 = torch.randn( + num_experts, + hidden_dim, + intermediate_dim, + dtype=torch.bfloat16, + device="cuda", + ) + topk_weights = torch.rand(num_tokens, topk, dtype=torch.float32, device="cuda") + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_ids = _make_topk_ids(num_tokens, num_experts, topk, device="cuda") + + torch.library.opcheck( + torch.ops.vllm.rocm_aiter_fused_moe, + (hidden_states, w1, w2, topk_weights, topk_ids), + kwargs={ + "expert_mask": None, + "activation_method": 0, + "quant_method": 0, + "doweight_stage1": False, + }, + test_utils=("test_faketensor",), + ) + + +# Env gating tests -------------------------------------------------------- + + +@pytest.mark.parametrize( + ("use_aiter", "use_moe", "expected"), + [ + (True, True, True), + (True, False, False), + (False, True, False), + (False, False, False), + ], +) +def test_aiter_moe_enablement_follows_env( + use_aiter: bool, + use_moe: bool, + expected: bool, + monkeypatch: pytest.MonkeyPatch, +): + """The fused-MoE gate should depend only on the main AITER toggle and the + MoE-specific toggle.""" + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1" if use_moe else "0") + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + assert rocm_aiter_ops.is_fused_moe_enabled() is expected + + +@pytest.mark.parametrize( + ("use_aiter", "use_moe", "use_shared", "expected"), + [ + (True, True, True, True), + (True, True, False, False), + (True, False, True, False), + (False, True, True, False), + ], +) +def test_aiter_moe_shared_experts_enablement_follows_env( + use_aiter: bool, + use_moe: bool, + use_shared: bool, + expected: bool, + monkeypatch: pytest.MonkeyPatch, +): + """Shared-expert fusion should only be enabled when the fused-MoE path is + enabled too.""" + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1" if use_moe else "0") + mp.setenv( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", + "1" if use_shared else "0", + ) + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected + + +@pytest.mark.parametrize("moe_padding", [True, False]) +def test_aiter_moe_padding_env_var( + moe_padding: bool, + monkeypatch: pytest.MonkeyPatch, +): + """The ROCm MoE padding env var should keep its exact parse contract.""" + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_MOE_PADDING", "1" if moe_padding else "0") + envs = _reload_envs() + assert envs.VLLM_ROCM_MOE_PADDING is moe_padding + + +# Enum tests -------------------------------------------------------------- + + +def test_quant_method_enum_values(): + """The AITER quant-method bridge enum should keep its wire values.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import QuantMethod + + assert QuantMethod.NO == 0 + assert QuantMethod.PER_TENSOR == 1 + assert QuantMethod.PER_TOKEN == 2 + assert QuantMethod.BLOCK_1X32 == 3 + assert QuantMethod.BLOCK_1X128 == 4 + assert QuantMethod.BLOCK_128x128 == 5 + + +def test_activation_method_enum_values(): + """The AITER activation-method bridge enum should keep its wire values.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + ) + + assert ActivationMethod.SILU == 0 + assert ActivationMethod.GELU == 1 + + +# MXFP4 kernel tests ------------------------------------------------------ + + +def test_aiter_mxfp4_quant_scheme_support_matches_gfx950(): + """AITER MXFP4 MoE support should stay gfx950-only.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + AiterExperts, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kMxfp4Static, + ) + + assert AiterExperts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950() + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy(): + """The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized + MXFP4 reference.""" + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + rocm_aiter_fused_experts, + ) + + _assert_aiter_supported() + case = _make_aiter_mxfp4_moe_case( + num_tokens=32, + hidden_dim=512, + intermediate_dim=1024, + num_experts=4, + topk=2, + seed=11, + ) + ref_out = ref_moe_forward( + case.hidden_states, + case.w1_ref, + case.w2_ref, + case.topk_weights, + case.topk_ids, + ) + out = rocm_aiter_fused_experts( + hidden_states=case.hidden_states, + w1=case.w1_kernel, + w2=case.w2_kernel, + topk_weights=case.topk_weights, + topk_ids=case.topk_ids, + activation=MoEActivation.SWIGLUOAI, + quant_config=case.quant_config, + moe_config=case.moe_config, + expert_map=None, + ) + + assert out.shape == case.hidden_states.shape + _assert_close_budget( + out.float(), + ref_out.float(), + label="mi350_mxfp4_w4a16_accuracy", + atol=0.1, + rtol=0.1, + pass_rate=0.99, + max_violation_factor=2.0, + ) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism(): + """The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise + deterministic.""" + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + rocm_aiter_fused_experts, + ) + + _assert_aiter_supported() + case = _make_aiter_mxfp4_moe_case( + num_tokens=8, + hidden_dim=512, + intermediate_dim=1024, + num_experts=4, + topk=2, + seed=13, + ) + + def run_mxfp4_moe(): + return rocm_aiter_fused_experts( + hidden_states=case.hidden_states, + w1=case.w1_kernel, + w2=case.w2_kernel, + topk_weights=case.topk_weights, + topk_ids=case.topk_ids, + activation=MoEActivation.SWIGLUOAI, + quant_config=case.quant_config, + moe_config=case.moe_config, + expert_map=None, + ) + + _assert_deterministic(run_mxfp4_moe, n_runs=4) + + +# FP8 group-quant tests --------------------------------------------------- + + +@pytest.mark.skipif( + not (on_gfx942() or on_gfx950()), + reason="gfx942/gfx950 ROCm only", +) +@pytest.mark.parametrize("num_tokens,hidden_dim", [(16, 2048), (64, 4096), (128, 8192)]) +def test_aiter_moe_group_fp8_quant_reconstructs_hidden_states( + num_tokens: int, + hidden_dim: int, +): + """The MoE-facing FP8 group-quant helper should reconstruct hidden states + within the expected FP8 error budget. + + The raw op's shape and standalone roundtrip contracts are covered in + ``tests/kernels/core/test_rocm_aiter_ops.py``. This test keeps the + representative hidden-state sizes that the fused-MoE path actually cares + about. + """ + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + torch.set_default_device("cuda") + set_random_seed(1) + + group_size = 128 + hidden_states = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16) + x_fp8, scales = rocm_aiter_ops.group_fp8_quant(hidden_states, group_size) + + assert x_fp8.shape == hidden_states.shape + assert scales.shape == (num_tokens, (hidden_dim + group_size - 1) // group_size) + assert scales.dtype == torch.float32 + + scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :hidden_dim] + dequantized = x_fp8.float() * scales_expanded + + _assert_group_quant_quality( + dequantized, + hidden_states, + label=f"group_fp8_quant shape=({num_tokens}, {hidden_dim})", + ) + + +# Kernel accuracy tests --------------------------------------------------- + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim,intermediate_dim", + [ + (16, 512, 1024), + (128, 2048, 4096), + (2048, 4096, 11008), + ], +) +def test_aiter_fused_moe_bf16_accuracy( + num_tokens: int, + hidden_dim: int, + intermediate_dim: int, +): + """The ROCm AITER fused-MoE BF16 path should match the float32 reference + on representative shapes.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=num_tokens, + hidden_dim=hidden_dim, + intermediate_dim=intermediate_dim, + num_experts=8, + topk=2, + seed=0, + ) + ref_out = ref_moe_forward( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation="silu", + ) + out = _run_fused_moe( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.NO), + ) + + assert out.shape == (num_tokens, hidden_dim) + _assert_close_budget( + out.float(), + ref_out, + label=f"bf16_accuracy shape=({num_tokens}, {hidden_dim}, {intermediate_dim})", + atol=0.05, + rtol=0.0, + ) + + +def test_aiter_fused_moe_gelu_accuracy(): + """The GELU activation variant should stay aligned with the float32 + reference.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=32, + hidden_dim=512, + intermediate_dim=1024, + num_experts=4, + topk=2, + seed=42, + ) + ref_out = ref_moe_forward( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation="gelu", + ) + out = _run_fused_moe( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation_method=int(ActivationMethod.GELU), + quant_method=int(QuantMethod.NO), + ) + + assert out.shape == case["hidden_states"].shape + _assert_close_budget( + out.float(), + ref_out, + label="gelu_accuracy", + atol=0.05, + rtol=0.0, + ) + + +def test_aiter_fused_moe_determinism(): + """The BF16 fused-MoE kernel should stay bitwise deterministic for the + same inputs.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=8, + hidden_dim=256, + intermediate_dim=512, + num_experts=4, + topk=2, + seed=2, + ) + + def run_moe(): + return _run_fused_moe( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.NO), + ) + + _assert_deterministic(run_moe, n_runs=4) + + +# Routed end-to-end tests ------------------------------------------------- + + +@pytest.mark.parametrize( + "num_tokens,num_experts,topk", + [ + (1, 8, 2), + (16, 8, 2), + (64, 16, 4), + ], +) +def test_aiter_fused_moe_end_to_end( + num_tokens: int, + num_experts: int, + topk: int, +): + """The full router-logits to top-k to fused-MoE path should stay accurate.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + torch.set_default_device("cuda") + set_random_seed(7) + + hidden_dim = 512 + intermediate_dim = 1024 + hidden_states = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16) + w1 = torch.randn( + num_experts, + intermediate_dim * 2, + hidden_dim, + dtype=torch.bfloat16, + ) / math.sqrt(hidden_dim) + w2 = torch.randn( + num_experts, + hidden_dim, + intermediate_dim, + dtype=torch.bfloat16, + ) / math.sqrt(intermediate_dim) + + router_logits = torch.randn(num_tokens, num_experts, device="cuda") + router_probs = torch.softmax(router_logits, dim=-1) + topk_weights, topk_ids = torch.topk(router_probs, k=topk, dim=-1) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_weights = topk_weights.float() + topk_ids = topk_ids.to(torch.int32) + + ref_out = ref_moe_forward(hidden_states, w1, w2, topk_weights, topk_ids) + out = _run_fused_moe( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.NO), + ) + + assert out.shape == (num_tokens, hidden_dim) + assert out.dtype == torch.bfloat16 + _assert_close_budget( + out.float(), + ref_out, + label=f"end_to_end tokens={num_tokens} experts={num_experts} topk={topk}", + atol=0.05, + rtol=0.0, + ) + + +# Arch-specific tests ----------------------------------------------------- + + +@pytest.mark.skipif( + not (on_gfx942() or on_gfx950()), + reason="gfx942/gfx950 ROCm only", +) +def test_aiter_fused_moe_mi3xx_bf16_accuracy(): + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=64, + hidden_dim=4096, + intermediate_dim=11008, + num_experts=8, + topk=2, + seed=42, + ) + ref_out = ref_moe_forward( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + ) + out = _run_fused_moe( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.NO), + ) + + _assert_close_budget( + out.float(), + ref_out, + label=(f"mi3xx_bf16_accuracy arch={'gfx942' if on_gfx942() else 'gfx950'}"), + atol=0.05, + rtol=0.0, + ) + + +@pytest.mark.skipif( + not (on_gfx942() or on_gfx950()), + reason="gfx942/gfx950 ROCm only", +) +def test_aiter_fused_moe_mi3xx_fp8_accuracy(): + """The MI3xx FP8 per-tensor MoE path should stay within the measured FP8 + error budget.""" + from tests.kernels.moe.utils import make_test_weights + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + _assert_aiter_supported() + torch.set_default_device("cuda") + set_random_seed(99) + + num_tokens = 32 + hidden_dim = 512 + intermediate_dim = 1024 + num_experts = 4 + topk = 2 + + hidden_states = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16) / 10 + fp8_dtype = current_platform.fp8_dtype() + (w1_bf16, w1_fp8, w1_scale, _), (w2_bf16, w2_fp8, w2_scale, _) = make_test_weights( + num_experts, + intermediate_dim * 2, + hidden_dim, + torch.bfloat16, + torch.float8_e4m3fn, + per_out_ch_quant=False, + ) + _, a1_scale = rocm_aiter_ops.per_tensor_quant(hidden_states, fp8_dtype) + router_logits = torch.randn(num_tokens, num_experts, device="cuda") + router_probs = torch.softmax(router_logits, dim=-1) + topk_weights, topk_ids = torch.topk(router_probs, k=topk, dim=-1) + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + topk_weights = topk_weights.float() + topk_ids = topk_ids.to(torch.int32) + + ref_out = ref_moe_forward( + hidden_states, + w1_bf16, + w2_bf16, + topk_weights, + topk_ids, + ) + + w1_shuffled, w2_shuffled = _shuffle_moe_weights(w1_fp8, w2_fp8) + out = torch.ops.vllm.rocm_aiter_fused_moe( + hidden_states, + w1_shuffled, + w2_shuffled, + topk_weights, + topk_ids, + expert_mask=None, + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.PER_TENSOR), + doweight_stage1=False, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + ) + + assert out.shape == hidden_states.shape + _assert_close_budget( + out.float(), + ref_out, + label=(f"mi3xx_fp8_accuracy arch={'gfx942' if on_gfx942() else 'gfx950'}"), + atol=0.02, + rtol=0.0, + pass_rate=1.0, + max_violation_factor=1.5, + ) diff --git a/tests/kernels/quantization/test_rocm_fp8.py b/tests/kernels/quantization/test_rocm_fp8.py new file mode 100644 index 0000000000..47e37df119 --- /dev/null +++ b/tests/kernels/quantization/test_rocm_fp8.py @@ -0,0 +1,541 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm-specific tests for FP8 quantization and kernels.""" + +import importlib +import math +import warnings + +import pytest +import torch + +import vllm._custom_ops as ops +from tests.kernels.quant_utils import ( + ref_dynamic_per_tensor_fp8_quant, + ref_dynamic_per_token_quant, +) +from tests.kernels.utils import _assert_deterministic +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + +DEVICE = "cuda" +FP8_DTYPE = current_platform.fp8_dtype() +fp8_only = pytest.mark.skipif( + not current_platform.supports_fp8(), reason="ROCm FP8 only" +) + +ENV_DEFAULTS = { + "VLLM_ROCM_FP8_PADDING": True, + "VLLM_ROCM_USE_AITER_FP8BMM": True, + "VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False, +} + +SCALED_MM_CASES = [ + pytest.param((1, 128, 128), 0.05, 0.15, 0.95, id="1x128x128"), + pytest.param((4, 256, 512), 0.10, 0.30, 0.96, id="4x256x512"), + pytest.param((8, 512, 1024), 0.20, 0.50, 0.99, id="8x512x1024"), +] + + +def _reload_envs(): + import vllm.envs as envs + + return importlib.reload(envs) + + +@pytest.fixture(autouse=True) +def _restore_rocm_env_state(): + """Restore global env + AITER flag state after every test. + + Tests here reload ``vllm.envs`` (a module singleton) and refresh the + cached ``rocm_aiter_ops`` env flags. Restoring in a fixture teardown + guarantees cleanup even when an assertion fails mid-test, preventing + state from leaking into other tests in the same worker. + """ + yield + _reload_envs() + from vllm._aiter_ops import rocm_aiter_ops + + rocm_aiter_ops.refresh_env_variables() + + +def _quantile(flat: torch.Tensor, q: float) -> float: + if flat.numel() == 1: + return flat.item() + return torch.quantile(flat, q).item() + + +def _format_observed_rate(count: int, total: int) -> str: + return f"{count / total:.4%} ({count}/{total})" + + +def _format_allowed_rate(rate: float, total: int) -> str: + allowed_count = math.floor(rate * total) + return f"{rate:.4%} (<= {allowed_count}/{total})" + + +def _assert_abs_error_budget( + actual: torch.Tensor, + expected: torch.Tensor, + *, + label: str, + tight_atol: float, + max_atol: float, + pass_rate: float, +) -> None: + diff = (actual.float() - expected.float()).abs().flatten() + total = diff.numel() + within_tight_count = int((diff <= tight_atol).sum().item()) + fail_count = total - within_tight_count + above_max_count = int((diff > max_atol).sum().item()) + within_tight = within_tight_count / total + allowed_fail_rate = 1.0 - pass_rate + above_max = above_max_count / total + max_fail_rate = 0.0 + max_diff = diff.max().item() + mean_diff = diff.mean().item() + p99 = _quantile(diff, 0.99) + p999 = _quantile(diff, 0.999) + msg = ( + f"[rocm_fp8] {label}: abs<={tight_atol} pass={within_tight:.4%} " + f"({within_tight_count}/{total}) " + f"fail={_format_observed_rate(fail_count, total)} " + f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} " + f"abs>{max_atol}={_format_observed_rate(above_max_count, total)} " + f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} " + f"max_diff={max_diff:.6g} mean_diff={mean_diff:.6g} " + f"p99={p99:.6g} p999={p999:.6g}" + ) + print(msg) + if within_tight < 1.0: + warnings.warn(msg, stacklevel=2) + assert within_tight >= pass_rate, msg + assert above_max <= max_fail_rate, msg + + +def _assert_rel_error_budget( + actual: torch.Tensor, + expected: torch.Tensor, + *, + label: str, + tight_rtol: float, + max_rtol: float, + pass_rate: float, + max_fail_rate: float, +) -> None: + rel = ( + (actual.float() - expected.float()).abs() + / expected.float().abs().clamp_min(1e-5) + ).flatten() + total = rel.numel() + within_tight_count = int((rel <= tight_rtol).sum().item()) + fail_count = total - within_tight_count + above_max_count = int((rel > max_rtol).sum().item()) + within_tight = within_tight_count / total + allowed_fail_rate = 1.0 - pass_rate + above_max = above_max_count / total + max_rel = rel.max().item() + mean_rel = rel.mean().item() + p99 = _quantile(rel, 0.99) + p999 = _quantile(rel, 0.999) + msg = ( + f"[rocm_fp8] {label}: rel<={tight_rtol} pass={within_tight:.4%} " + f"({within_tight_count}/{total}) " + f"fail={_format_observed_rate(fail_count, total)} " + f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} " + f"rel>{max_rtol}={_format_observed_rate(above_max_count, total)} " + f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} " + f"max_rel={max_rel:.6g} mean_rel={mean_rel:.6g} " + f"p99={p99:.6g} p999={p999:.6g}" + ) + print(msg) + if within_tight < 1.0: + warnings.warn(msg, stacklevel=2) + assert within_tight >= pass_rate, msg + assert above_max <= max_fail_rate, msg + + +def _ref_paged_attention( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + *, + query_lens: list[int], + kv_lens: list[int], + block_tables: torch.Tensor, + scale: float, +) -> torch.Tensor: + block_tables_np = block_tables.cpu().numpy() + _, block_size, num_kv_heads, head_size = key_cache.shape + + outputs = [] + start_idx = 0 + for i, query_len in enumerate(query_lens): + kv_len = kv_lens[i] + q = query[start_idx : start_idx + query_len] * scale + + num_kv_blocks = (kv_len + block_size - 1) // block_size + block_indices = block_tables_np[i, :num_kv_blocks] + + k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len] + v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len] + + if q.shape[1] != k.shape[1]: + repeat = q.shape[1] // k.shape[1] + k = torch.repeat_interleave(k, repeat, dim=1) + v = torch.repeat_interleave(v, repeat, dim=1) + + attn = torch.einsum("qhd,khd->hqk", q, k).float() + mask = torch.triu( + torch.ones(query_len, kv_len, device=q.device), + diagonal=kv_len - query_len + 1, + ).bool() + attn.masked_fill_(mask, float("-inf")) + attn = torch.softmax(attn, dim=-1).to(v.dtype) + outputs.append(torch.einsum("hqk,khd->qhd", attn, v)) + start_idx += query_len + + return torch.cat(outputs, dim=0) + + +def test_rocm_fp8_env_defaults(monkeypatch): + with monkeypatch.context() as mp: + for env_name in ENV_DEFAULTS: + mp.delenv(env_name, raising=False) + envs = _reload_envs() + for env_name, expected in ENV_DEFAULTS.items(): + assert getattr(envs, env_name) is expected + + +@pytest.mark.parametrize( + ("env_name", "raw_value", "expected"), + [ + ("VLLM_ROCM_FP8_PADDING", "0", False), + ("VLLM_ROCM_FP8_PADDING", "1", True), + ("VLLM_ROCM_USE_AITER_FP8BMM", "0", False), + ("VLLM_ROCM_USE_AITER_FP8BMM", "1", True), + ("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0", False), + ("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "1", True), + ], +) +def test_rocm_fp8_env_overrides(env_name, raw_value, expected, monkeypatch): + with monkeypatch.context() as mp: + mp.setenv(env_name, raw_value) + envs = _reload_envs() + assert getattr(envs, env_name) is expected + + +@fp8_only +def test_maybe_pad_fp8_weight_respects_env(monkeypatch): + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _maybe_pad_fp8_weight, + ) + + weight = torch.randn(64, 512, device=DEVICE).to(FP8_DTYPE) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_FP8_PADDING", "1") + _reload_envs() + padded = _maybe_pad_fp8_weight(weight) + assert padded.shape == weight.shape + assert padded.dtype == weight.dtype + assert padded.data_ptr() != weight.data_ptr() + torch.testing.assert_close(padded.float(), weight.float()) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_FP8_PADDING", "0") + _reload_envs() + unpadded = _maybe_pad_fp8_weight(weight) + assert unpadded.data_ptr() == weight.data_ptr() + + +@fp8_only +@pytest.mark.parametrize( + ("use_aiter", "fp8bmm_enabled", "expected"), + [ + (True, True, True), + (True, False, False), + (False, True, False), + ], +) +def test_aiter_fp8bmm_enabled_api_respects_env( + use_aiter, fp8bmm_enabled, expected, monkeypatch +): + from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops + + assert is_aiter_found_and_supported() + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv("VLLM_ROCM_USE_AITER_FP8BMM", "1" if fp8bmm_enabled else "0") + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + assert rocm_aiter_ops.is_fp8bmm_enabled() is expected + + +@fp8_only +@pytest.mark.parametrize( + ("shape", "tight_atol", "max_atol", "pass_rate"), SCALED_MM_CASES +) +def test_fp8_scaled_mm_matches_dequantized_reference( + shape, tight_atol, max_atol, pass_rate +): + torch.manual_seed(0) + + batch, m_dim, k_dim = shape + n_dim = k_dim + a = torch.randn(batch * m_dim, k_dim, device=DEVICE) + b = torch.randn(n_dim, k_dim, device=DEVICE) + + a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a) + b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b) + out = torch._scaled_mm( + a_fp8, + b_fp8.t(), + out_dtype=torch.bfloat16, + scale_a=scale_a, + scale_b=scale_b, + ) + + a_dq = a_fp8.float() * scale_a.float() + b_dq = b_fp8.float() * scale_b.float() + ref = torch.mm(a_dq, b_dq.t()) + + assert out.shape == (batch * m_dim, n_dim) + assert out.dtype == torch.bfloat16 + assert not torch.isnan(out).any() + + _assert_abs_error_budget( + out.float(), + ref, + label=f"scaled_mm shape={shape}", + tight_atol=tight_atol, + max_atol=max_atol, + pass_rate=pass_rate, + ) + + +@fp8_only +def test_fp8_scaled_mm_determinism(): + torch.manual_seed(5) + + m_dim, k_dim, n_dim = 64, 128, 128 + a = torch.randn(m_dim, k_dim, device=DEVICE) + b = torch.randn(n_dim, k_dim, device=DEVICE) + a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a) + b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b) + + def run_gemm(): + return torch._scaled_mm( + a_fp8, + b_fp8.t(), + out_dtype=torch.bfloat16, + scale_a=scale_a, + scale_b=scale_b, + ) + + _assert_deterministic(run_gemm, n_runs=4) + + +@fp8_only +def test_aiter_fp8_kv_attention_matches_bf16_reference(): + import aiter + + from vllm._aiter_ops import is_aiter_found_and_supported + from vllm.v1.attention.backends.rocm_aiter_fa import cp_mha_gather_cache + + assert is_aiter_found_and_supported() + + set_random_seed(10) + + num_q_heads, num_kv_heads = 8, 8 + head_size = 128 + query_len, kv_len = 4, 128 + block_size = 16 + num_blocks = 2048 + scale = head_size**-0.5 + + query = torch.randn( + query_len, num_q_heads, head_size, dtype=torch.bfloat16, device=DEVICE + ) + key_cache_fp8 = torch.clamp( + torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE), + min=-1.0, + max=1.0, + ).to(FP8_DTYPE) + value_cache_fp8 = torch.clamp( + torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE), + min=-1.0, + max=1.0, + ).to(FP8_DTYPE) + + cu_query_lens = torch.tensor([0, query_len], dtype=torch.int32, device=DEVICE) + cu_seq_lens = torch.tensor([0, kv_len], dtype=torch.int32, device=DEVICE) + block_tables = torch.randint( + 0, + num_blocks, + (1, (kv_len + block_size - 1) // block_size), + dtype=torch.int32, + device=DEVICE, + ) + token_to_batch = torch.zeros(kv_len, dtype=torch.int32, device=DEVICE) + seq_starts = torch.zeros(1, dtype=torch.int32, device=DEVICE) + gathered_key = torch.empty( + kv_len, num_kv_heads, head_size, dtype=torch.bfloat16, device=DEVICE + ) + gathered_value = torch.empty_like(gathered_key) + + cp_mha_gather_cache( + key_cache=key_cache_fp8, + value_cache=value_cache_fp8, + key=gathered_key, + value=gathered_value, + block_tables=block_tables, + k_scales=torch.ones(1, dtype=torch.float32, device=DEVICE), + v_scales=torch.ones(1, dtype=torch.float32, device=DEVICE), + cu_seqlens_kv=cu_seq_lens, + token_to_batch=token_to_batch, + seq_starts=seq_starts, + dequant=True, + kv_cache_layout="NHD", + total_tokens=kv_len, + ) + + output = torch.empty_like(query) + aiter.flash_attn_varlen_func( + q=query, + k=gathered_key, + v=gathered_value, + cu_seqlens_q=cu_query_lens, + cu_seqlens_k=cu_seq_lens, + max_seqlen_q=query_len, + max_seqlen_k=kv_len, + min_seqlen_q=1, + dropout_p=0.0, + softmax_scale=scale, + causal=True, + window_size=(-1, -1), + alibi_slopes=None, + return_lse=False, + out=output, + ) + + ref = _ref_paged_attention( + query=query, + key_cache=key_cache_fp8.to(torch.bfloat16), + value_cache=value_cache_fp8.to(torch.bfloat16), + query_lens=[query_len], + kv_lens=[kv_len], + block_tables=block_tables, + scale=scale, + ) + + _assert_abs_error_budget( + output.float(), + ref.float(), + label="aiter_fp8_kv_attention", + tight_atol=0.15, + max_atol=0.25, + pass_rate=0.98, + ) + + +@fp8_only +@pytest.mark.parametrize("shape", [(128, 256), (512, 1024), (1024, 4096)]) +def test_fp8_per_tensor_quant_matches_reference(shape): + torch.manual_seed(0) + + x = torch.randn(*shape, dtype=torch.bfloat16, device=DEVICE) + + out, scale = ops.scaled_fp8_quant(x) + ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x) + + assert out.shape == x.shape + assert out.dtype == FP8_DTYPE + + torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4) + code_match_rate = ( + (out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item() + ) + assert code_match_rate >= 0.999, ( + f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)" + ) + + dequant = out.float() * scale.float() + _assert_rel_error_budget( + dequant, + x.float(), + label=f"per_tensor_quant shape={shape}", + tight_rtol=0.06, + max_rtol=0.60, + pass_rate=0.99, + max_fail_rate=1e-4, + ) + + +@fp8_only +def test_aiter_group_fp8_quant_roundtrip(): + from aiter import dtypes + + from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops + + assert is_aiter_found_and_supported() + + torch.manual_seed(0) + + m_dim, n_dim = 64, 4096 + group_size = 128 + x = torch.randn(m_dim, n_dim, dtype=torch.bfloat16, device=DEVICE) + + out, scales = rocm_aiter_ops.group_fp8_quant(x, group_size) + assert out.dtype == dtypes.fp8 + assert scales.dtype == torch.float32 + + scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :n_dim] + dequant = out.float() * scales_expanded + _assert_rel_error_budget( + dequant, + x.float(), + label="aiter_group_fp8_quant", + tight_rtol=0.06, + max_rtol=0.30, + pass_rate=0.99, + max_fail_rate=1e-4, + ) + + +@fp8_only +def test_fp8_per_token_quant_matches_reference(): + torch.manual_seed(0) + + x = torch.randn(32, 4096, dtype=torch.bfloat16, device=DEVICE) + + out, scale = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True) + ref_out, ref_scale = ref_dynamic_per_token_quant(x, FP8_DTYPE) + + assert out.shape == x.shape + assert out.dtype == FP8_DTYPE + assert scale.shape == (x.shape[0], 1) + + torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4) + code_match_rate = ( + (out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item() + ) + assert code_match_rate >= 0.999, ( + f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)" + ) + + dequant = out.float() * scale.float() + _assert_rel_error_budget( + dequant, + x.float(), + label="per_token_quant", + tight_rtol=0.06, + max_rtol=0.30, + pass_rate=0.99, + max_fail_rate=1e-4, + ) diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py new file mode 100644 index 0000000000..db01213bf6 --- /dev/null +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -0,0 +1,638 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm AITER FP4 and MXFP4 tests. + +This file keeps the ROCm-specific FP4 coverage: +- ROCm AITER FP4 env and enablement gates +- large-shape MXFP4 wrapper checks through vLLM's public helper op +- AITER Triton MXFP4 quant format and determinism +- FP4 GEMM, preshuffled-scale, hardware-quant, and skinny decode + paths + +Generic MXFP4 wrapper/reference checks already live in +``tests/kernels/quantization/test_quark.py``. +""" + +import importlib +import warnings + +import pytest +import torch + +from tests.kernels.utils import _assert_accurate, _assert_deterministic +from tests.quantization.reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch +from vllm.platforms import current_platform +from vllm.platforms.rocm import on_gfx950 + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + +LLAMA_SHAPES = [ + (64, 4096), + (64, 11008), + (32, 14336), +] +GEMM_ATOL = 0.5 +GEMM_RTOL = 0.0 +GEMM_MAX_VIOLATION_FACTOR = 2.0 +A4W4_GEMM_PASS_RATES = { + (64, 128, 64): 0.9999, + (128, 256, 128): 0.9999, + (128, 4096, 4096): 0.9999, + (256, 4096, 11008): 0.9999, + (64, 8192, 28672): 0.99765, +} +SKINNY_GEMM_PASS_RATES = { + (4096, 4096): 0.9999, + (4096, 11008): 0.99235, + (8192, 8192): 0.99765, +} +PRESHUFFLED_SHAPES = [ + (64, 4096, 8192), + (32, 8192, 8192), +] + + +def _reload_envs(): + import vllm.envs as envs + + return importlib.reload(envs) + + +@pytest.fixture(autouse=True) +def _restore_rocm_env_state(): + """Restore global env + AITER flag state after every test.""" + yield + _reload_envs() + from vllm._aiter_ops import rocm_aiter_ops + + rocm_aiter_ops.refresh_env_variables() + + +def _assert_aiter_supported() -> None: + from vllm._aiter_ops import is_aiter_found_and_supported + + assert is_aiter_found_and_supported(), ( + "aiter is required on supported ROCm hardware for this test" + ) + + +def _format_observed_rate(count: int, total: int) -> str: + return f"{count / total:.4%} ({count}/{total})" + + +def _format_allowed_rate(rate: float, total: int) -> str: + allowed_count = int(rate * total) + return f"{rate:.4%} (<= {allowed_count}/{total})" + + +def _print_close_stats( + label: str, + actual: torch.Tensor, + expected: torch.Tensor, + *, + atol: float, + rtol: float, + pass_rate: float = 1.0, + max_atol: float | None = None, +) -> None: + abs_diff = (actual - expected).abs().float().flatten() + expected_abs = expected.abs().float().flatten() + allowed = atol + rtol * expected_abs + within = abs_diff <= allowed + + total = abs_diff.numel() + passed = int(within.sum().item()) + failed = total - passed + allowed_fail_rate = 1.0 - pass_rate + + max_abs = abs_diff.max().item() + mean_abs = abs_diff.mean().item() + p99_abs = torch.quantile(abs_diff, 0.99).item() + p999_abs = torch.quantile(abs_diff, 0.999).item() + worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item() + + msg = ( + "[rocm_aiter_fp4] " + f"{label}: " + f"pass={passed / total:.4%} ({passed}/{total}) " + f"fail={_format_observed_rate(failed, total)} " + f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} " + f"atol={atol:g} " + f"rtol={rtol:g} " + ) + if max_atol is not None: + above_max_count = int((abs_diff > max_atol).sum().item()) + msg += ( + f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} " + f"allowed_above_max={_format_allowed_rate(0.0, total)} " + ) + msg += ( + f"max_abs={max_abs:.6g} " + f"mean_abs={mean_abs:.6g} " + f"p99_abs={p99_abs:.6g} " + f"p999_abs={p999_abs:.6g} " + f"worst_ratio={worst_ratio:.6g}" + ) + print(msg) + if failed > 0: + warnings.warn(msg, stacklevel=2) + + +# Env and enablement tests ------------------------------------------------ + + +def test_fp4_env_defaults(): + """ROCm FP4 env defaults should stay stable for the AITER gates.""" + import vllm.envs as envs + + assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False + assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True + + +@pytest.mark.parametrize( + ( + "use_aiter", + "use_fp4_asm_gemm", + "use_fp4bmm", + ), + [ + (True, True, True), + (True, True, False), + (True, False, True), + (False, True, True), + ], +) +def test_rocm_aiter_fp4_enablement_follows_env_and_arch( + use_aiter, + use_fp4_asm_gemm, + use_fp4bmm, + monkeypatch, +): + """The ROCm FP4 AITER gates should depend only on the env toggles and the + gfx950 hardware check. + + Expected values are derived from the *actual* platform rather than a patched + arch (gfx950 -> MI355, gfx942 -> MI300). A failure in this group therefore + points straight at the hardware it ran on instead of a mocked value. + """ + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + + on_gfx950_value = on_gfx950() + expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value + expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv( + "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", + "1" if use_fp4_asm_gemm else "0", + ) + mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0") + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + assert ( + rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm + ) + assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm + + +# Large-shape MXFP4 wrapper tests ----------------------------------------- + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("shape", LLAMA_SHAPES) +def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype): + """The public vLLM MXFP4 QDQ helper should stay exact against the torch + reference even on larger Llama-like shapes.""" + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + quant_dequant_mxfp4, + ) + + torch.set_default_device("cuda") + torch.manual_seed(0) + rows, cols = shape + x = torch.randn(rows, cols, dtype=dtype) + out = quant_dequant_mxfp4(x) + ref = qdq_mxfp4_torch(x, "even") + + assert out.shape == x.shape + assert out.dtype == dtype + assert torch.isfinite(out).all() + torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0) + + +def test_vllm_quant_dequant_mxfp4_is_deterministic(): + """The public vLLM MXFP4 QDQ helper should stay bitwise deterministic.""" + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + quant_dequant_mxfp4, + ) + + torch.set_default_device("cuda") + torch.manual_seed(3) + + x = torch.randn(64, 128, dtype=torch.bfloat16) + + _assert_deterministic(quant_dequant_mxfp4, x, n_runs=4) + + +# Triton MXFP4 quant tests ------------------------------------------------ + + +def test_aiter_dynamic_mxfp4_quant_output_format(): + """dynamic_mxfp4_quant returns packed uint8 FP4 values and E8M0 uint8 scales. + + OCP MXFP4: block_size=32, 2 FP4 E2M1 values packed per byte. + Scale shape: (M, K // 32); one E8M0 exponent byte per 32-element block. + """ + _assert_aiter_supported() + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + torch.set_default_device("cuda") + M, K = 64, 256 + x = torch.randn(M, K, dtype=torch.bfloat16) + + x_fp4, x_scale = dynamic_mxfp4_quant(x) + + # FP4 values packed 2-per-byte; shape (M, K // 2) + assert x_fp4.shape == (M, K // 2), ( + f"Expected fp4 shape ({M}, {K // 2}), got {x_fp4.shape}" + ) + assert x_fp4.dtype == torch.uint8 + + # One E8M0 scale byte per 32-element block; shape (M, K // 32) + assert x_scale.shape == (M, K // 32), ( + f"Expected scale shape ({M}, {K // 32}), got {x_scale.shape}" + ) + assert x_scale.dtype == torch.uint8 + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize( + "shape", + [ + (128, 4096), # Llama-7B hidden + (256, 11008), # Llama-7B FFN + (64, 14336), # Llama-70B FFN + (512, 8192), # Llama-70B hidden + (32, 28672), # DeepSeek-style large FFN + ], +) +def test_aiter_dynamic_mxfp4_quant_llama_shapes(shape, dtype): + """MXFP4 quantization output format is correct for Llama-class weight shapes. + + Targets B200 parity: NVIDIA nvfp4 tests parametrize over Llama shapes + (7168, 14336, 28672...). This covers gfx950 MXFP4 quant format validation. + """ + _assert_aiter_supported() + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + torch.set_default_device("cuda") + M, K = shape + x = torch.randn(M, K, dtype=dtype) + + x_fp4, x_scale = dynamic_mxfp4_quant(x) + + assert x_fp4.shape == (M, K // 2) + assert x_fp4.dtype == torch.uint8 + assert x_scale.shape == (M, K // 32) + assert x_scale.dtype == torch.uint8 + assert not torch.any(torch.isnan(x_fp4.float())) + + +def test_aiter_dynamic_mxfp4_quant_determinism(): + """dynamic_mxfp4_quant is bitwise deterministic across repeated runs.""" + _assert_aiter_supported() + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + torch.set_default_device("cuda") + torch.manual_seed(7) + x = torch.randn(128, 256, dtype=torch.bfloat16) + + _assert_deterministic(dynamic_mxfp4_quant, x, n_runs=4) + + +# gfx950 hardware FP4 GEMM tests ------------------------------------------ + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +@pytest.mark.parametrize( + "shape", + [ + (64, 128, 64), # small square-ish + (128, 256, 128), # medium + (128, 4096, 4096), # Llama-7B hidden square + (256, 4096, 11008), # Llama-7B FFN + (64, 8192, 28672), # Llama-70B FFN + ], +) +def test_aiter_fp4_gemm_a4w4_accuracy(shape): + """AITER A4W4 FP4 GEMM output is close to matmul on dequantized weights. + + Tests gfx950 parity with B200 test_nvfp4_scaled_mm.py: E2M1 x E2M1 GEMM + with block-scaled quantization, multiple Llama-class shapes. + Requires gfx950 hardware; gracefully skips elsewhere. + """ + atol = GEMM_ATOL + rtol = GEMM_RTOL + pass_rate = A4W4_GEMM_PASS_RATES[shape] + _assert_aiter_supported() + from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4 + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + quant_dequant_mxfp4, + ) + + torch.set_default_device("cuda") + torch.manual_seed(0) + M, K, N = shape + + A = torch.randn(M, K, dtype=torch.bfloat16) + B = torch.randn(N, K, dtype=torch.bfloat16) # weight stored as (N, K) + + # Quantize + A_fp4, A_scale = dynamic_mxfp4_quant(A) + B_fp4, B_scale = dynamic_mxfp4_quant(B) + + # FP4 GEMM computes A_fp4 (M, K) @ B_fp4.T (K, N) into (M, N). + out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale) + + assert out.shape == (M, N), f"Expected ({M}, {N}), got {out.shape}" + assert out.dtype == torch.bfloat16 + assert not torch.any(torch.isnan(out)) + + # Reference: matmul on dequantized FP4 inputs (quark roundtrip) + A_dq = quant_dequant_mxfp4(A) + B_dq = quant_dequant_mxfp4(B) + ref = torch.matmul(A_dq.float(), B_dq.t().float()) + + # FP4 has only 3 mantissa bits; large K amplifies the accumulation error. + # These budgets were measured on gfx950 and keep p99/p999 drift visible + # without pretending the kernel is close to BF16 matmul everywhere. + _print_close_stats( + f"a4w4_gemm shape={shape}", + out.float(), + ref, + atol=atol, + rtol=rtol, + pass_rate=pass_rate, + max_atol=GEMM_MAX_VIOLATION_FACTOR * atol, + ) + _assert_accurate( + out.float(), + ref, + atol=atol, + rtol=rtol, + pass_rate=pass_rate, + max_violation_factor=GEMM_MAX_VIOLATION_FACTOR, + ) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +@pytest.mark.parametrize( + "shape", + PRESHUFFLED_SHAPES, +) +def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape): + """The preshuffled FP4 GEMM path should run on tuned shapes with the same + packed inputs vLLM uses in production. + + This kernel only advertises a narrow tuned `(N, K)` set on gfx950. The + important contract here is that those tuned shapes execute with the + production-style shuffled weights and stay deterministic. + """ + _assert_aiter_supported() + from aiter import per_1x32_f4_quant_hip + from aiter.ops.shuffle import shuffle_weight + from aiter.ops.triton.gemm_afp4wfp4 import ( + gemm_afp4wfp4_preshuffled_weight_scales, + ) + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + from vllm._aiter_ops import rocm_aiter_ops + + torch.set_default_device("cuda") + torch.manual_seed(1) + M, K, N = shape + + assert M <= 64 + assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K) + + A = torch.randn(M, K, dtype=torch.bfloat16) + B = torch.randn(N, K, dtype=torch.bfloat16) + + B_fp4, B_scale = dynamic_mxfp4_quant(B) + A_q, A_s = per_1x32_f4_quant_hip(A, shuffle=M >= 32) + + scale_rows, scale_cols = B_scale.shape + B_scale = ( + B_scale.view(scale_rows // 32, 2, 16, scale_cols // 8, 2, 4, 1) + .permute(0, 3, 5, 2, 4, 1, 6) + .contiguous() + .view(scale_rows, scale_cols) + ) + B_fp4 = shuffle_weight(B_fp4, layout=(16, 16)) + + if M >= 32: + A_s = A_s.contiguous().view(torch.uint8).reshape(A_s.shape[0] // 32, -1) + else: + A_s = A_s[:M, ...].contiguous().view(torch.uint8) + B_scale = B_scale.contiguous().view(torch.uint8).reshape(B_scale.shape[0] // 32, -1) + + def run_preshuffled() -> torch.Tensor: + y = torch.empty(M, N, device="cuda", dtype=torch.bfloat16) + return gemm_afp4wfp4_preshuffled_weight_scales( + A_q.contiguous().view(torch.uint8), + B_fp4.contiguous().view(torch.uint8).reshape(B_fp4.shape[0] // 16, -1), + A_s, + B_scale, + torch.bfloat16, + y, + ) + + out = run_preshuffled() + assert out.shape == (M, N) + assert out.dtype == torch.bfloat16 + assert torch.isfinite(out).all() + + _assert_deterministic(run_preshuffled, n_runs=3) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_aiter_fp4_gemm_a4w4_determinism(): + """AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs. + + Targets parity with B200 FP4 determinism requirements. + Requires gfx950. + """ + _assert_aiter_supported() + from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4 + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + torch.set_default_device("cuda") + torch.manual_seed(2) + + M, K, N = 128, 256, 128 + A = torch.randn(M, K, dtype=torch.bfloat16) + B = torch.randn(N, K, dtype=torch.bfloat16) + + A_fp4, A_scale = dynamic_mxfp4_quant(A) + B_fp4, B_scale = dynamic_mxfp4_quant(B) + + def run_gemm(): + return gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale) + + _assert_deterministic(run_gemm, n_runs=4) + + +# gfx950 hardware FP4 dynamic quantization -------------------------------- + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +@pytest.mark.parametrize( + "shape", + [ + (128, 256), + (256, 4096), + (64, 14336), + ], +) +def test_aiter_hardware_fp4_dynamic_quant_format(shape): + """aiter hardware FP4 dynamic quant produces correct output format. + + Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1). + Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output. + Requires gfx950. + """ + _assert_aiter_supported() + + from aiter import dynamic_per_group_scaled_quant_fp4 + + torch.set_default_device("cuda") + M, K = shape + group_size = 32 + + x = torch.randn(M, K, dtype=torch.bfloat16) + out_fp4 = torch.empty(M, K // 2, dtype=torch.uint8) + scales = torch.empty(M, K // group_size, dtype=torch.uint8) + + dynamic_per_group_scaled_quant_fp4(out_fp4, x, scales, group_size) + + assert out_fp4.shape == (M, K // 2), ( + f"Shape {shape}: expected fp4 ({M}, {K // 2}), got {out_fp4.shape}" + ) + assert scales.shape == (M, K // group_size), ( + f"Shape {shape}: expected scale ({M}, {K // group_size}), got {scales.shape}" + ) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_aiter_hardware_fp4_quant_vs_triton(): + """Hardware FP4 quant should dequantize back to the same MXFP4 values as + the Triton path. + + The packed bytes are not identical on gfx950, but with + ``shuffle_scale=False`` the dequantized results match tightly. + """ + _assert_aiter_supported() + + from aiter import dynamic_per_group_scaled_quant_fp4 + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + torch.set_default_device("cuda") + torch.manual_seed(5) + + M, K = 128, 256 + group_size = 32 + x = torch.randn(M, K, dtype=torch.bfloat16) + + # Hardware path + out_hw = torch.empty(M, K // 2, dtype=torch.uint8) + scales_hw = torch.empty(M, K // group_size, dtype=torch.uint8) + dynamic_per_group_scaled_quant_fp4( + out_hw, x, scales_hw, group_size, shuffle_scale=False + ) + + # Triton path + out_triton, scales_triton = dynamic_mxfp4_quant(x) + + assert out_hw.shape == out_triton.shape + assert scales_hw.shape == scales_triton.shape + + dq_hw = dq_mxfp4_torch(out_hw, scales_hw, torch.bfloat16) + dq_triton = dq_mxfp4_torch(out_triton, scales_triton, torch.bfloat16) + + _print_close_stats( + "hardware_quant_vs_triton", + dq_hw.float(), + dq_triton.float(), + atol=0.25, + rtol=0.0, + ) + torch.testing.assert_close(dq_hw, dq_triton, atol=0.25, rtol=0.0) + + +# ROCm skinny GEMM FP4 tests ---------------------------------------------- + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +@pytest.mark.parametrize("M", [1, 2, 4, 8]) # decode / skinny batch sizes +@pytest.mark.parametrize("N, K", [(4096, 4096), (4096, 11008), (8192, 8192)]) +def test_aiter_fp4_gemm_skinny_shapes(M, N, K): + """FP4 GEMM accuracy for skinny (small-M) shapes (decode phase). + + Skinny GEMMs (M=1..8) are the bottleneck in decode phase. + Tests parity with B200 nvfp4_scaled_mm tests at decode batch sizes. + Requires gfx950. + """ + atol = GEMM_ATOL + rtol = GEMM_RTOL + pass_rate = SKINNY_GEMM_PASS_RATES[(N, K)] + _assert_aiter_supported() + from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4 + from aiter.ops.triton.quant import dynamic_mxfp4_quant + + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + quant_dequant_mxfp4, + ) + + torch.set_default_device("cuda") + torch.manual_seed(0) + + A = torch.randn(M, K, dtype=torch.bfloat16) + B = torch.randn(N, K, dtype=torch.bfloat16) + + A_fp4, A_scale = dynamic_mxfp4_quant(A) + B_fp4, B_scale = dynamic_mxfp4_quant(B) + + out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale) + + assert out.shape == (M, N) + assert not torch.any(torch.isnan(out)) + + A_dq = quant_dequant_mxfp4(A) + B_dq = quant_dequant_mxfp4(B) + ref = torch.matmul(A_dq.float(), B_dq.t().float()) + + _print_close_stats( + f"skinny_gemm M={M} N={N} K={K}", + out.float(), + ref, + atol=atol, + rtol=rtol, + pass_rate=pass_rate, + max_atol=GEMM_MAX_VIOLATION_FACTOR * atol, + ) + _assert_accurate( + out.float(), + ref, + atol=atol, + rtol=rtol, + pass_rate=pass_rate, + max_violation_factor=GEMM_MAX_VIOLATION_FACTOR, + ) diff --git a/tests/kernels/utils.py b/tests/kernels/utils.py index cc1d1bbf88..8bd63872f3 100644 --- a/tests/kernels/utils.py +++ b/tests/kernels/utils.py @@ -36,6 +36,63 @@ ALL_OPCHECK_TEST_UTILS: tuple[str, ...] = ( ) +def _assert_accurate( + actual: torch.Tensor, + expected: torch.Tensor, + atol: float, + rtol: float = 0.0, + pass_rate: float = 0.99999, + max_violation_factor: float = 3.0, +) -> None: + """Check numeric accuracy with pass-rate, max-error, and mean-error bounds.""" + a = actual.detach().float().flatten() + e = expected.detach().float().flatten() + + abs_err = (a - e).abs() + tol = atol + rtol * e.abs() + + rate = (abs_err <= tol).float().mean().item() + assert rate >= pass_rate, ( + f"Accuracy pass rate {rate:.6f} < {pass_rate} (atol={atol}, rtol={rtol})" + ) + + max_err = abs_err.max().item() + assert max_err <= max_violation_factor * atol, ( + f"Max absolute error {max_err:.6f} exceeds {max_violation_factor} * atol={atol}" + ) + + mean_err = abs_err.mean().item() + assert mean_err <= atol * 0.25, ( + f"Mean absolute error {mean_err:.6f} >= atol * 0.25 = {atol * 0.25:.6f}" + ) + + +def _assert_deterministic( + fn, + *args, + n_runs: int = 4, + **kwargs, +) -> None: + """Verify that repeated calls produce bitwise-identical tensor outputs.""" + + def _collect(result: Any) -> list[torch.Tensor]: + if isinstance(result, torch.Tensor): + return [result.detach().clone()] + if isinstance(result, (tuple, list)): + return [t.detach().clone() for t in result if isinstance(t, torch.Tensor)] + raise TypeError(f"Unexpected return type {type(result)}") + + reference = _collect(fn(*args, **kwargs)) + + for run in range(1, n_runs): + outputs = _collect(fn(*args, **kwargs)) + for idx, (ref, out) in enumerate(zip(reference, outputs)): + assert torch.equal(ref, out), ( + f"Run {run}: output[{idx}] differs from run 0 " + f"(max diff = {(out.float() - ref.float()).abs().max().item():.2e})" + ) + + class QKVInputs(NamedTuple): """ Data structure for representing unpacked attention inputs,