[ROCm][Perf] Fused MoE W4A16 HIP kernel for AMD RDNA3 (gfx1100) (#44075)

Signed-off-by: JartX <sagformas@epdcenter.es>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
JartX
2026-06-06 15:30:39 -05:00
committed by GitHub
co-authored by TJian
parent fa27d4e9cf
commit 062b05ff3a
11 changed files with 1906 additions and 5 deletions
@@ -0,0 +1,503 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compile-guard tests for the ROCm RDNA3 W4A16 kernels (dense + MoE).
Verifies that the gfx1100 compilation and dispatch guards are hermetic:
- On gfx1100: all ops exist, dispatch selects RDNA3 kernels.
- On CDNA (gfx942/gfx950) or other non-gfx1100: ops must NOT exist,
dispatch must fall through to Triton/Marlin, and no RDNA3 code
path is reachable.
The negative (non-gfx1100) tests verify at three layers:
1. Compile-level: on non-gfx1100 hardware, the RDNA3 ops are absent
from the compiled _rocm_C extension — real binary verification.
2. Static source analysis: parses CMakeLists.txt and torch_bindings.cpp
to verify that all RDNA3 .cu files and op registrations are inside
gfx1100-only guards.
3. Runtime mock: patches on_gfx1100() to False and verifies that the
Python dispatch chain rejects the RDNA3 path.
Run `pytest tests/kernels/quantization/test_rdna3_compile_guards.py`.
"""
from pathlib import Path
from unittest.mock import patch
import pytest
import regex as re
import torch
import vllm
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 compile-guard tests are ROCm-only", allow_module_level=True)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
gfx1100_only = pytest.mark.skipif(
not on_gfx1100(),
reason="Requires gfx1100 hardware",
)
not_gfx1100 = pytest.mark.skipif(
on_gfx1100(),
reason="This test verifies non-gfx1100 builds — skip on gfx1100",
)
RDNA3_OPS = ["gptq_gemm_rdna3", "gptq_gemm_rdna3_wmma", "moe_gptq_gemm_rdna3"]
RDNA3_CU_FILES = [
"q_gemm_rdna3.cu",
"q_gemm_rdna3_wmma.cu",
"moe_q_gemm_rdna3.cu",
]
def _find_repo_root() -> Path | None:
"""Walk up from this file to find the repo root (has CMakeLists.txt)."""
for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
if (parent / "CMakeLists.txt").exists() and (parent / "csrc").is_dir():
return parent
return None
REPO_ROOT = _find_repo_root()
# Directory of the *installed* vllm python package. The .py guard checks read
# from here so they verify the code that is actually imported at runtime — this
# works even on CI images that ship the wheel instead of the python source tree
# (where only csrc/ + CMakeLists.txt are checked out for building).
VLLM_PKG_DIR: Path | None = (
Path(vllm.__file__).parent if getattr(vllm, "__file__", None) else None
)
needs_source = pytest.mark.skipif(
REPO_ROOT is None,
reason="C/CMake source tree not available (installed package only)",
)
def _read_source_or_skip(*relparts: str) -> str:
"""Read a C/CMake source file from the repo tree, or skip if absent.
Used for csrc/ and CMakeLists.txt — these only exist in a source checkout,
not in the installed wheel.
"""
assert REPO_ROOT is not None # callers are gated by @needs_source
path = REPO_ROOT.joinpath(*relparts)
if not path.exists():
pytest.skip(f"{path} not present in this source tree")
return path.read_text()
def _read_pkg_source_or_skip(*relparts: str) -> str:
"""Read a python source file from the installed vllm package.
Reflects the code actually loaded at runtime, so these guard checks run in
CI against the wheel — no source checkout required. Only skips for an
exotic install layout (namespace/zipimport) where __file__ is unavailable.
"""
if VLLM_PKG_DIR is None:
pytest.skip("vllm package directory not resolvable (zip/namespace?)")
assert VLLM_PKG_DIR is not None # narrow for mypy (skip above is NoReturn)
path = VLLM_PKG_DIR.joinpath(*relparts)
if not path.exists():
pytest.skip(f"{path} not present in installed vllm package")
return path.read_text()
# ============================================================================
# Part A: POSITIVE — on gfx1100, ops exist and dispatch works
# ============================================================================
@gfx1100_only
@pytest.mark.parametrize("op_name", RDNA3_OPS)
def test_op_registered_on_gfx1100(op_name):
"""On gfx1100, all RDNA3 ops must be registered in _rocm_C."""
assert hasattr(torch.ops, "_rocm_C"), "_rocm_C module not loaded"
assert hasattr(torch.ops._rocm_C, op_name), (
f"_rocm_C.{op_name} not registered — "
"check CMakeLists.txt VLLM_ROCM_HAS_GFX1100 "
"and torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100"
)
@gfx1100_only
def test_all_ops_present_or_all_absent():
"""The 3 RDNA3 ops are behind the same #ifdef — all present or all absent.
Catches someone accidentally moving an op outside the guard.
"""
has_rocm_c = hasattr(torch.ops, "_rocm_C")
if not has_rocm_c:
pytest.skip("_rocm_C not loaded")
present = {op: hasattr(torch.ops._rocm_C, op) for op in RDNA3_OPS}
values = set(present.values())
assert len(values) == 1, (
f"Guard inconsistency — some RDNA3 ops registered, others not: "
f"{present}. Check torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100 block."
)
# ============================================================================
# Part B: NEGATIVE — compile-level verification on non-gfx1100
# ============================================================================
@not_gfx1100
@pytest.mark.parametrize("op_name", RDNA3_OPS)
def test_op_absent_on_non_gfx1100(op_name):
"""On non-gfx1100 (CDNA), RDNA3 ops must NOT exist in _rocm_C.
This is the real compile-level check: the binary was built without
gfx1100 support, so the ops should not have been compiled or registered.
"""
if not hasattr(torch.ops, "_rocm_C"):
return
assert not hasattr(torch.ops._rocm_C, op_name), (
f"_rocm_C.{op_name} is registered on non-gfx1100 hardware — "
"compile guard is broken: check CMakeLists.txt "
"VLLM_ROCM_HAS_GFX1100 and torch_bindings.cpp #ifdef"
)
@not_gfx1100
def test_rocm_moe_not_supported_on_non_gfx1100():
"""rocm_moe.is_supported() must return False on non-gfx1100 hardware."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
rocm_moe,
)
wq = type("WQ", (), {"num_bits": 4})()
assert rocm_moe.is_supported(wq) is False, (
"rocm_moe.is_supported() returned True on non-gfx1100 — "
"dispatch guard is broken"
)
@not_gfx1100
def test_dense_kernel_rejects_on_non_gfx1100():
"""RDNA3W4A16LinearKernel.can_implement must reject on non-gfx1100."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
RDNA3W4A16LinearKernel,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
# ============================================================================
# Part C: Static source analysis (build-level guards)
# ============================================================================
@needs_source
class TestCMakeGuards:
"""Verify CMakeLists.txt only compiles RDNA3 .cu files for gfx1100."""
@staticmethod
def _read_cmake():
return _read_source_or_skip("CMakeLists.txt")
def test_rdna3_cu_files_inside_gfx1100_conditional(self):
"""All RDNA3 .cu files must be listed inside the
``if(VLLM_GPU_ARCHES MATCHES "gfx1100")`` block, not unconditionally.
"""
cmake = self._read_cmake()
for cu_file in RDNA3_CU_FILES:
assert cu_file in cmake, f"{cu_file} not found in CMakeLists.txt"
lines = cmake.splitlines()
in_gfx1100_block = False
for line in lines:
if 'VLLM_GPU_ARCHES MATCHES "gfx1100"' in line:
in_gfx1100_block = True
if in_gfx1100_block and "endif()" in line:
in_gfx1100_block = False
if cu_file in line:
assert in_gfx1100_block, (
f"{cu_file} is listed OUTSIDE the gfx1100 "
f"conditional in CMakeLists.txt — CDNA builds "
f"would compile RDNA3 code. Line: {line.strip()}"
)
def test_compile_definition_only_for_gfx1100(self):
"""VLLM_ROCM_GFX1100 compile definition must be conditional."""
cmake = self._read_cmake()
lines = cmake.splitlines()
in_gfx1100_block = False
for line in lines:
if "VLLM_ROCM_HAS_GFX1100)" in line:
in_gfx1100_block = True
if in_gfx1100_block and "endif()" in line:
in_gfx1100_block = False
if "VLLM_ROCM_GFX1100" in line and "target_compile_definitions" in line:
assert in_gfx1100_block, (
"VLLM_ROCM_GFX1100 compile definition is set outside "
"the VLLM_ROCM_HAS_GFX1100 conditional — CDNA builds "
f"would define it. Line: {line.strip()}"
)
@needs_source
class TestTorchBindingsGuards:
"""Verify torch_bindings.cpp gates all RDNA3 ops behind #ifdef."""
@staticmethod
def _read_bindings():
return _read_source_or_skip("csrc", "rocm", "torch_bindings.cpp")
def test_all_rdna3_ops_inside_ifdef(self):
"""Every rdna3 op def/impl must be between #ifdef VLLM_ROCM_GFX1100
and #endif. If any is outside, a CDNA build would try to register
the op and link a symbol that doesn't exist.
"""
src = self._read_bindings()
lines = src.splitlines()
inside_guard = False
rdna3_lines_outside = []
for i, line in enumerate(lines, 1):
if "#ifdef VLLM_ROCM_GFX1100" in line:
inside_guard = True
elif line.strip() == "#endif" and inside_guard:
inside_guard = False
if (
"rdna3" in line.lower()
and not line.strip().startswith("//")
and not inside_guard
):
rdna3_lines_outside.append((i, line.strip()))
assert not rdna3_lines_outside, (
"RDNA3 op references found OUTSIDE #ifdef VLLM_ROCM_GFX1100 "
"in torch_bindings.cpp — these would break CDNA builds:\n"
+ "\n".join(f" L{n}: {s}" for n, s in rdna3_lines_outside)
)
def test_no_unconditional_rdna3_includes(self):
"""No #include of RDNA3-specific headers outside the guard."""
src = self._read_bindings()
lines = src.splitlines()
inside_guard = False
for i, line in enumerate(lines, 1):
if "#ifdef VLLM_ROCM_GFX1100" in line:
inside_guard = True
elif line.strip() == "#endif" and inside_guard:
inside_guard = False
if "#include" in line and "rdna3" in line.lower():
assert inside_guard, (
f"L{i}: RDNA3 include outside gfx1100 guard: {line.strip()}"
)
class TestCustomOpsGuards:
"""Verify _custom_ops.py gates register_fake behind hasattr checks."""
@staticmethod
def _read_custom_ops():
return _read_pkg_source_or_skip("_custom_ops.py")
def test_register_fake_guarded_by_hasattr(self):
"""Every register_fake for an RDNA3 op must be preceded by a hasattr
check — otherwise it would crash on import on CDNA where the ops
don't exist.
"""
src = self._read_custom_ops()
for op in RDNA3_OPS:
pattern = rf'register_fake\(\s*"_rocm_C::{op}"\s*\)'
match = re.search(pattern, src)
if match is None:
continue
preceding = src[: match.start()]
last_hasattr = preceding.rfind(f'hasattr(torch.ops._rocm_C, "{op}")')
assert last_hasattr != -1, (
f'register_fake("_rocm_C::{op}") is not preceded by a '
f"hasattr check — would crash on CDNA import"
)
gap = preceding[last_hasattr:].count("\n")
assert gap <= 5, (
f"hasattr guard for {op} is {gap} lines before "
f"register_fake — suspiciously far; verify it's the "
f"actual guard and not a coincidence"
)
def test_no_toplevel_rocm_c_import(self):
"""No top-level ``from vllm._rocm_C import`` — would crash on CDNA."""
src = self._read_custom_ops()
for line in src.splitlines():
stripped = line.strip()
if stripped.startswith("#") or stripped.startswith("//"):
continue
assert "from vllm._rocm_C import" not in stripped, (
f"Top-level import of _rocm_C in _custom_ops.py would "
f"crash on CDNA: {stripped}"
)
# ============================================================================
# Part D: Runtime mock (simulate CDNA on gfx1100 hardware)
# ============================================================================
class _FakeWeightQuant:
"""Minimal stand-in for a weight quantization config."""
def __init__(self, num_bits):
self.num_bits = num_bits
class TestMoEDispatchMocked:
"""Mock on_gfx1100() to False and verify RDNA3 MoE is unreachable."""
def test_is_supported_false_when_mocked_cdna(self):
"""rocm_moe.is_supported() must return False when not on gfx1100."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
rocm_moe,
)
with patch("vllm.platforms.rocm.on_gfx1100", return_value=False):
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
@pytest.mark.parametrize("num_bits", [2, 3, 8, 16])
def test_is_supported_rejects_non_w4(self, num_bits):
"""is_supported() rejects non-4-bit even before checking arch."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
rocm_moe,
)
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=num_bits)) is False
def test_is_supported_false_when_op_missing(self):
"""is_supported() returns False when the C++ op doesn't exist."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
rocm_moe,
)
fake_rocm_c = type("FakeRocmC", (), {"gptq_gemm_rdna3": None})()
with patch.object(torch, "ops", create=True) as mock_ops:
mock_ops._rocm_C = fake_rocm_c
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
def test_is_supported_false_when_rocm_c_absent(self):
"""is_supported() returns False when _rocm_C doesn't exist at all."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
rocm_moe,
)
fake_ops = type("FakeOps", (), {})()
with patch.object(torch, "ops", fake_ops):
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
class TestDenseKernelSelectionMocked:
"""Mock on_gfx1100() and verify dense RDNA3 kernel is not selected."""
@gfx1100_only
def test_can_implement_rejects_when_mocked_cdna(self):
"""RDNA3W4A16LinearKernel.can_implement must reject on mocked CDNA."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
RDNA3W4A16LinearKernel,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is True
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
patch("vllm.platforms.rocm._ON_GFX1100", False),
):
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 kernel accepted on simulated CDNA: {reason}"
@gfx1100_only
def test_chooser_skips_rdna3_when_mocked_cdna(self):
"""choose_mp_linear_kernel must NOT return RDNA3 on mocked CDNA."""
from vllm.model_executor.kernels.linear import (
choose_mp_linear_kernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
patch("vllm.platforms.rocm._ON_GFX1100", False),
):
chosen = choose_mp_linear_kernel(config)
assert chosen.__name__ != "RDNA3W4A16LinearKernel", (
"RDNA3 kernel was selected on simulated CDNA — "
"choose_mp_linear_kernel guard is broken"
)
class TestCompressedTensorsMoEDispatchGuard:
"""Verify compressed_tensors_moe.py only enters rocm_moe under is_rocm()."""
def test_rocm_guard_in_dispatch_source(self):
"""The rocm_moe import and call must be inside an is_rocm() check."""
src = _read_pkg_source_or_skip(
"model_executor",
"layers",
"quantization",
"compressed_tensors",
"compressed_tensors_moe",
"compressed_tensors_moe.py",
)
lines = src.splitlines()
for i, line in enumerate(lines, 1):
stripped = line.strip()
if "rocm_moe" in stripped and not stripped.startswith("#"):
found_guard = False
for j in range(i - 1, max(0, i - 15), -1):
if "is_rocm()" in lines[j - 1]:
found_guard = True
break
assert found_guard, (
f"L{i}: rocm_moe reference not protected by "
f"is_rocm() guard: {stripped}"
)
@@ -0,0 +1,367 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the ROCm RDNA3 fused MoE W4A16 HIP kernel (gfx1100).
Tests ``moe_gptq_gemm_rdna3`` against the dense ``gptq_gemm_rdna3`` as
reference: builds RDNA3-format weights (shuffled int32, synthesized qzeros),
runs the fused MoE kernel, and compares per-expert results.
Model parameters taken from:
- cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit
(hidden=2048, inter=768, E=128, top_k=8, G=32)
- Qwen3.6-35B-A3B-GPTQ-W4A16-G32
(hidden=2048, inter=512, E=256, top_k=8, G=32)
Run `pytest tests/kernels/quantization/test_rdna3_moe_w4a16.py`.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 MoE W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm import _custom_ops as ops # noqa: E402
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( # noqa: E402
moe_align_block_size,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
pack_quantized_values_into_int32,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
device = "cuda"
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "moe_gptq_gemm_rdna3")
),
reason="Requires gfx1100 with moe_gptq_gemm_rdna3 op",
)
# Model configurations: real K/N/top_k/group_size dims, E capped at 16 to
# fit in test GPU memory (full E=128/256 would need >20GB for weights alone).
# Kernel behavior is E-independent (per-expert tiling), so E=16 is sufficient.
MODEL_CONFIGS = [
# cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit dims (E capped)
pytest.param(16, 2048, 768, 8, 32, id="Qwen3-30B-A3B"),
# Qwen3.6-35B-A3B-GPTQ-W4A16-G32 dims (E capped)
pytest.param(16, 2048, 512, 8, 32, id="Qwen3.6-35B-A3B"),
]
# Token counts: decode (1), small batch (4), medium (16), prefill (64)
NUM_TOKENS = [1, 4, 16, 64, 256, 512]
def _make_packed_weights(E, K, N):
"""Create random 4-bit packed weights [E, K/8, N] int32 + shuffle."""
w = torch.randint(0, 16, (E, K, N), dtype=torch.int32, device=device)
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
for i in range(8):
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
for e in range(E):
we = packed[e].contiguous()
ops.gptq_shuffle(we, g_idx, 4)
packed[e] = we
return packed
def _make_scales(E, groups, N, dtype):
return torch.rand(E, groups, N, dtype=dtype, device=device) * 0.1
def _make_qzeros(E, groups, N):
zeros = torch.full(
(groups, N),
scalar_types.uint4b8.bias - 1,
dtype=torch.int32,
device=device,
)
qz = pack_quantized_values_into_int32(
zeros,
scalar_types.uint4b8,
packed_dim=1,
)
return qz.unsqueeze(0).expand(E, -1, -1).contiguous()
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("block_size_m", [1, 4])
def test_fused_moe_w1_matches_dense(
E, K, N_inter, top_k, group_size, M, dtype, block_size_m
):
"""w1 GEMM via fused kernel matches per-expert dense kernel."""
N_gate_up = N_inter * 2
groups = K // group_size
torch.manual_seed(42)
x = torch.randn(M, K, dtype=dtype, device=device)
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, groups, N_gate_up, dtype)
w13_z = _make_qzeros(E, groups, N_gate_up)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
# Fused kernel
fused_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
fused_out,
w13,
w13_s,
w13_z,
torch.empty(0, device=device),
si,
ei,
ntp,
top_k,
block_size_m,
False,
0,
)
# Per-expert dense reference
ref_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
for m in range(M):
for k in range(top_k):
e = topk_ids[m, k].item()
flat = m * top_k + k
ref = ops.gptq_gemm_rdna3(
x[m : m + 1],
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
ref_out[flat] = ref.squeeze()
# Split-K atomics can cause minor fp16/bf16 rounding differences
# at large K (e.g. K=2048 → 8 K-blocks). Use allclose, not equal.
atol = 0.5 if dtype == torch.bfloat16 else 0.1
assert torch.allclose(fused_out, ref_out, atol=atol, rtol=0.01), (
f"max diff: {(fused_out - ref_out).abs().max().item()}"
)
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_fused_moe_output_topk_reduces(E, K, N_inter, top_k, group_size, M, dtype):
"""output_topk fuses moe_sum: multiple experts write to same output row."""
groups = K // group_size
torch.manual_seed(123)
x = torch.randn(M * top_k, K, dtype=dtype, device=device)
w = _make_packed_weights(E, K, N_inter)
ws = _make_scales(E, groups, N_inter, dtype)
wz = _make_qzeros(E, groups, N_inter)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
torch.randn(M, top_k, device=device),
dim=-1,
).float()
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
# Without output_topk: write to [M*top_k, N] then moe_sum
flat_out = torch.zeros(M * top_k, N_inter, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
flat_out,
w,
ws,
wz,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
0,
)
ref = torch.zeros(M, N_inter, dtype=dtype, device=device)
ops.moe_sum(flat_out.view(M, top_k, N_inter), ref)
# With output_topk: write directly to [M, N]
fused = torch.zeros(M, N_inter, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
fused,
w,
ws,
wz,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
top_k,
)
atol = 1.0 if dtype == torch.bfloat16 else 0.1
assert torch.allclose(fused, ref, atol=atol, rtol=0.01), (
f"max diff: {(fused - ref).abs().max().item()}"
)
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
"""Full MoE forward: w1 + silu_and_mul + w2 with output_topk reduce."""
N_gate_up = N_inter * 2
hidden = K
torch.manual_seed(7)
x = torch.randn(M, K, dtype=dtype, device=device)
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, K // group_size, N_gate_up, dtype)
w13_z = _make_qzeros(E, K // group_size, N_gate_up)
w2 = _make_packed_weights(E, N_inter, hidden)
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
torch.randn(M, top_k, device=device),
dim=-1,
).float()
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
# Fused path (what apply() does)
w1_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
w1_out,
w13,
w13_s,
w13_z,
torch.empty(0, device=device),
si,
ei,
ntp,
top_k,
1,
False,
0,
)
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
fused = torch.zeros(M, hidden, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
act_out,
fused,
w2,
w2_s,
w2_z,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
top_k,
)
# Per-expert reference
ref = torch.zeros(M, hidden, dtype=dtype, device=device)
for m_idx in range(M):
for k_idx in range(top_k):
e = topk_ids[m_idx, k_idx].item()
w = topk_w[m_idx, k_idx].item()
r1 = ops.gptq_gemm_rdna3(
x[m_idx : m_idx + 1],
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
a = torch.empty(1, N_inter, dtype=dtype, device=device)
apply_moe_activation(MoEActivation.SILU, a, r1)
r2 = ops.gptq_gemm_rdna3(
a,
w2[e],
w2_z[e],
w2_s[e],
g_idx,
False,
)
ref[m_idx] += r2.squeeze() * w
# E2E chains w1 + activation + w2 + topk_w + output_topk reduce.
# Each step accumulates rounding error (split-K atomics, topk_w
# multiply order). Use relative L2 norm like the dense kernel test.
diff_l2 = torch.norm(fused.float() - ref.float())
ref_l2 = torch.norm(ref.float())
rel_l2 = (diff_l2 / ref_l2).item() if ref_l2 > 0 else 0.0
threshold = 0.05 if dtype == torch.float16 else 0.10
assert rel_l2 < threshold, (
f"rel L2 = {rel_l2:.4f} (threshold {threshold}), "
f"max abs diff: {(fused - ref).abs().max().item()}"
)
@gfx1100_only
def test_expert_id_minus_one():
"""Kernel handles expert_id == -1 (expert parallelism) without crash."""
# Qwen3-30B-A3B dims (E capped for memory)
E, K, N = 16, 2048, 768
groups = K // 32
w = _make_packed_weights(E, K, N)
ws = _make_scales(E, groups, N, torch.bfloat16)
wz = _make_qzeros(E, groups, N)
x = torch.randn(1, K, dtype=torch.bfloat16, device=device)
# Manually create sorted_token_ids/expert_ids with -1
sorted_ids = torch.tensor([0], dtype=torch.int32, device=device)
expert_ids = torch.tensor([-1], dtype=torch.int32, device=device)
ntp = torch.tensor([1], dtype=torch.int32, device=device)
out = torch.zeros(1, N, dtype=torch.bfloat16, device=device)
ops.moe_gptq_gemm_rdna3(
x,
out,
w,
ws,
wz,
torch.empty(0, device=device),
sorted_ids,
expert_ids,
ntp,
1,
1,
False,
0,
)
current_platform.synchronize()
# Output should remain zero (expert skipped)
assert torch.equal(out, torch.zeros_like(out))