[ROCm][Perf] Fused MoE W4A16 HIP kernel for AMD RDNA3 (gfx1100) (#44075)
Signed-off-by: JartX <sagformas@epdcenter.es> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -0,0 +1,503 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compile-guard tests for the ROCm RDNA3 W4A16 kernels (dense + MoE).
|
||||
|
||||
Verifies that the gfx1100 compilation and dispatch guards are hermetic:
|
||||
- On gfx1100: all ops exist, dispatch selects RDNA3 kernels.
|
||||
- On CDNA (gfx942/gfx950) or other non-gfx1100: ops must NOT exist,
|
||||
dispatch must fall through to Triton/Marlin, and no RDNA3 code
|
||||
path is reachable.
|
||||
|
||||
The negative (non-gfx1100) tests verify at three layers:
|
||||
1. Compile-level: on non-gfx1100 hardware, the RDNA3 ops are absent
|
||||
from the compiled _rocm_C extension — real binary verification.
|
||||
2. Static source analysis: parses CMakeLists.txt and torch_bindings.cpp
|
||||
to verify that all RDNA3 .cu files and op registrations are inside
|
||||
gfx1100-only guards.
|
||||
3. Runtime mock: patches on_gfx1100() to False and verifies that the
|
||||
Python dispatch chain rejects the RDNA3 path.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_compile_guards.py`.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
import vllm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 compile-guard tests are ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not on_gfx1100(),
|
||||
reason="Requires gfx1100 hardware",
|
||||
)
|
||||
|
||||
not_gfx1100 = pytest.mark.skipif(
|
||||
on_gfx1100(),
|
||||
reason="This test verifies non-gfx1100 builds — skip on gfx1100",
|
||||
)
|
||||
|
||||
RDNA3_OPS = ["gptq_gemm_rdna3", "gptq_gemm_rdna3_wmma", "moe_gptq_gemm_rdna3"]
|
||||
RDNA3_CU_FILES = [
|
||||
"q_gemm_rdna3.cu",
|
||||
"q_gemm_rdna3_wmma.cu",
|
||||
"moe_q_gemm_rdna3.cu",
|
||||
]
|
||||
|
||||
|
||||
def _find_repo_root() -> Path | None:
|
||||
"""Walk up from this file to find the repo root (has CMakeLists.txt)."""
|
||||
for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
|
||||
if (parent / "CMakeLists.txt").exists() and (parent / "csrc").is_dir():
|
||||
return parent
|
||||
return None
|
||||
|
||||
|
||||
REPO_ROOT = _find_repo_root()
|
||||
|
||||
# Directory of the *installed* vllm python package. The .py guard checks read
|
||||
# from here so they verify the code that is actually imported at runtime — this
|
||||
# works even on CI images that ship the wheel instead of the python source tree
|
||||
# (where only csrc/ + CMakeLists.txt are checked out for building).
|
||||
VLLM_PKG_DIR: Path | None = (
|
||||
Path(vllm.__file__).parent if getattr(vllm, "__file__", None) else None
|
||||
)
|
||||
|
||||
needs_source = pytest.mark.skipif(
|
||||
REPO_ROOT is None,
|
||||
reason="C/CMake source tree not available (installed package only)",
|
||||
)
|
||||
|
||||
|
||||
def _read_source_or_skip(*relparts: str) -> str:
|
||||
"""Read a C/CMake source file from the repo tree, or skip if absent.
|
||||
|
||||
Used for csrc/ and CMakeLists.txt — these only exist in a source checkout,
|
||||
not in the installed wheel.
|
||||
"""
|
||||
assert REPO_ROOT is not None # callers are gated by @needs_source
|
||||
path = REPO_ROOT.joinpath(*relparts)
|
||||
if not path.exists():
|
||||
pytest.skip(f"{path} not present in this source tree")
|
||||
return path.read_text()
|
||||
|
||||
|
||||
def _read_pkg_source_or_skip(*relparts: str) -> str:
|
||||
"""Read a python source file from the installed vllm package.
|
||||
|
||||
Reflects the code actually loaded at runtime, so these guard checks run in
|
||||
CI against the wheel — no source checkout required. Only skips for an
|
||||
exotic install layout (namespace/zipimport) where __file__ is unavailable.
|
||||
"""
|
||||
if VLLM_PKG_DIR is None:
|
||||
pytest.skip("vllm package directory not resolvable (zip/namespace?)")
|
||||
assert VLLM_PKG_DIR is not None # narrow for mypy (skip above is NoReturn)
|
||||
path = VLLM_PKG_DIR.joinpath(*relparts)
|
||||
if not path.exists():
|
||||
pytest.skip(f"{path} not present in installed vllm package")
|
||||
return path.read_text()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part A: POSITIVE — on gfx1100, ops exist and dispatch works
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("op_name", RDNA3_OPS)
|
||||
def test_op_registered_on_gfx1100(op_name):
|
||||
"""On gfx1100, all RDNA3 ops must be registered in _rocm_C."""
|
||||
assert hasattr(torch.ops, "_rocm_C"), "_rocm_C module not loaded"
|
||||
assert hasattr(torch.ops._rocm_C, op_name), (
|
||||
f"_rocm_C.{op_name} not registered — "
|
||||
"check CMakeLists.txt VLLM_ROCM_HAS_GFX1100 "
|
||||
"and torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
def test_all_ops_present_or_all_absent():
|
||||
"""The 3 RDNA3 ops are behind the same #ifdef — all present or all absent.
|
||||
|
||||
Catches someone accidentally moving an op outside the guard.
|
||||
"""
|
||||
has_rocm_c = hasattr(torch.ops, "_rocm_C")
|
||||
if not has_rocm_c:
|
||||
pytest.skip("_rocm_C not loaded")
|
||||
|
||||
present = {op: hasattr(torch.ops._rocm_C, op) for op in RDNA3_OPS}
|
||||
values = set(present.values())
|
||||
assert len(values) == 1, (
|
||||
f"Guard inconsistency — some RDNA3 ops registered, others not: "
|
||||
f"{present}. Check torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100 block."
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part B: NEGATIVE — compile-level verification on non-gfx1100
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
@pytest.mark.parametrize("op_name", RDNA3_OPS)
|
||||
def test_op_absent_on_non_gfx1100(op_name):
|
||||
"""On non-gfx1100 (CDNA), RDNA3 ops must NOT exist in _rocm_C.
|
||||
|
||||
This is the real compile-level check: the binary was built without
|
||||
gfx1100 support, so the ops should not have been compiled or registered.
|
||||
"""
|
||||
if not hasattr(torch.ops, "_rocm_C"):
|
||||
return
|
||||
assert not hasattr(torch.ops._rocm_C, op_name), (
|
||||
f"_rocm_C.{op_name} is registered on non-gfx1100 hardware — "
|
||||
"compile guard is broken: check CMakeLists.txt "
|
||||
"VLLM_ROCM_HAS_GFX1100 and torch_bindings.cpp #ifdef"
|
||||
)
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
def test_rocm_moe_not_supported_on_non_gfx1100():
|
||||
"""rocm_moe.is_supported() must return False on non-gfx1100 hardware."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
rocm_moe,
|
||||
)
|
||||
|
||||
wq = type("WQ", (), {"num_bits": 4})()
|
||||
assert rocm_moe.is_supported(wq) is False, (
|
||||
"rocm_moe.is_supported() returned True on non-gfx1100 — "
|
||||
"dispatch guard is broken"
|
||||
)
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
def test_dense_kernel_rejects_on_non_gfx1100():
|
||||
"""RDNA3W4A16LinearKernel.can_implement must reject on non-gfx1100."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part C: Static source analysis (build-level guards)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@needs_source
|
||||
class TestCMakeGuards:
|
||||
"""Verify CMakeLists.txt only compiles RDNA3 .cu files for gfx1100."""
|
||||
|
||||
@staticmethod
|
||||
def _read_cmake():
|
||||
return _read_source_or_skip("CMakeLists.txt")
|
||||
|
||||
def test_rdna3_cu_files_inside_gfx1100_conditional(self):
|
||||
"""All RDNA3 .cu files must be listed inside the
|
||||
``if(VLLM_GPU_ARCHES MATCHES "gfx1100")`` block, not unconditionally.
|
||||
"""
|
||||
cmake = self._read_cmake()
|
||||
for cu_file in RDNA3_CU_FILES:
|
||||
assert cu_file in cmake, f"{cu_file} not found in CMakeLists.txt"
|
||||
|
||||
lines = cmake.splitlines()
|
||||
in_gfx1100_block = False
|
||||
for line in lines:
|
||||
if 'VLLM_GPU_ARCHES MATCHES "gfx1100"' in line:
|
||||
in_gfx1100_block = True
|
||||
if in_gfx1100_block and "endif()" in line:
|
||||
in_gfx1100_block = False
|
||||
if cu_file in line:
|
||||
assert in_gfx1100_block, (
|
||||
f"{cu_file} is listed OUTSIDE the gfx1100 "
|
||||
f"conditional in CMakeLists.txt — CDNA builds "
|
||||
f"would compile RDNA3 code. Line: {line.strip()}"
|
||||
)
|
||||
|
||||
def test_compile_definition_only_for_gfx1100(self):
|
||||
"""VLLM_ROCM_GFX1100 compile definition must be conditional."""
|
||||
cmake = self._read_cmake()
|
||||
lines = cmake.splitlines()
|
||||
in_gfx1100_block = False
|
||||
for line in lines:
|
||||
if "VLLM_ROCM_HAS_GFX1100)" in line:
|
||||
in_gfx1100_block = True
|
||||
if in_gfx1100_block and "endif()" in line:
|
||||
in_gfx1100_block = False
|
||||
if "VLLM_ROCM_GFX1100" in line and "target_compile_definitions" in line:
|
||||
assert in_gfx1100_block, (
|
||||
"VLLM_ROCM_GFX1100 compile definition is set outside "
|
||||
"the VLLM_ROCM_HAS_GFX1100 conditional — CDNA builds "
|
||||
f"would define it. Line: {line.strip()}"
|
||||
)
|
||||
|
||||
|
||||
@needs_source
|
||||
class TestTorchBindingsGuards:
|
||||
"""Verify torch_bindings.cpp gates all RDNA3 ops behind #ifdef."""
|
||||
|
||||
@staticmethod
|
||||
def _read_bindings():
|
||||
return _read_source_or_skip("csrc", "rocm", "torch_bindings.cpp")
|
||||
|
||||
def test_all_rdna3_ops_inside_ifdef(self):
|
||||
"""Every rdna3 op def/impl must be between #ifdef VLLM_ROCM_GFX1100
|
||||
and #endif. If any is outside, a CDNA build would try to register
|
||||
the op and link a symbol that doesn't exist.
|
||||
"""
|
||||
src = self._read_bindings()
|
||||
lines = src.splitlines()
|
||||
|
||||
inside_guard = False
|
||||
rdna3_lines_outside = []
|
||||
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "#ifdef VLLM_ROCM_GFX1100" in line:
|
||||
inside_guard = True
|
||||
elif line.strip() == "#endif" and inside_guard:
|
||||
inside_guard = False
|
||||
|
||||
if (
|
||||
"rdna3" in line.lower()
|
||||
and not line.strip().startswith("//")
|
||||
and not inside_guard
|
||||
):
|
||||
rdna3_lines_outside.append((i, line.strip()))
|
||||
|
||||
assert not rdna3_lines_outside, (
|
||||
"RDNA3 op references found OUTSIDE #ifdef VLLM_ROCM_GFX1100 "
|
||||
"in torch_bindings.cpp — these would break CDNA builds:\n"
|
||||
+ "\n".join(f" L{n}: {s}" for n, s in rdna3_lines_outside)
|
||||
)
|
||||
|
||||
def test_no_unconditional_rdna3_includes(self):
|
||||
"""No #include of RDNA3-specific headers outside the guard."""
|
||||
src = self._read_bindings()
|
||||
lines = src.splitlines()
|
||||
|
||||
inside_guard = False
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "#ifdef VLLM_ROCM_GFX1100" in line:
|
||||
inside_guard = True
|
||||
elif line.strip() == "#endif" and inside_guard:
|
||||
inside_guard = False
|
||||
|
||||
if "#include" in line and "rdna3" in line.lower():
|
||||
assert inside_guard, (
|
||||
f"L{i}: RDNA3 include outside gfx1100 guard: {line.strip()}"
|
||||
)
|
||||
|
||||
|
||||
class TestCustomOpsGuards:
|
||||
"""Verify _custom_ops.py gates register_fake behind hasattr checks."""
|
||||
|
||||
@staticmethod
|
||||
def _read_custom_ops():
|
||||
return _read_pkg_source_or_skip("_custom_ops.py")
|
||||
|
||||
def test_register_fake_guarded_by_hasattr(self):
|
||||
"""Every register_fake for an RDNA3 op must be preceded by a hasattr
|
||||
check — otherwise it would crash on import on CDNA where the ops
|
||||
don't exist.
|
||||
"""
|
||||
src = self._read_custom_ops()
|
||||
for op in RDNA3_OPS:
|
||||
pattern = rf'register_fake\(\s*"_rocm_C::{op}"\s*\)'
|
||||
match = re.search(pattern, src)
|
||||
if match is None:
|
||||
continue
|
||||
preceding = src[: match.start()]
|
||||
last_hasattr = preceding.rfind(f'hasattr(torch.ops._rocm_C, "{op}")')
|
||||
assert last_hasattr != -1, (
|
||||
f'register_fake("_rocm_C::{op}") is not preceded by a '
|
||||
f"hasattr check — would crash on CDNA import"
|
||||
)
|
||||
gap = preceding[last_hasattr:].count("\n")
|
||||
assert gap <= 5, (
|
||||
f"hasattr guard for {op} is {gap} lines before "
|
||||
f"register_fake — suspiciously far; verify it's the "
|
||||
f"actual guard and not a coincidence"
|
||||
)
|
||||
|
||||
def test_no_toplevel_rocm_c_import(self):
|
||||
"""No top-level ``from vllm._rocm_C import`` — would crash on CDNA."""
|
||||
src = self._read_custom_ops()
|
||||
for line in src.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("#") or stripped.startswith("//"):
|
||||
continue
|
||||
assert "from vllm._rocm_C import" not in stripped, (
|
||||
f"Top-level import of _rocm_C in _custom_ops.py would "
|
||||
f"crash on CDNA: {stripped}"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part D: Runtime mock (simulate CDNA on gfx1100 hardware)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class _FakeWeightQuant:
|
||||
"""Minimal stand-in for a weight quantization config."""
|
||||
|
||||
def __init__(self, num_bits):
|
||||
self.num_bits = num_bits
|
||||
|
||||
|
||||
class TestMoEDispatchMocked:
|
||||
"""Mock on_gfx1100() to False and verify RDNA3 MoE is unreachable."""
|
||||
|
||||
def test_is_supported_false_when_mocked_cdna(self):
|
||||
"""rocm_moe.is_supported() must return False when not on gfx1100."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
rocm_moe,
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.rocm.on_gfx1100", return_value=False):
|
||||
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
|
||||
|
||||
@pytest.mark.parametrize("num_bits", [2, 3, 8, 16])
|
||||
def test_is_supported_rejects_non_w4(self, num_bits):
|
||||
"""is_supported() rejects non-4-bit even before checking arch."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
rocm_moe,
|
||||
)
|
||||
|
||||
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=num_bits)) is False
|
||||
|
||||
def test_is_supported_false_when_op_missing(self):
|
||||
"""is_supported() returns False when the C++ op doesn't exist."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
rocm_moe,
|
||||
)
|
||||
|
||||
fake_rocm_c = type("FakeRocmC", (), {"gptq_gemm_rdna3": None})()
|
||||
with patch.object(torch, "ops", create=True) as mock_ops:
|
||||
mock_ops._rocm_C = fake_rocm_c
|
||||
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
|
||||
|
||||
def test_is_supported_false_when_rocm_c_absent(self):
|
||||
"""is_supported() returns False when _rocm_C doesn't exist at all."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
rocm_moe,
|
||||
)
|
||||
|
||||
fake_ops = type("FakeOps", (), {})()
|
||||
with patch.object(torch, "ops", fake_ops):
|
||||
assert rocm_moe.is_supported(_FakeWeightQuant(num_bits=4)) is False
|
||||
|
||||
|
||||
class TestDenseKernelSelectionMocked:
|
||||
"""Mock on_gfx1100() and verify dense RDNA3 kernel is not selected."""
|
||||
|
||||
@gfx1100_only
|
||||
def test_can_implement_rejects_when_mocked_cdna(self):
|
||||
"""RDNA3W4A16LinearKernel.can_implement must reject on mocked CDNA."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is True
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
patch("vllm.platforms.rocm._ON_GFX1100", False),
|
||||
):
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 kernel accepted on simulated CDNA: {reason}"
|
||||
|
||||
@gfx1100_only
|
||||
def test_chooser_skips_rdna3_when_mocked_cdna(self):
|
||||
"""choose_mp_linear_kernel must NOT return RDNA3 on mocked CDNA."""
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
patch("vllm.platforms.rocm._ON_GFX1100", False),
|
||||
):
|
||||
chosen = choose_mp_linear_kernel(config)
|
||||
assert chosen.__name__ != "RDNA3W4A16LinearKernel", (
|
||||
"RDNA3 kernel was selected on simulated CDNA — "
|
||||
"choose_mp_linear_kernel guard is broken"
|
||||
)
|
||||
|
||||
|
||||
class TestCompressedTensorsMoEDispatchGuard:
|
||||
"""Verify compressed_tensors_moe.py only enters rocm_moe under is_rocm()."""
|
||||
|
||||
def test_rocm_guard_in_dispatch_source(self):
|
||||
"""The rocm_moe import and call must be inside an is_rocm() check."""
|
||||
src = _read_pkg_source_or_skip(
|
||||
"model_executor",
|
||||
"layers",
|
||||
"quantization",
|
||||
"compressed_tensors",
|
||||
"compressed_tensors_moe",
|
||||
"compressed_tensors_moe.py",
|
||||
)
|
||||
lines = src.splitlines()
|
||||
|
||||
for i, line in enumerate(lines, 1):
|
||||
stripped = line.strip()
|
||||
if "rocm_moe" in stripped and not stripped.startswith("#"):
|
||||
found_guard = False
|
||||
for j in range(i - 1, max(0, i - 15), -1):
|
||||
if "is_rocm()" in lines[j - 1]:
|
||||
found_guard = True
|
||||
break
|
||||
assert found_guard, (
|
||||
f"L{i}: rocm_moe reference not protected by "
|
||||
f"is_rocm() guard: {stripped}"
|
||||
)
|
||||
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the ROCm RDNA3 fused MoE W4A16 HIP kernel (gfx1100).
|
||||
|
||||
Tests ``moe_gptq_gemm_rdna3`` against the dense ``gptq_gemm_rdna3`` as
|
||||
reference: builds RDNA3-format weights (shuffled int32, synthesized qzeros),
|
||||
runs the fused MoE kernel, and compares per-expert results.
|
||||
|
||||
Model parameters taken from:
|
||||
- cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit
|
||||
(hidden=2048, inter=768, E=128, top_k=8, G=32)
|
||||
- Qwen3.6-35B-A3B-GPTQ-W4A16-G32
|
||||
(hidden=2048, inter=512, E=256, top_k=8, G=32)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_moe_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 MoE W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm import _custom_ops as ops # noqa: E402
|
||||
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( # noqa: E402
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "moe_gptq_gemm_rdna3")
|
||||
),
|
||||
reason="Requires gfx1100 with moe_gptq_gemm_rdna3 op",
|
||||
)
|
||||
|
||||
# Model configurations: real K/N/top_k/group_size dims, E capped at 16 to
|
||||
# fit in test GPU memory (full E=128/256 would need >20GB for weights alone).
|
||||
# Kernel behavior is E-independent (per-expert tiling), so E=16 is sufficient.
|
||||
MODEL_CONFIGS = [
|
||||
# cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit dims (E capped)
|
||||
pytest.param(16, 2048, 768, 8, 32, id="Qwen3-30B-A3B"),
|
||||
# Qwen3.6-35B-A3B-GPTQ-W4A16-G32 dims (E capped)
|
||||
pytest.param(16, 2048, 512, 8, 32, id="Qwen3.6-35B-A3B"),
|
||||
]
|
||||
|
||||
# Token counts: decode (1), small batch (4), medium (16), prefill (64)
|
||||
NUM_TOKENS = [1, 4, 16, 64, 256, 512]
|
||||
|
||||
|
||||
def _make_packed_weights(E, K, N):
|
||||
"""Create random 4-bit packed weights [E, K/8, N] int32 + shuffle."""
|
||||
w = torch.randint(0, 16, (E, K, N), dtype=torch.int32, device=device)
|
||||
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
|
||||
for i in range(8):
|
||||
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
for e in range(E):
|
||||
we = packed[e].contiguous()
|
||||
ops.gptq_shuffle(we, g_idx, 4)
|
||||
packed[e] = we
|
||||
return packed
|
||||
|
||||
|
||||
def _make_scales(E, groups, N, dtype):
|
||||
return torch.rand(E, groups, N, dtype=dtype, device=device) * 0.1
|
||||
|
||||
|
||||
def _make_qzeros(E, groups, N):
|
||||
zeros = torch.full(
|
||||
(groups, N),
|
||||
scalar_types.uint4b8.bias - 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
qz = pack_quantized_values_into_int32(
|
||||
zeros,
|
||||
scalar_types.uint4b8,
|
||||
packed_dim=1,
|
||||
)
|
||||
return qz.unsqueeze(0).expand(E, -1, -1).contiguous()
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("block_size_m", [1, 4])
|
||||
def test_fused_moe_w1_matches_dense(
|
||||
E, K, N_inter, top_k, group_size, M, dtype, block_size_m
|
||||
):
|
||||
"""w1 GEMM via fused kernel matches per-expert dense kernel."""
|
||||
N_gate_up = N_inter * 2
|
||||
groups = K // group_size
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, groups, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, groups, N_gate_up)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
|
||||
|
||||
# Fused kernel
|
||||
fused_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
fused_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
block_size_m,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
|
||||
# Per-expert dense reference
|
||||
ref_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
for m in range(M):
|
||||
for k in range(top_k):
|
||||
e = topk_ids[m, k].item()
|
||||
flat = m * top_k + k
|
||||
ref = ops.gptq_gemm_rdna3(
|
||||
x[m : m + 1],
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref_out[flat] = ref.squeeze()
|
||||
|
||||
# Split-K atomics can cause minor fp16/bf16 rounding differences
|
||||
# at large K (e.g. K=2048 → 8 K-blocks). Use allclose, not equal.
|
||||
atol = 0.5 if dtype == torch.bfloat16 else 0.1
|
||||
assert torch.allclose(fused_out, ref_out, atol=atol, rtol=0.01), (
|
||||
f"max diff: {(fused_out - ref_out).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_fused_moe_output_topk_reduces(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
"""output_topk fuses moe_sum: multiple experts write to same output row."""
|
||||
groups = K // group_size
|
||||
|
||||
torch.manual_seed(123)
|
||||
x = torch.randn(M * top_k, K, dtype=dtype, device=device)
|
||||
w = _make_packed_weights(E, K, N_inter)
|
||||
ws = _make_scales(E, groups, N_inter, dtype)
|
||||
wz = _make_qzeros(E, groups, N_inter)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
torch.randn(M, top_k, device=device),
|
||||
dim=-1,
|
||||
).float()
|
||||
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
# Without output_topk: write to [M*top_k, N] then moe_sum
|
||||
flat_out = torch.zeros(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
flat_out,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
0,
|
||||
)
|
||||
ref = torch.zeros(M, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_sum(flat_out.view(M, top_k, N_inter), ref)
|
||||
|
||||
# With output_topk: write directly to [M, N]
|
||||
fused = torch.zeros(M, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
fused,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
atol = 1.0 if dtype == torch.bfloat16 else 0.1
|
||||
assert torch.allclose(fused, ref, atol=atol, rtol=0.01), (
|
||||
f"max diff: {(fused - ref).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
"""Full MoE forward: w1 + silu_and_mul + w2 with output_topk reduce."""
|
||||
N_gate_up = N_inter * 2
|
||||
hidden = K
|
||||
|
||||
torch.manual_seed(7)
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, K // group_size, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, K // group_size, N_gate_up)
|
||||
w2 = _make_packed_weights(E, N_inter, hidden)
|
||||
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
|
||||
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
torch.randn(M, top_k, device=device),
|
||||
dim=-1,
|
||||
).float()
|
||||
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
# Fused path (what apply() does)
|
||||
w1_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
w1_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
|
||||
fused = torch.zeros(M, hidden, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
act_out,
|
||||
fused,
|
||||
w2,
|
||||
w2_s,
|
||||
w2_z,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
# Per-expert reference
|
||||
ref = torch.zeros(M, hidden, dtype=dtype, device=device)
|
||||
for m_idx in range(M):
|
||||
for k_idx in range(top_k):
|
||||
e = topk_ids[m_idx, k_idx].item()
|
||||
w = topk_w[m_idx, k_idx].item()
|
||||
r1 = ops.gptq_gemm_rdna3(
|
||||
x[m_idx : m_idx + 1],
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
a = torch.empty(1, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, a, r1)
|
||||
r2 = ops.gptq_gemm_rdna3(
|
||||
a,
|
||||
w2[e],
|
||||
w2_z[e],
|
||||
w2_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref[m_idx] += r2.squeeze() * w
|
||||
|
||||
# E2E chains w1 + activation + w2 + topk_w + output_topk reduce.
|
||||
# Each step accumulates rounding error (split-K atomics, topk_w
|
||||
# multiply order). Use relative L2 norm like the dense kernel test.
|
||||
diff_l2 = torch.norm(fused.float() - ref.float())
|
||||
ref_l2 = torch.norm(ref.float())
|
||||
rel_l2 = (diff_l2 / ref_l2).item() if ref_l2 > 0 else 0.0
|
||||
threshold = 0.05 if dtype == torch.float16 else 0.10
|
||||
assert rel_l2 < threshold, (
|
||||
f"rel L2 = {rel_l2:.4f} (threshold {threshold}), "
|
||||
f"max abs diff: {(fused - ref).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
def test_expert_id_minus_one():
|
||||
"""Kernel handles expert_id == -1 (expert parallelism) without crash."""
|
||||
# Qwen3-30B-A3B dims (E capped for memory)
|
||||
E, K, N = 16, 2048, 768
|
||||
groups = K // 32
|
||||
|
||||
w = _make_packed_weights(E, K, N)
|
||||
ws = _make_scales(E, groups, N, torch.bfloat16)
|
||||
wz = _make_qzeros(E, groups, N)
|
||||
x = torch.randn(1, K, dtype=torch.bfloat16, device=device)
|
||||
|
||||
# Manually create sorted_token_ids/expert_ids with -1
|
||||
sorted_ids = torch.tensor([0], dtype=torch.int32, device=device)
|
||||
expert_ids = torch.tensor([-1], dtype=torch.int32, device=device)
|
||||
ntp = torch.tensor([1], dtype=torch.int32, device=device)
|
||||
|
||||
out = torch.zeros(1, N, dtype=torch.bfloat16, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
out,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
torch.empty(0, device=device),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
|
||||
# Output should remain zero (expert skipped)
|
||||
assert torch.equal(out, torch.zeros_like(out))
|
||||
Reference in New Issue
Block a user