[ROCm][CI] Extended Fused MoE and FP8 MoE test support (#41100)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-19 10:19:42 -05:00
committed by GitHub
parent be06873198
commit 160f7f0840
9 changed files with 331 additions and 31 deletions
@@ -8,6 +8,7 @@ from __future__ import annotations
import pytest
import torch
import torch.nn.functional as F
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe imp
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import set_random_seed
from .utils import make_dummy_moe_config, shuffle_weight
@@ -104,13 +106,30 @@ def make_weights(dtype, k, n, e):
w1_tri = shuffle_weight(w1_tri)
w1_bias_tri = shuffle_weight(w1_bias_tri)
if current_platform.is_rocm():
k_align, n2_align = 256, 512
else:
k_align, n2_align = 64, 128
w1_bottom_pad = round_up(w1_tri.shape[1], k_align) - w1_tri.shape[1]
w1_right_pad = round_up(w1_tri.shape[2], n2_align) - w1_tri.shape[2]
w2_bottom_pad = w1_right_pad // 2
w2_right_pad = w1_bottom_pad
w1_tri = F.pad(w1_tri, (0, w1_right_pad, 0, w1_bottom_pad, 0, 0))
w2_tri = F.pad(w2_tri, (0, w2_right_pad, 0, w2_bottom_pad, 0, 0))
w1_bias_tri = F.pad(w1_bias_tri, (0, w1_right_pad, 0, 0))
w2_bias_tri = F.pad(w2_bias_tri, (0, w2_right_pad, 0, 0))
# quant triton_weights
w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1)
w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, dtype, axis=1)
w1 = w1[..., :k, : 2 * n]
w1 = unshuffle_weight(w1)
w2_tri, w2_scale_tri = downcast_to_mxfp(w2_tri, torch.uint8, axis=1)
w2 = upcast_from_mxfp(w2_tri, w2_scale_tri, dtype, axis=1)
w2 = w2[..., :n, :k]
num_warps = 8
w_layout, w_layout_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=1)
@@ -150,6 +169,7 @@ def make_weights(dtype, k, n, e):
w2_bias_tri,
w1_precision_config,
w2_precision_config,
w1_bottom_pad,
)
@@ -237,7 +257,7 @@ def oai_triton_moe_impl(
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
not current_platform.is_cuda_alike(), reason="Requires CUDA-alike platform."
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("m,n,k", MNK)
@@ -256,6 +276,7 @@ def test_oai_triton_moe(
):
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
set_random_seed(0)
(
w1,
w2,
@@ -267,9 +288,11 @@ def test_oai_triton_moe(
w2_bias_tri,
w1_precision_config,
w2_precision_config,
x_pad,
) = make_weights(dtype, k, n, num_experts)
x = torch.randn((m, k), dtype=dtype, device="cuda")
x_tri = F.pad(x, (0, x_pad, 0, 0))
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
@@ -278,7 +301,7 @@ def test_oai_triton_moe(
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
out = oai_triton_moe_impl(
x,
x_tri,
w1_tri,
w2_tri,
w1_precision_config,
@@ -290,6 +313,7 @@ def test_oai_triton_moe(
topk_ids,
unfused,
)
out = out[..., :k]
assert_close(ref=out_ref, tri=out, maxtol=0.025, rmstol=0.005)
@@ -321,6 +345,7 @@ def test_unfused_oai_triton_experts_apply_direct_deepseek_v4_topology(workspace_
w2_bias_tri,
w1_precision_config,
w2_precision_config,
_x_pad,
) = make_weights(dtype, k, n, num_experts)
x = torch.randn((m, k), dtype=dtype, device="cuda")
+41
View File
@@ -16,6 +16,7 @@ from torch.nn import Parameter
from torch.nn import functional as F
import vllm.model_executor.layers.fused_moe # noqa
import vllm.model_executor.layers.fused_moe.fused_moe as fused_moe_module
from tests.kernels.moe.utils import (
fused_moe,
make_dummy_moe_config,
@@ -68,6 +69,46 @@ from vllm.utils.torch_utils import set_random_seed
DEVICE_TYPE = current_platform.device_type
def test_triton_moe_launcher_passes_scalar_scale_as_pointer(monkeypatch) -> None:
captured: dict[str, torch.Tensor] = {}
class FakeKernel:
def __getitem__(self, grid):
def launch(*args, **kwargs) -> None:
captured["a_scale"] = args[4]
return launch
monkeypatch.setattr(fused_moe_module, "fused_moe_kernel", FakeKernel())
a_scale = torch.tensor(0.5)
fused_moe_module.invoke_fused_moe_triton_kernel(
A=torch.ones((1, 1)),
B=torch.ones((1, 1, 1)),
C=torch.empty((1, 1, 1)),
A_scale=a_scale,
B_scale=torch.ones(1),
topk_weights=torch.ones((1, 1)),
sorted_token_ids=None,
expert_ids=torch.zeros(1, dtype=torch.int32),
num_tokens_post_padded=torch.ones(1, dtype=torch.int32),
mul_routed_weight=True,
top_k=1,
config={"BLOCK_SIZE_M": 1, "BLOCK_SIZE_N": 1, "BLOCK_SIZE_K": 1},
compute_type=tl.float32,
use_fp8_w8a8=True,
use_int8_w8a8=False,
use_int8_w8a16=False,
use_int4_w4a16=False,
per_channel_quant=False,
)
captured_scale = captured["a_scale"]
assert a_scale.ndim == 0
assert captured_scale.shape == (1,)
assert captured_scale.data_ptr() == a_scale.data_ptr()
def iterative_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
+106 -26
View File
@@ -7,9 +7,11 @@ Run `pytest tests/kernels/test_moe_layer.py`.
import functools
import os
import tempfile
import traceback
import types
from collections.abc import Callable
from contextlib import suppress
from dataclasses import astuple, dataclass, fields
from itertools import product
from typing import get_args
@@ -71,6 +73,15 @@ from vllm.v1.worker.workspace import (
is_workspace_manager_initialized,
)
def on_gfx950() -> bool:
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950 as rocm_on_gfx950
return rocm_on_gfx950()
return False
fp8_dtype = torch.float8_e4m3fn # current_platform.fp8_dtype
SHAPE_COMBOS = [
@@ -114,6 +125,7 @@ if has_nixl_ep():
BACKENDS += ["nixl_ep"]
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
QUANT_METHODS = [
None,
@@ -484,12 +496,12 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
"leads to large differences.",
)
# Skip modelopt_fp4 if not on B100+ (compute capability 10.0+)
if (
config.quantization == "modelopt_fp4"
and not current_platform.has_device_capability(100)
# Skip modelopt_fp4 if not on B100+ (compute capability 10.0+) or gfx950.
if config.quantization == "modelopt_fp4" and not (
(current_platform.is_rocm() and on_gfx950())
or current_platform.has_device_capability(100)
):
return False, "modelopt_fp4 not supported on H100+ GPUs"
return False, "modelopt_fp4 requires native NVFP4 or emulation"
# Skip flashinfer_nvlink if not on H100+ (compute capability 10.0+)
if (
@@ -508,6 +520,17 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
f"{config.backend} does not support quantization={config.quantization}",
)
if config.backend in MORI_BACKENDS:
if os.environ.get("VLLM_TEST_ENABLE_MORI_MOE_LAYER") != "1":
return False, "mori MoE layer matrix is opt-in"
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_fused_moe_enabled():
return False, "mori requires AITER fused MoE"
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
return False, "mori does not support AITER shared expert fusion"
if config.backend == "deepep_low_latency":
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll import ( # noqa: E501
DeepEPLLPrepareAndFinalize,
@@ -1695,14 +1718,16 @@ def _parallel_worker(
cpu_group,
test_configs: list[MoETestConfig],
verbosity: int,
failure_report_path: str | None = None,
**kwargs,
) -> None:
set_random_seed(7)
is_logging_rank = pgi.rank == 0
total = 0
passed = 0
failed = 0
fail_ids = []
failure_details = []
dp_rank = vllm_config.parallel_config.data_parallel_rank
@@ -1717,9 +1742,11 @@ def _parallel_worker(
tp_rank = pgi.rank % test_config.tp_size
if verbosity > 0:
if verbosity > 0 and is_logging_rank:
print(f"subtest: {test_config.id()}", end="")
local_failed = False
local_error: str | None = None
try:
_run_one_config(
vllm_config,
@@ -1743,19 +1770,9 @@ def _parallel_worker(
use_gate=test_config.use_gate,
use_routed_input_transform=test_config.use_routed_input_transform,
)
if verbosity > 0:
print(" PASSED")
else:
print(".", end="")
passed = passed + 1
except Exception as ex:
fail_ids.append(test_config.id())
failed = failed + 1
if verbosity > 0:
traceback.print_exc()
print(f"\n{str(ex)}\nFAILED")
else:
print("F", end="")
except Exception:
local_failed = True
local_error = traceback.format_exc()
finally:
# DeepEP managers are not reliably reusable across many subtests in
# a single worker process. Tear them down after each DeepEP case so
@@ -1771,6 +1788,40 @@ def _parallel_worker(
total = total + 1
torch.distributed.barrier()
any_failed_tensor = torch.tensor(
[int(local_failed)], device=pgi.device, dtype=torch.int32
)
torch.distributed.all_reduce(
any_failed_tensor, op=torch.distributed.ReduceOp.MAX
)
any_failed = bool(any_failed_tensor.item())
if any_failed:
failed = failed + 1
gathered_errors = [None] * pgi.world_size
torch.distributed.all_gather_object(
gathered_errors, local_error, group=cpu_group
)
first_error = next(
(error for error in gathered_errors if error is not None),
"unknown distributed failure",
)
assert first_error is not None
failure_details.append(f"{test_config.id()}\n{first_error.rstrip()}")
if verbosity > 0 and is_logging_rank:
print(" FAILED")
print(first_error.rstrip())
elif is_logging_rank:
print("F", end="")
else:
passed = passed + 1
if verbosity > 0 and is_logging_rank:
print(" PASSED")
elif is_logging_rank:
print(".", end="")
skipped = total - (passed + failed)
fails = f"{failed} failed" if failed > 0 else ""
@@ -1780,17 +1831,24 @@ def _parallel_worker(
passes = f"{sep}{passed} passed" if passed > 0 else ""
report = (
f"============= {fails}{skips}{passes} of {total} total tests ============="
f"============= {fails}{skips}{passes} of {total} total subcases ============="
)
sep = "\n" if verbosity == 0 else ""
print(f"{sep}{report}")
if is_logging_rank:
sep = "\n" if verbosity == 0 else ""
print(f"{sep}{report}")
if failed > 0:
fail_ids_str = "\n".join(fail_ids)
raise RuntimeError(
f"\n============= Failed subtests =============\n{fail_ids_str}\n{report}"
failure_details_str = "\n\n".join(failure_details)
failure_report = (
f"\n============= Failed subcases =============\n"
f"{failure_details_str}\n{report}"
)
if is_logging_rank and failure_report_path is not None:
with open(failure_report_path, "w", encoding="utf-8") as report_file:
report_file.write(failure_report)
if is_logging_rank:
raise RuntimeError(failure_report)
# TODO: add cudagraphs/torch.compile tests
@@ -1836,6 +1894,17 @@ def test_moe_layer(
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
if (
backend in MORI_BACKENDS
and os.environ.get("VLLM_TEST_ENABLE_MORI_MOE_LAYER") == "1"
):
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "0")
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
# TODO: cover FlashInfer MoE backends via moe_backend, e.g.
# moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
# (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
@@ -1881,6 +1950,11 @@ def test_moe_layer(
if len(test_configs) == 0:
pytest.skip("No supported configs found for this testpoint.")
with tempfile.NamedTemporaryFile(
prefix="moe-layer-failures-", delete=False
) as failure_report_file:
failure_report_path = failure_report_file.name
try:
parallel_launch_with_config(
world_size,
@@ -1889,7 +1963,13 @@ def test_moe_layer(
None,
test_configs,
verbosity,
failure_report_path=failure_report_path,
)
if os.path.getsize(failure_report_path) > 0:
with open(failure_report_path, encoding="utf-8") as report_file:
pytest.fail(report_file.read())
finally:
with suppress(FileNotFoundError):
os.remove(failure_report_path)
torch.accelerator.synchronize() # TODO: Is this needed?
torch.accelerator.empty_cache()
+26 -1
View File
@@ -33,6 +33,10 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
ref_nvfp4_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
@@ -294,13 +298,34 @@ def moe_quantize_weights_2d(
assert not per_token_quant
w_amax = torch.abs(w).max().to(torch.float32)
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
w, w_s = ops.scaled_fp4_quant(w, w_gs)
if current_platform.is_rocm():
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
else:
w, w_s = ops.scaled_fp4_quant(w, w_gs)
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
return w, w_s, w_gs
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
assert fp4_values.shape[-1] % 2 == 0
abs_values = fp4_values.abs()
codes = torch.empty_like(abs_values, dtype=torch.uint8)
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
codes[abs_values == value] = code
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
return codes[..., 0::2] | (codes[..., 1::2] << 4)
def _scaled_fp4_quant_emulated(
w: torch.Tensor, w_gs: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
def moe_quantize_weights(
w: torch.Tensor,
w_s: torch.Tensor | None,