[ROCm][DI] Enable MoRI FP4 dispatch for mxfp4 w4a4 moe (#59596)

Signed-off-by: lcskrishna <lollachaitanya@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Chaitanya Sri Krishna Lolla
2026-10-14 23:20:14 -05:00
committed by GitHub
co-authored by Cursor
parent 10d2c6083d
commit 833665cded
4 changed files with 103 additions and 23 deletions
+63 -16
View File
@@ -40,11 +40,36 @@ MNKS = [
]
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
def _mxfp4_aiter_weights(w1, w2, start, end):
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_quant_config,
)
def quantize(w):
w = w[start:end]
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2
)
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterFp8ChannelExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
rocm_aiter_fused_experts,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
@@ -53,6 +78,8 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
rocm_aiter_ops.refresh_env_variables()
init_workspace_manager(pgi.device)
set_random_seed(47)
fp8 = scheme == "fp8_token_channel"
mxfp4 = scheme == "mxfp4"
dtype = current_platform.fp8_dtype() if fp8 else None
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
@@ -86,29 +113,44 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
moe_parallel_config=parallel,
)
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
if mxfp4:
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
else:
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
prepare = maybe_make_prepare_finalize(config, quant)
assert isinstance(prepare, MoriPrepareAndFinalize)
assert prepare.use_fp8_dispatch == fp8
assert prepare.use_fp4_dispatch == mxfp4
assert prepare.num_dispatchers() == 2
experts_cls = AiterFp8ChannelExperts if fp8 else AiterUnquantizedExperts
experts_cls = {
"bf16": AiterUnquantizedExperts,
"fp8_token_channel": AiterFp8ChannelExperts,
"mxfp4": AiterMxfp4Experts,
}[scheme]
experts = experts_cls(config, quant)
assert not experts.expects_unquantized_inputs
kernel = FusedMoEKernel(prepare, experts)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
expert_mask[start:end] = 1
def reference(route_weights):
if mxfp4:
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
# as FP4 dispatch, so the BF16 tolerances below still apply.
return rocm_aiter_fused_experts(
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
)
return torch_experts(
x,
w1,
@@ -181,9 +223,14 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
)
@pytest.mark.parametrize("m,n,k", MNKS)
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
@multi_gpu_test(num_gpus=2)
def test_mori_moe(m, n, k, fp8):
def test_mori_moe(m, n, k, scheme):
if scheme == "mxfp4":
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
config = VllmConfig(
parallel_config=ParallelConfig(
data_parallel_size=2,
@@ -197,4 +244,4 @@ def test_mori_moe(m, n, k, fp8):
"VLLM_ROCM_USE_AITER_MOE": "1",
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
}
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)