[ROCm][DI] Enable MoRI FP4 dispatch for mxfp4 w4a4 moe (#59596)
Signed-off-by: lcskrishna <lollachaitanya@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
10d2c6083d
commit
833665cded
@@ -40,11 +40,36 @@ MNKS = [
|
||||
]
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
def _mxfp4_aiter_weights(w1, w2, start, end):
|
||||
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_quant_config,
|
||||
)
|
||||
|
||||
def quantize(w):
|
||||
w = w[start:end]
|
||||
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
|
||||
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
|
||||
|
||||
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
|
||||
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
|
||||
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
backend, None, q1, q2, s1, s2
|
||||
)
|
||||
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterFp8ChannelExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
@@ -53,6 +78,8 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
init_workspace_manager(pgi.device)
|
||||
set_random_seed(47)
|
||||
fp8 = scheme == "fp8_token_channel"
|
||||
mxfp4 = scheme == "mxfp4"
|
||||
dtype = current_platform.fp8_dtype() if fp8 else None
|
||||
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
|
||||
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
|
||||
@@ -86,29 +113,44 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
moe_parallel_config=parallel,
|
||||
)
|
||||
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
if mxfp4:
|
||||
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
|
||||
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
|
||||
else:
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
prepare = maybe_make_prepare_finalize(config, quant)
|
||||
assert isinstance(prepare, MoriPrepareAndFinalize)
|
||||
assert prepare.use_fp8_dispatch == fp8
|
||||
assert prepare.use_fp4_dispatch == mxfp4
|
||||
assert prepare.num_dispatchers() == 2
|
||||
experts_cls = AiterFp8ChannelExperts if fp8 else AiterUnquantizedExperts
|
||||
experts_cls = {
|
||||
"bf16": AiterUnquantizedExperts,
|
||||
"fp8_token_channel": AiterFp8ChannelExperts,
|
||||
"mxfp4": AiterMxfp4Experts,
|
||||
}[scheme]
|
||||
experts = experts_cls(config, quant)
|
||||
assert not experts.expects_unquantized_inputs
|
||||
kernel = FusedMoEKernel(prepare, experts)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
|
||||
expert_mask[start:end] = 1
|
||||
|
||||
def reference(route_weights):
|
||||
if mxfp4:
|
||||
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
|
||||
# as FP4 dispatch, so the BF16 tolerances below still apply.
|
||||
return rocm_aiter_fused_experts(
|
||||
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
|
||||
)
|
||||
return torch_experts(
|
||||
x,
|
||||
w1,
|
||||
@@ -181,9 +223,14 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
|
||||
)
|
||||
@pytest.mark.parametrize("m,n,k", MNKS)
|
||||
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
|
||||
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_mori_moe(m, n, k, fp8):
|
||||
def test_mori_moe(m, n, k, scheme):
|
||||
if scheme == "mxfp4":
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
data_parallel_size=2,
|
||||
@@ -197,4 +244,4 @@ def test_mori_moe(m, n, k, fp8):
|
||||
"VLLM_ROCM_USE_AITER_MOE": "1",
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
|
||||
}
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)
|
||||
|
||||
Reference in New Issue
Block a user