From 8921c4be88effbd295dd7c2410fd21411256f819 Mon Sep 17 00:00:00 2001 From: TJian Date: Fri, 26 Jun 2026 21:43:27 +0800 Subject: [PATCH] [ROCm] [Performance] Optimize aiter moe for DeepSeekV4 (#46122) Signed-off-by: tjtanaa --- .../layers/fused_moe/oracle/mxfp4.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index e6e9d17925..56a7a6482d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1429,38 +1429,51 @@ def convert_weight_to_mxfp4_moe_kernel_format( ) elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16: - from vllm._aiter_ops import rocm_aiter_ops # noqa: F401 + # Initially introduced for DeepSeekV4 if w13_bias is not None: w13_bias = w13_bias.data.to(torch.float32) if w2_bias is not None: w2_bias = w2_bias.data.to(torch.float32) - e, n, k = w13_weight.shape + import os - # No de-interleave: standard _load_w13 already produces - # [gate_all, up_all] layout. Use aiter-native shuffle functions - # (matching aiter/ops/flydsl/test_flydsl_moe_a4w4.py pattern). + from aiter.ops.shuffle import shuffle_scale as _shuf_s from aiter.ops.shuffle import shuffle_weight as _shuf_w - from aiter.utility.fp4_utils import e8m0_shuffle as _e8m0_shuf - # w13 (gate+up, stage1): shuffle_weight with layout (16,16) + # TODO: Remove this once AITER is fixed + # Necessary for AITER side from crashing + os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0" + w13_weight = torch.nn.Parameter( - _shuf_w(w13_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)), + _shuf_w( + w13_weight.data.view(torch.float4_e2m1fn_x2), + is_guinterleave=True, + gate_up=True, + ), requires_grad=False, ) - shuffled_w13_scale = _e8m0_shuf( - w13_weight_scale.view(-1, w13_weight_scale.shape[-1]) + shuffled_w13_scale = _shuf_s( + w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]), + num_experts, + True, + True, ) - # w2 (down-proj, stage2): same shuffle as w13 for a4w4 fp4x2 - # (tuning script uses shuffle_weight((16,16)) + e8m0_shuffle for both) w2_weight = torch.nn.Parameter( - _shuf_w(w2_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)), + _shuf_w( + w2_weight.data.view(torch.float4_e2m1fn_x2), + is_guinterleave=True, + gate_up=False, + ), requires_grad=False, ) - shuffled_w2_scale = _e8m0_shuf( - w2_weight_scale.view(-1, w2_weight_scale.shape[-1]) + # use_gu_interleave + shuffled_w2_scale = _shuf_s( + w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]), + num_experts, + True, + False, ) return (