[ROCm] [Performance] Optimize aiter moe for DeepSeekV4 (#46122)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-06-26 06:43:27 -07:00
committed by GitHub
parent 8e394244a5
commit 8921c4be88
@@ -1429,38 +1429,51 @@ def convert_weight_to_mxfp4_moe_kernel_format(
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
# Initially introduced for DeepSeekV4
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
e, n, k = w13_weight.shape
import os
# No de-interleave: standard _load_w13 already produces
# [gate_all, up_all] layout. Use aiter-native shuffle functions
# (matching aiter/ops/flydsl/test_flydsl_moe_a4w4.py pattern).
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w
from aiter.utility.fp4_utils import e8m0_shuffle as _e8m0_shuf
# w13 (gate+up, stage1): shuffle_weight with layout (16,16)
# TODO: Remove this once AITER is fixed
# Necessary for AITER side from crashing
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
w13_weight = torch.nn.Parameter(
_shuf_w(w13_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)),
_shuf_w(
w13_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
gate_up=True,
),
requires_grad=False,
)
shuffled_w13_scale = _e8m0_shuf(
w13_weight_scale.view(-1, w13_weight_scale.shape[-1])
shuffled_w13_scale = _shuf_s(
w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]),
num_experts,
True,
True,
)
# w2 (down-proj, stage2): same shuffle as w13 for a4w4 fp4x2
# (tuning script uses shuffle_weight((16,16)) + e8m0_shuffle for both)
w2_weight = torch.nn.Parameter(
_shuf_w(w2_weight.data.view(torch.float4_e2m1fn_x2), (16, 16)),
_shuf_w(
w2_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
gate_up=False,
),
requires_grad=False,
)
shuffled_w2_scale = _e8m0_shuf(
w2_weight_scale.view(-1, w2_weight_scale.shape[-1])
# use_gu_interleave
shuffled_w2_scale = _shuf_s(
w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]),
num_experts,
True,
False,
)
return (