From 1044d19998df96408832b33ccc7120716c615969 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:57:27 -0400 Subject: [PATCH] [ROCm][DSV4] Preserve native MXFP4 TP8 shard allocation (#51473) Signed-off-by: Fangzhou Ai --- tests/kernels/moe/test_ocp_mx_moe.py | 60 ++++++++++++++++++- .../layers/fused_moe/oracle/mxfp4.py | 6 ++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 4d96011251..9e74ec2fd3 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -2,13 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import types -from dataclasses import dataclass +from dataclasses import dataclass, replace import pytest import torch from tests.kernels.moe.utils import check_accuracy -from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops +from vllm._aiter_ops import ( + is_aiter_found, + is_aiter_found_and_supported, + rocm_aiter_ops, +) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer @@ -1598,8 +1602,58 @@ def test_rocm_mxfp4_moe_oracle( # ----------------------------------------------------------------------------- -# MXFP4 emulation size-rounding tests +# MXFP4 size-rounding tests # ----------------------------------------------------------------------------- + + +@pytest.mark.skipif(not ROCM_AVAILABLE, reason="ROCm-specific test") +@pytest.mark.skipif( + not is_aiter_found_and_supported(), reason="AITER is not installed or supported" +) +def test_aiter_mxfp4_bf16_silu_preserves_native_tp_shard(): + """Keep 128-aligned SiLU shards at their native widths.""" + from vllm.model_executor.layers.fused_moe import FusedMoEConfig + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + RoutingMethodType, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + ) + from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod + + moe_parallel_config = replace( + FusedMoEParallelConfig.make_no_parallel(), + tp_size=8, + ) + moe_config = FusedMoEConfig( + num_experts=384, + experts_per_token=6, + hidden_dim=7168, + intermediate_size=3072, + num_local_experts=384, + num_logical_experts=384, + activation=MoEActivation.SILU, + device="cpu", + routing_method=RoutingMethodType.Renormalize, + moe_parallel_config=moe_parallel_config, + in_dtype=torch.bfloat16, + ) + method = object.__new__(Mxfp4MoEMethod) + method.moe = moe_config + method.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16 + + rounded_shape = method.maybe_roundup_sizes( + hidden_size=7168, + intermediate_size_per_partition=moe_config.intermediate_size_per_partition, + act_dtype=torch.bfloat16, + moe_parallel_config=moe_config.moe_parallel_config, + ) + + assert rounded_shape == (7168, 384) + + # Emulation needs each per-partition dim rounded up to OCP_MX_BLOCK_SIZE (32); # a non-block-aligned shard (e.g. GPT-OSS 2880 // 4 = 720) otherwise truncates # the scale buffer and fails weight loading. diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 3722528c21..edb54bcf40 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -651,6 +651,12 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( # which would inflate weights and OOM. intermediate_size = round_up(intermediate_size, 128) hidden_size = round_up(hidden_size, 128) + elif ( + backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SILU + ): + # AITER's A16W4 SiLU kernel handles native dimensions aligned to 128. + intermediate_size = round_up(intermediate_size, 128) + hidden_size = round_up(hidden_size, 128) elif backend == Mxfp4MoeBackend.EMULATION: # Emulation has no kernel tile; it only needs OCP MX block alignment so the # per-block scale buffers (`dim // OCP_MX_BLOCK_SIZE`) aren't floor-truncated