From c794754062d49a8fdb63ab3c5215b488b865030c Mon Sep 17 00:00:00 2001 From: sroberts-amd <286448083+stacyroberts@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:38:04 -0500 Subject: [PATCH] [ROCm]Remove special-case SiTU support model-specific gating (#50597) Signed-off-by: Stacy Roberts Signed-off-by: Stacy Roberts Co-authored-by: Stacy Roberts --- .../fused_moe/experts/rocm_aiter_moe.py | 1 + .../layers/fused_moe/oracle/mxfp4.py | 64 +++++-- .../layers/quantization/mxfp4.py | 178 +++++------------- 3 files changed, 97 insertions(+), 146 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 3b80b5b1e3..a433b18487 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -484,6 +484,7 @@ class AiterExperts(mk.FusedMoEExpertsModular): return activation in [ MoEActivation.SILU, MoEActivation.GELU, + MoEActivation.SITU, MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE, ] diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index edb54bcf40..d09587b6d5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -644,20 +644,7 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( activation: MoEActivation | None = None, ) -> tuple[int, int]: """Round up hidden_size and intermediate_size based on backend requirements.""" - if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SITU: - # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size - # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a - # no-op for K3's shapes) rather than the generic ROCm 256 round-up, - # which would inflate weights and OOM. - intermediate_size = round_up(intermediate_size, 128) - hidden_size = round_up(hidden_size, 128) - elif ( - backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SILU - ): - # AITER's A16W4 SiLU kernel handles native dimensions aligned to 128. - intermediate_size = round_up(intermediate_size, 128) - hidden_size = round_up(hidden_size, 128) - elif backend == Mxfp4MoeBackend.EMULATION: + if backend == Mxfp4MoeBackend.EMULATION: # Emulation has no kernel tile; it only needs OCP MX block alignment so the # per-block scale buffers (`dim // OCP_MX_BLOCK_SIZE`) aren't floor-truncated # by a non-block-aligned TP/DP shard (e.g. 2880 // 4 = 720). @@ -683,8 +670,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( intermediate_size = round_up(intermediate_size, 128) hidden_size = round_up(hidden_size, 128) elif current_platform.is_rocm(): - intermediate_size = round_up(intermediate_size, 256) - hidden_size = round_up(hidden_size, 256) + if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and ( + activation == MoEActivation.SITU or activation == MoEActivation.SILU + ): + # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size + # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a + # no-op for K3's shapes) rather than the generic ROCm 256 round-up, + # which would inflate weights and OOM. + intermediate_size = round_up(intermediate_size, 128) + hidden_size = round_up(hidden_size, 128) + else: + intermediate_size = round_up(intermediate_size, 256) + hidden_size = round_up(hidden_size, 256) elif backend == Mxfp4MoeBackend.CPU: # CPU AMX kernel uses BLOCK_N=32, align to 32 intermediate_size = round_up(intermediate_size, 32) @@ -1280,6 +1277,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( w13_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, _cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None, + activation: MoEActivation | None = None, ) -> tuple[ torch.Tensor, torch.Tensor, @@ -1479,13 +1477,41 @@ def convert_weight_to_mxfp4_moe_kernel_format( import os - from aiter.ops.shuffle import shuffle_scale as _shuf_s - from aiter.ops.shuffle import shuffle_weight as _shuf_w - # TODO: Remove this once AITER is fixed # Necessary for AITER side from crashing os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0" + if activation == MoEActivation.SITU: + from aiter.utility.fp4_utils import e8m0_shuffle + + from vllm._aiter_ops import rocm_aiter_ops + + fp4_dtype = torch.float4_e2m1fn_x2 + e8m0_dtype = torch.float8_e8m0fnu + # a8w4 uses gate/up-interleaved flydsl kernels; + # default a16w4 keeps the separated layout. + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled() + w13 = rocm_aiter_ops.shuffle_weight_a16w4( + w13_weight.data.view(fp4_dtype), 16, guinterleave + ) + w2 = rocm_aiter_ops.shuffle_weight_a16w4( + w2_weight.data.view(fp4_dtype), 16, False + ) + w13_scale_raw = w13_weight_scale.data.view(e8m0_dtype) + w2_scale_raw = w2_weight_scale.data.view(e8m0_dtype) + w13_scale = rocm_aiter_ops.shuffle_scale_a16w4( + w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), + num_experts, + guinterleave, + ) + w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1])) + w13.is_shuffled = True + w2.is_shuffled = True + return (w13, w2, w13_scale, w2_scale, w13_bias, w2_bias) + + from aiter.ops.shuffle import shuffle_scale as _shuf_s + from aiter.ops.shuffle import shuffle_weight as _shuf_w + w13_weight = torch.nn.Parameter( _shuf_w( w13_weight.data.view(torch.float4_e2m1fn_x2), diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index dff705b1ec..6506c1fce1 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import os - import torch from vllm.logger import init_logger @@ -22,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, - backend_to_kernel_cls, convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, convert_weight_to_mxfp4_moe_kernel_format, make_mxfp4_moe_kernel, @@ -472,51 +469,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): ) -def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool: - """Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel. - - K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic - MXFP4 backend selector does not cover; route it to AITER on gfx950. - """ - if not current_platform.is_rocm(): - return False - from vllm._aiter_ops import rocm_aiter_ops - from vllm.model_executor.layers.fused_moe.activation import MoEActivation - from vllm.platforms.rocm import on_gfx950 - - return ( - rocm_aiter_ops.is_fused_moe_enabled() - and on_gfx950() - and moe.activation == MoEActivation.SITU - and moe.activation_situ_linear_beta is not None - and rocm_aiter_ops.get_aiter_activation_type("situ") is not None - ) - - class Mxfp4MoEMethod(FusedMoEMethodBase): """MXFP4 MoE quantization method.""" def __init__(self, moe: FusedMoEConfig): super().__init__(moe) - self.weight_dtype = "mxfp4" - self.is_k3_situ_aiter = _use_k3_situ_aiter(moe) - self.experts_cls: type[mk.FusedMoEExperts] | None - if self.is_k3_situ_aiter: - self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16 - self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0] - logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.") - from vllm._aiter_ops import rocm_aiter_ops - if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled(): - # AITER keeps bf16 activations below this token count, which - # would not match the fp8 a8w4 kernels the interleaved SiTU - # path is tuned for. The a16w4 path never reads it. - # TODO: Remove once AITER takes this as a kernel argument. - os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0" - else: - self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend( - moe - ) + self.weight_dtype = "mxfp4" + self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe) self.max_capture_size = moe.max_capture_size @@ -714,62 +674,62 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): hidden_size = self.hidden_size sf_block_size = 32 - # Shape assertions - assert ( - w13.dim() == 3 - and w13.shape[0] == num_experts - and w13.shape[1] == intermediate_size * self.moe.w13_num_shards - and w13.shape[2] == hidden_size // 2 - ) - assert ( - w13_scale.dim() == 3 - and w13_scale.shape[0] == num_experts - and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards - and w13_scale.shape[2] == hidden_size // sf_block_size - ) - assert ( - w2.dim() == 3 - and w2.shape[0] == num_experts - and w2.shape[1] == hidden_size - and w2.shape[2] == intermediate_size // 2 - ) - assert ( - w2_scale.dim() == 3 - and w2_scale.shape[1] == hidden_size - and w2_scale.shape[2] == intermediate_size // sf_block_size - ) - if w13_bias is not None: + # Shape assertions — skipped for SITU since its kernel handles native + # (non-256-aligned) intermediate sizes without prior round-up. + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + if self.moe.activation != MoEActivation.SITU: assert ( - w13_bias.dim() == 2 - and w13_bias.shape[0] == num_experts - and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards + w13.dim() == 3 + and w13.shape[0] == num_experts + and w13.shape[1] == intermediate_size * self.moe.w13_num_shards + and w13.shape[2] == hidden_size // 2 ) - if w2_bias is not None: assert ( - w2_bias.dim() == 2 - and w2_bias.shape[0] == num_experts - and w2_bias.shape[1] == hidden_size + w13_scale.dim() == 3 + and w13_scale.shape[0] == num_experts + and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards + and w13_scale.shape[2] == hidden_size // sf_block_size ) + assert ( + w2.dim() == 3 + and w2.shape[0] == num_experts + and w2.shape[1] == hidden_size + and w2.shape[2] == intermediate_size // 2 + ) + assert ( + w2_scale.dim() == 3 + and w2_scale.shape[1] == hidden_size + and w2_scale.shape[2] == intermediate_size // sf_block_size + ) + if w13_bias is not None: + assert ( + w13_bias.dim() == 2 + and w13_bias.shape[0] == num_experts + and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards + ) + if w2_bias is not None: + assert ( + w2_bias.dim() == 2 + and w2_bias.shape[0] == num_experts + and w2_bias.shape[1] == hidden_size + ) # Convert weights to kernel format - if self.is_k3_situ_aiter: - w13, w2, w13_scale, w2_scale = ( - self._convert_k3_situ_weight_to_kernel_format(layer) - ) - else: - w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( - convert_weight_to_mxfp4_moe_kernel_format( - mxfp4_backend=self.mxfp4_backend, - layer=layer, - w13_weight=w13, - w2_weight=w2, - w13_weight_scale=w13_scale, - w2_weight_scale=w2_scale, - w13_bias=w13_bias, - w2_bias=w2_bias, - _cache_permute_indices=self._cache_permute_indices, - ) + w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( + convert_weight_to_mxfp4_moe_kernel_format( + mxfp4_backend=self.mxfp4_backend, + layer=layer, + w13_weight=w13, + w2_weight=w2, + w13_weight_scale=w13_scale, + w2_weight_scale=w2_scale, + w13_bias=w13_bias, + w2_bias=w2_bias, + _cache_permute_indices=self._cache_permute_indices, + activation=self.moe.activation, ) + ) # For TRITON backends, weights are wrapped tensors from triton_kernels # that don't support .detach(). Manually assign parameters. @@ -810,42 +770,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): routing_tables=layer._expert_routing_tables(), ) - def _convert_k3_situ_weight_to_kernel_format( - self, layer: RoutedExperts - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - # K3's AITER A16W4 kernel wants the separated ([gate_all, up_all]) - # stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in - # convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here. - from aiter.utility.fp4_utils import e8m0_shuffle - - from vllm._aiter_ops import rocm_aiter_ops - - fp4_dtype = torch.float4_e2m1fn_x2 - e8m0_dtype = torch.float8_e8m0fnu - num_experts = layer.w13_weight.shape[0] - - # a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up- - # interleaved (_gui_) fp8 flydsl kernels, which need w13 weight+scale - # in interleave layout. Default a16w4 keeps the separated layout. - guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled() - w13 = rocm_aiter_ops.shuffle_weight_a16w4( - layer.w13_weight.data.view(fp4_dtype), 16, guinterleave - ) - w2 = rocm_aiter_ops.shuffle_weight_a16w4( - layer.w2_weight.data.view(fp4_dtype), 16, False - ) - w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype) - w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype) - w13_scale = rocm_aiter_ops.shuffle_scale_a16w4( - w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave - ) - w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1])) - - w13.is_shuffled = True - w2.is_shuffled = True - - return w13, w2, w13_scale, w2_scale - def process_weights_after_loading(self, layer): w13 = layer.w13_weight w2 = layer.w2_weight