[ROCm]Remove special-case SiTU support model-specific gating (#50597)

Signed-off-by: Stacy Roberts <sroberts@amd.com>
Signed-off-by: Stacy Roberts <robers23@utexas.edu>
Co-authored-by: Stacy Roberts <robers23@utexas.edu>
This commit is contained in:
sroberts-amd
2026-08-14 13:38:04 -05:00
committed by GitHub
co-authored by Stacy Roberts
parent f473870ecf
commit c794754062
3 changed files with 97 additions and 146 deletions
@@ -484,6 +484,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.SITU,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
]
@@ -644,20 +644,7 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
activation: MoEActivation | None = None,
) -> tuple[int, int]:
"""Round up hidden_size and intermediate_size based on backend requirements."""
if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SITU:
# K3's AITER A16W4 SiTU kernel handles K3's native intermediate size
# (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a
# no-op for K3's shapes) rather than the generic ROCm 256 round-up,
# which would inflate weights and OOM.
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif (
backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SILU
):
# AITER's A16W4 SiLU kernel handles native dimensions aligned to 128.
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif backend == Mxfp4MoeBackend.EMULATION:
if backend == Mxfp4MoeBackend.EMULATION:
# Emulation has no kernel tile; it only needs OCP MX block alignment so the
# per-block scale buffers (`dim // OCP_MX_BLOCK_SIZE`) aren't floor-truncated
# by a non-block-aligned TP/DP shard (e.g. 2880 // 4 = 720).
@@ -683,8 +670,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif current_platform.is_rocm():
intermediate_size = round_up(intermediate_size, 256)
hidden_size = round_up(hidden_size, 256)
if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and (
activation == MoEActivation.SITU or activation == MoEActivation.SILU
):
# K3's AITER A16W4 SiTU kernel handles K3's native intermediate size
# (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a
# no-op for K3's shapes) rather than the generic ROCm 256 round-up,
# which would inflate weights and OOM.
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
else:
intermediate_size = round_up(intermediate_size, 256)
hidden_size = round_up(hidden_size, 256)
elif backend == Mxfp4MoeBackend.CPU:
# CPU AMX kernel uses BLOCK_N=32, align to 32
intermediate_size = round_up(intermediate_size, 32)
@@ -1280,6 +1277,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
activation: MoEActivation | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -1479,13 +1477,41 @@ def convert_weight_to_mxfp4_moe_kernel_format(
import os
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w
# TODO: Remove this once AITER is fixed
# Necessary for AITER side from crashing
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
if activation == MoEActivation.SITU:
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# a8w4 uses gate/up-interleaved flydsl kernels;
# default a16w4 keeps the separated layout.
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
w2_weight.data.view(fp4_dtype), 16, False
)
w13_scale_raw = w13_weight_scale.data.view(e8m0_dtype)
w2_scale_raw = w2_weight_scale.data.view(e8m0_dtype)
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]),
num_experts,
guinterleave,
)
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
w13.is_shuffled = True
w2.is_shuffled = True
return (w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w
w13_weight = torch.nn.Parameter(
_shuf_w(
w13_weight.data.view(torch.float4_e2m1fn_x2),
+51 -127
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import torch
from vllm.logger import init_logger
@@ -22,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -472,51 +469,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Whether Kimi-K3's SiTU MXFP4 MoE should use the AITER A16W4 kernel.
K3 is weight-only MXFP4 (W4A16) with SiTU activation, which the generic
MXFP4 backend selector does not cover; route it to AITER on gfx950.
"""
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled():
# AITER keeps bf16 activations below this token count, which
# would not match the fp8 a8w4 kernels the interleaved SiTU
# path is tuned for. The a16w4 path never reads it.
# TODO: Remove once AITER takes this as a kernel argument.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.max_capture_size = moe.max_capture_size
@@ -714,62 +674,62 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
hidden_size = self.hidden_size
sf_block_size = 32
# Shape assertions
assert (
w13.dim() == 3
and w13.shape[0] == num_experts
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13.shape[2] == hidden_size // 2
)
assert (
w13_scale.dim() == 3
and w13_scale.shape[0] == num_experts
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13_scale.shape[2] == hidden_size // sf_block_size
)
assert (
w2.dim() == 3
and w2.shape[0] == num_experts
and w2.shape[1] == hidden_size
and w2.shape[2] == intermediate_size // 2
)
assert (
w2_scale.dim() == 3
and w2_scale.shape[1] == hidden_size
and w2_scale.shape[2] == intermediate_size // sf_block_size
)
if w13_bias is not None:
# Shape assertions — skipped for SITU since its kernel handles native
# (non-256-aligned) intermediate sizes without prior round-up.
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
if self.moe.activation != MoEActivation.SITU:
assert (
w13_bias.dim() == 2
and w13_bias.shape[0] == num_experts
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
w13.dim() == 3
and w13.shape[0] == num_experts
and w13.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13.shape[2] == hidden_size // 2
)
if w2_bias is not None:
assert (
w2_bias.dim() == 2
and w2_bias.shape[0] == num_experts
and w2_bias.shape[1] == hidden_size
w13_scale.dim() == 3
and w13_scale.shape[0] == num_experts
and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards
and w13_scale.shape[2] == hidden_size // sf_block_size
)
assert (
w2.dim() == 3
and w2.shape[0] == num_experts
and w2.shape[1] == hidden_size
and w2.shape[2] == intermediate_size // 2
)
assert (
w2_scale.dim() == 3
and w2_scale.shape[1] == hidden_size
and w2_scale.shape[2] == intermediate_size // sf_block_size
)
if w13_bias is not None:
assert (
w13_bias.dim() == 2
and w13_bias.shape[0] == num_experts
and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards
)
if w2_bias is not None:
assert (
w2_bias.dim() == 2
and w2_bias.shape[0] == num_experts
and w2_bias.shape[1] == hidden_size
)
# Convert weights to kernel format
if self.is_k3_situ_aiter:
w13, w2, w13_scale, w2_scale = (
self._convert_k3_situ_weight_to_kernel_format(layer)
)
else:
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
w13_weight_scale=w13_scale,
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
)
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
w13_weight_scale=w13_scale,
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
activation=self.moe.activation,
)
)
# For TRITON backends, weights are wrapped tensors from triton_kernels
# that don't support .detach(). Manually assign parameters.
@@ -810,42 +770,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
routing_tables=layer._expert_routing_tables(),
)
def _convert_k3_situ_weight_to_kernel_format(
self, layer: RoutedExperts
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
num_experts = layer.w13_weight.shape[0]
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
# interleaved (_gui_) fp8 flydsl kernels, which need w13 weight+scale
# in interleave layout. Default a16w4 keeps the separated layout.
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
)
w2 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w2_weight.data.view(fp4_dtype), 16, False
)
w13_scale_raw = layer.w13_weight_scale.data.view(e8m0_dtype)
w2_scale_raw = layer.w2_weight_scale.data.view(e8m0_dtype)
w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
)
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
w13.is_shuffled = True
w2.is_shuffled = True
return w13, w2, w13_scale, w2_scale
def process_weights_after_loading(self, layer):
w13 = layer.w13_weight
w2 = layer.w2_weight