[ROCm][MoE] Split AITER CK and Triton MXFP4 W4A16 into separate backends (#50622)

Signed-off-by: afriedri <afriedri@amd.com>
Signed-off-by: Andy Friedrich <afriedri@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
afriedri
2026-08-31 23:14:14 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent 446c769482
commit dc9114b201
6 changed files with 464 additions and 390 deletions
+25 -21
View File
@@ -134,10 +134,17 @@ def test_mxfp4_loading_and_execution_moe(vllm_runner, model_case: ModelCase):
assert output
def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = None):
# Note we add an extra bias of 1 to the linear layer
# Uses chunked layout: first half is gate, second half is up
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
def swiglu(
x,
alpha: float = 1.702,
beta: float = 1.0,
limit: float | None = None,
interleaved: bool = False,
):
if interleaved:
x_glu, x_linear = x[..., ::2], x[..., 1::2]
else:
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
if limit is not None:
x_glu = x_glu.clamp(max=limit)
x_linear = x_linear.clamp(min=-limit, max=limit)
@@ -145,16 +152,6 @@ def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = Non
return out_glu * (x_linear + beta)
def swigluoai(x, alpha: float = 1.702, limit: float = 7.0):
# OAI swiglu uses interleaved layout: gate/up alternating
# See SwigluOAIAndMul in vllm/model_executor/layers/activation.py
gate, up = x[..., ::2], x[..., 1::2]
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
glu = gate * torch.sigmoid(gate * alpha)
return (up + 1) * glu
fp4_lookup_table = [0, 0.5, 1, 1.5, 2, 3, 4, 6, -0, -0.5, -1, -1.5, -2, -3, -4, -6]
@@ -242,12 +239,9 @@ def reference_moe(
# Apply activation
if activation in ("swiglu", "silu"):
if use_interleaved_layout:
# SWIGLUOAI: interleaved gate/up layout
t = swigluoai(t, alpha=alpha, limit=limit)
else:
# Standard swiglu/silu: chunked layout
t = swiglu(t, alpha=alpha, beta=beta, limit=limit)
t = swiglu(
t, alpha=alpha, beta=beta, limit=limit, interleaved=use_interleaved_layout
)
elif activation == "relu2":
# RELU2_NO_MUL: relu(x)^2
t = torch.relu(t)
@@ -1403,6 +1397,14 @@ ROCM_BACKEND_CONFIGS = {
"requires_aiter": True,
"requires_gfx950": True,
},
"AITER_TRITON_MXFP4_BF16": {
"activation": "SILU",
"rtol": 0.3,
"percent": 0.95,
"requires_aiter": True,
"requires_gfx950": False,
"interleaved_layout": True,
},
"AITER_MXFP4_FP8": {
"activation": "SWIGLUOAI",
"rtol": 0.5,
@@ -1670,7 +1672,9 @@ def test_rocm_mxfp4_moe_oracle(
# Determine activation type and layout
# SWIGLUOAI uses interleaved layout (gate/up alternating)
# SILU uses chunked layout (first half gate, second half up)
use_interleaved = activation == MoEActivation.SWIGLUOAI
use_interleaved = bool(
config.get("interleaved_layout", activation == MoEActivation.SWIGLUOAI)
)
if activation in [MoEActivation.SWIGLUOAI, MoEActivation.SILU]:
act_name = "swiglu"
else:
@@ -31,6 +31,8 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
aiter_triton_kernel_w4a8_moe_forward,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
aiter_triton_kernel_w4a16_moe_forward,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
+3
View File
@@ -137,6 +137,7 @@ MoEBackend = Literal[
"humming",
"triton_unfused",
"aiter",
"aiter_triton_mxfp4_bf16",
"flydsl",
"hpc",
"emulation",
@@ -260,6 +261,8 @@ class KernelConfig:
- "humming": Use Humming Mixed Precision kernels
- "triton_unfused": Use Triton unfused MoE kernels
- "aiter": Use AMD AITer kernels (ROCm only)
- "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16
(moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250)
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
- "hpc": Use HPC kernels (FP8 and Hopper only)
- "emulation": use BF16/FP16 GEMM, dequantizing weights and
@@ -0,0 +1,403 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
patch_gating_output,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
__all__ = [
"AiterW4A16ExpertsMonolithic",
"aiter_triton_kernel_w4a16_moe_forward",
]
def _aiter_raw(t):
"""Unwrap the `triton_kernels.tensor.Tensor` that `_swizzle_mxfp4` returns."""
if t is None or isinstance(t, torch.Tensor):
return t
assert hasattr(t, "storage"), (
f"expected a triton_kernels wrapped tensor with a .storage attribute, "
f"got {type(t)}"
)
return t.storage.data
def _aiter_w4a16_silu_via_a8w4(
hidden_states: torch.Tensor,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
w1_bias,
w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input: bool,
swiglu_limit: float,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
) -> torch.Tensor:
"""
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
"""
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from aiter.ops.triton.quant import dynamic_mxfp8_quant
try:
from aiter.ops.triton.moe.moe_op_gemm_a8w4 import moe_gemm_a8w4
except ImportError:
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
g2_gammas = None if apply_router_weight_on_input else gammas
hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype)
raw_gate_up = moe_gemm_a8w4(
hidden_q,
w1_data,
a1_scale,
w1_wscale,
None,
None,
w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=g1_gammas,
swizzle_mx_scale=swz,
out_dtype=torch.bfloat16,
apply_swiglu=False,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
if unpadded_N_w1 is not None:
raw_gate_up = raw_gate_up[:, :unpadded_N_w1]
# convert interleaved -> chunked for kernel
gate, up = raw_gate_up[:, ::2], raw_gate_up[:, 1::2]
raw_gate_up = torch.cat((gate, up), dim=1).contiguous()
interim_fp8, a2_scale = fused_clamp_act_mul(
raw_gate_up,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=quant_dtype,
scale_dtype_fmt="ue8m0",
quant_block_size=32,
)
out = moe_gemm_a8w4(
interim_fp8,
w2_data,
a2_scale,
w2_wscale,
None,
None,
w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=g2_gammas,
swizzle_mx_scale=swz,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
def aiter_triton_kernel_w4a16_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
except ImportError:
from aiter.ops.triton.moe_op_gemm_a16w4 import moe_gemm_a16w4
try:
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
# See context in #50859.
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
# AITER released, and AITER pin in vLLM increased from 0.1.19.
gating_output = patch_gating_output(gating_output, global_num_experts)
if score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
assert quant_config.w1_precision is not None
assert quant_config.w2_precision is not None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale)
w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
# 7.0 is the SwigluOAI clamp, so only default to it for that activation;
# a plain-SILU model that configures no limit must not be clamped.
if quant_config.gemm1_clamp_limit is not None:
swiglu_limit = quant_config.gemm1_clamp_limit
elif activation == MoEActivation.SWIGLUOAI:
swiglu_limit = 7.0
else:
swiglu_limit = None
# SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults.
if activation == MoEActivation.SILU and on_gfx1250():
return _aiter_w4a16_silu_via_a8w4(
hidden_states,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
quant_config.w1_bias,
quant_config.w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input,
# fused_clamp_act_mul encodes "no clamp" as 0, not None.
swiglu_limit if swiglu_limit is not None else 0.0,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
)
# SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu.
swiglu_add_residual = activation != MoEActivation.SILU
intermediate = moe_gemm_a16w4(
hidden_states,
w1_data,
None,
w1_wscale,
None,
None,
quant_config.w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=None,
apply_swiglu=True,
alpha=swiglu_alpha,
limit=swiglu_limit,
swiglu_add_residual=swiglu_add_residual,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
out = moe_gemm_a16w4(
intermediate,
w2_data,
None,
w2_wscale,
None,
None,
quant_config.w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=None,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx942, on_gfx950, on_gfx1250
return on_gfx942() or on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, None)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
)
@@ -21,9 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
__all__ = [
"AiterW4A8ExpertsMonolithic",
"AiterW4A16ExpertsMonolithic",
"aiter_triton_kernel_w4a8_moe_forward",
"aiter_triton_kernel_w4a16_moe_forward",
]
@@ -371,363 +369,3 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
)
def _aiter_raw(t):
if t is None or isinstance(t, torch.Tensor):
return t
return t.storage.data if hasattr(t, "storage") else t
def _aiter_w4a16_silu_via_a8w4(
hidden_states: torch.Tensor,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
w1_bias,
w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input: bool,
swiglu_limit: float,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
) -> torch.Tensor:
"""
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
"""
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from aiter.ops.triton.quant import dynamic_mxfp8_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
g2_gammas = None if apply_router_weight_on_input else gammas
hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype)
raw_gate_up = moe_gemm_a8w4(
hidden_q,
w1_data,
a1_scale,
w1_wscale,
None,
None,
w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=g1_gammas,
swizzle_mx_scale=swz,
out_dtype=torch.bfloat16,
apply_swiglu=False,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
if unpadded_N_w1 is not None:
raw_gate_up = raw_gate_up[:, :unpadded_N_w1]
interim_fp8, a2_scale = fused_clamp_act_mul(
raw_gate_up,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=quant_dtype,
scale_dtype_fmt="ue8m0",
quant_block_size=32,
)
out = moe_gemm_a8w4(
interim_fp8,
w2_data,
a2_scale,
w2_wscale,
None,
None,
w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=g2_gammas,
swizzle_mx_scale=swz,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
def aiter_triton_kernel_w4a16_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
# See context in #50859.
# TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged,
# AITER released, and AITER pin in vLLM increased from 0.1.19.
gating_output = patch_gating_output(gating_output, global_num_experts)
if score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
assert quant_config.w1_precision is not None
assert quant_config.w2_precision is not None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale)
w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
swiglu_limit = (
quant_config.gemm1_clamp_limit
if quant_config.gemm1_clamp_limit is not None
else 7.0
)
# SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults.
if activation == MoEActivation.SILU and on_gfx1250():
return _aiter_w4a16_silu_via_a8w4(
hidden_states,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
quant_config.w1_bias,
quant_config.w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input,
swiglu_limit,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
)
# SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu.
swiglu_add_residual = activation != MoEActivation.SILU
intermediate = moe_gemm_a16w4(
hidden_states,
w1_data,
None,
w1_wscale,
None,
None,
quant_config.w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=None,
apply_swiglu=True,
alpha=swiglu_alpha,
limit=swiglu_limit,
swiglu_add_residual=swiglu_add_residual,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
out = moe_gemm_a16w4(
intermediate,
w2_data,
None,
w2_wscale,
None,
None,
quant_config.w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=None,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, None)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=self.moe_config.intermediate_size_per_partition_unpadded * 2,
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
)
@@ -116,9 +116,12 @@ class Mxfp4MoeBackend(Enum):
BATCHED_MARLIN = "BATCHED_MARLIN"
MARLIN = "MARLIN"
# ROCm AITER backends
AITER_MXFP4_BF16 = "AITER_MXFP4_BF16" # W4A16: CK kernel
# Keep the legacy name as an alias while the ROCm split backend rename settles.
AITER_MXFP4_BF16 = "AITER_MXFP4_BF16" # W4A16: CK kernel (gfx950)
# Legacy alias, resolves to the CK kernel. New code should name either
# AITER_MXFP4_BF16 (CK) or AITER_TRITON_MXFP4_BF16 (Triton) explicitly.
AITER = "AITER_MXFP4_BF16"
# W4A16: aiter Triton moe_gemm_a16w4 kernel (gfx942/gfx950/gfx1250)
AITER_TRITON_MXFP4_BF16 = "AITER_TRITON_MXFP4_BF16"
AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel
# Triton
@@ -143,6 +146,8 @@ TRTLLM_BACKENDS = (
TRITON_BACKENDS = (
Mxfp4MoeBackend.TRITON,
Mxfp4MoeBackend.TRITON_UNFUSED,
# aiter Triton W4A16 shares the triton_kernels weight format
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
)
B12X_BACKENDS = (
@@ -232,14 +237,18 @@ def backend_to_kernel_cls(
return [BatchedMarlinExperts]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
return [AiterExperts, AiterW4A16ExpertsMonolithic]
return [AiterExperts]
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
return [AiterW4A16ExpertsMonolithic]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
@@ -301,9 +310,11 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
"marlin": [Mxfp4MoeBackend.MARLIN],
"aiter": [
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
],
"aiter_triton_mxfp4_bf16": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"aiter_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_MXFP4_MXFP4],
"xpu": [Mxfp4MoeBackend.XPU],
@@ -325,6 +336,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.TRITON,
@@ -1647,7 +1659,18 @@ def convert_weight_to_mxfp4_moe_kernel_format(
):
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
if mxfp4_backend == Mxfp4MoeBackend.TRITON:
if mxfp4_backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16:
# AITER moe_gemm_a16w4 needs gate/up interleaved
def interleave_gate_up(w: torch.Tensor) -> torch.Tensor:
gate, up = w.chunk(2, dim=1)
return torch.stack((gate, up), dim=2).reshape(w.shape)
w13_weight = interleave_gate_up(w13_weight)
w13_weight_scale = interleave_gate_up(w13_weight_scale)
if w13_bias is not None:
w13_bias = interleave_gate_up(w13_bias.to(torch.float32))
elif mxfp4_backend == Mxfp4MoeBackend.TRITON:
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
shape = w.shape
@@ -1891,6 +1914,7 @@ def make_mxfp4_moe_quant_config(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
Mxfp4MoeBackend.CPU,
):
return mxfp4_w4a16_moe_quant_config(