diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 00d3cf29c8..9157d4eaa2 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -134,10 +134,17 @@ def test_mxfp4_loading_and_execution_moe(vllm_runner, model_case: ModelCase): assert output -def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = None): - # Note we add an extra bias of 1 to the linear layer - # Uses chunked layout: first half is gate, second half is up - x_glu, x_linear = torch.chunk(x, 2, dim=-1) +def swiglu( + x, + alpha: float = 1.702, + beta: float = 1.0, + limit: float | None = None, + interleaved: bool = False, +): + if interleaved: + x_glu, x_linear = x[..., ::2], x[..., 1::2] + else: + x_glu, x_linear = torch.chunk(x, 2, dim=-1) if limit is not None: x_glu = x_glu.clamp(max=limit) x_linear = x_linear.clamp(min=-limit, max=limit) @@ -145,16 +152,6 @@ def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = Non return out_glu * (x_linear + beta) -def swigluoai(x, alpha: float = 1.702, limit: float = 7.0): - # OAI swiglu uses interleaved layout: gate/up alternating - # See SwigluOAIAndMul in vllm/model_executor/layers/activation.py - gate, up = x[..., ::2], x[..., 1::2] - gate = gate.clamp(max=limit) - up = up.clamp(min=-limit, max=limit) - glu = gate * torch.sigmoid(gate * alpha) - return (up + 1) * glu - - fp4_lookup_table = [0, 0.5, 1, 1.5, 2, 3, 4, 6, -0, -0.5, -1, -1.5, -2, -3, -4, -6] @@ -242,12 +239,9 @@ def reference_moe( # Apply activation if activation in ("swiglu", "silu"): - if use_interleaved_layout: - # SWIGLUOAI: interleaved gate/up layout - t = swigluoai(t, alpha=alpha, limit=limit) - else: - # Standard swiglu/silu: chunked layout - t = swiglu(t, alpha=alpha, beta=beta, limit=limit) + t = swiglu( + t, alpha=alpha, beta=beta, limit=limit, interleaved=use_interleaved_layout + ) elif activation == "relu2": # RELU2_NO_MUL: relu(x)^2 t = torch.relu(t) @@ -1403,6 +1397,14 @@ ROCM_BACKEND_CONFIGS = { "requires_aiter": True, "requires_gfx950": True, }, + "AITER_TRITON_MXFP4_BF16": { + "activation": "SILU", + "rtol": 0.3, + "percent": 0.95, + "requires_aiter": True, + "requires_gfx950": False, + "interleaved_layout": True, + }, "AITER_MXFP4_FP8": { "activation": "SWIGLUOAI", "rtol": 0.5, @@ -1670,7 +1672,9 @@ def test_rocm_mxfp4_moe_oracle( # Determine activation type and layout # SWIGLUOAI uses interleaved layout (gate/up alternating) # SILU uses chunked layout (first half gate, second half up) - use_interleaved = activation == MoEActivation.SWIGLUOAI + use_interleaved = bool( + config.get("interleaved_layout", activation == MoEActivation.SWIGLUOAI) + ) if activation in [MoEActivation.SWIGLUOAI, MoEActivation.SILU]: act_name = "swiglu" else: diff --git a/tests/models/quantization/test_gpt_oss.py b/tests/models/quantization/test_gpt_oss.py index f1325f7147..209f152759 100644 --- a/tests/models/quantization/test_gpt_oss.py +++ b/tests/models/quantization/test_gpt_oss.py @@ -31,6 +31,8 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import ( aiter_triton_kernel_w4a8_moe_forward, +) +from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import ( aiter_triton_kernel_w4a16_moe_forward, ) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 0b5c368dcc..3e597b01fc 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -137,6 +137,7 @@ MoEBackend = Literal[ "humming", "triton_unfused", "aiter", + "aiter_triton_mxfp4_bf16", "flydsl", "hpc", "emulation", @@ -260,6 +261,8 @@ class KernelConfig: - "humming": Use Humming Mixed Precision kernels - "triton_unfused": Use Triton unfused MoE kernels - "aiter": Use AMD AITer kernels (ROCm only) + - "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16 + (moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250) - "flydsl": Use AMD FlyDSL kernels (ROCm only) - "hpc": Use HPC kernels (FP8 and Hopper only) - "emulation": use BF16/FP16 GEMM, dequantizing weights and diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py new file mode 100644 index 0000000000..3a0bdf428b --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py @@ -0,0 +1,403 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import rocm_aiter_ops +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import ( + patch_gating_output, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kMxfp4Static, +) + +__all__ = [ + "AiterW4A16ExpertsMonolithic", + "aiter_triton_kernel_w4a16_moe_forward", +] + + +def _aiter_raw(t): + """Unwrap the `triton_kernels.tensor.Tensor` that `_swizzle_mxfp4` returns.""" + if t is None or isinstance(t, torch.Tensor): + return t + assert hasattr(t, "storage"), ( + f"expected a triton_kernels wrapped tensor with a .storage attribute, " + f"got {type(t)}" + ) + return t.storage.data + + +def _aiter_w4a16_silu_via_a8w4( + hidden_states: torch.Tensor, + w1_data, + w2_data, + w1_wscale, + w2_wscale, + w1_bias, + w2_bias, + routing_data, + gather_idx, + scatter_idx, + gammas, + apply_router_weight_on_input: bool, + swiglu_limit: float, + unpadded_N_w1, + unpadded_K_w1, + unpadded_N_w2, + unpadded_K_w2, +) -> torch.Tensor: + """ + MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation. + """ + from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul + from aiter.ops.triton.quant import dynamic_mxfp8_quant + + try: + from aiter.ops.triton.moe.moe_op_gemm_a8w4 import moe_gemm_a8w4 + except ImportError: + from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4 + + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + should_use_cdna4_mx_scale_swizzle, + ) + + swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None + quant_dtype = torch.float8_e4m3fn + + g1_gammas = gammas if apply_router_weight_on_input else None + g2_gammas = None if apply_router_weight_on_input else gammas + + hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype) + raw_gate_up = moe_gemm_a8w4( + hidden_q, + w1_data, + a1_scale, + w1_wscale, + None, + None, + w1_bias, + routing_data, + gather_indx=gather_idx, + gammas=g1_gammas, + swizzle_mx_scale=swz, + out_dtype=torch.bfloat16, + apply_swiglu=False, + unpadded_N=unpadded_N_w1, + unpadded_K=unpadded_K_w1, + ) + if unpadded_N_w1 is not None: + raw_gate_up = raw_gate_up[:, :unpadded_N_w1] + + # convert interleaved -> chunked for kernel + gate, up = raw_gate_up[:, ::2], raw_gate_up[:, 1::2] + raw_gate_up = torch.cat((gate, up), dim=1).contiguous() + + interim_fp8, a2_scale = fused_clamp_act_mul( + raw_gate_up, + swiglu_limit=swiglu_limit, + activation="silu", + dtype_quant=quant_dtype, + scale_dtype_fmt="ue8m0", + quant_block_size=32, + ) + + out = moe_gemm_a8w4( + interim_fp8, + w2_data, + a2_scale, + w2_wscale, + None, + None, + w2_bias, + routing_data, + scatter_indx=scatter_idx, + gammas=g2_gammas, + swizzle_mx_scale=swz, + unpadded_N=unpadded_N_w2, + unpadded_K=unpadded_K_w2, + ) + return out + + +def aiter_triton_kernel_w4a16_moe_forward( + hidden_states: torch.Tensor, + w1, + w2, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + activation: MoEActivation = MoEActivation.SWIGLUOAI, + quant_config: FusedMoEQuantConfig | None = None, + apply_router_weight_on_input: bool = False, + global_num_experts: int = -1, + expert_map: torch.Tensor | None = None, + unpadded_N_w1=None, + unpadded_K_w1=None, + unpadded_N_w2=None, + unpadded_K_w2=None, + num_expert_group: int | None = None, + topk_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + score_mode: str | None = None, +): + assert quant_config is not None and rocm_aiter_ops.is_enabled() + from vllm.platforms.rocm import on_gfx1250 + + try: + from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4 + except ImportError: + from aiter.ops.triton.moe_op_gemm_a16w4 import moe_gemm_a16w4 + try: + from aiter.ops.triton.moe.moe_routing import routing as _routing_mod + except ImportError: + from aiter.ops.triton.moe_routing import routing as _routing_mod + + if on_gfx1250(): + _routing_mod.is_tdm_avail = lambda: False + aiter_routing = _routing_mod.routing + + # See context in #50859. + # TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged, + # AITER released, and AITER pin in vLLM increased from 0.1.19. + gating_output = patch_gating_output(gating_output, global_num_experts) + + if score_mode is not None: + use_grouped_topk = num_expert_group is not None and num_expert_group > 1 + routing_data, gather_idx, scatter_idx = aiter_routing( + gating_output, + topk, + score_mode=score_mode, + bias=e_score_correction_bias, + renorm=renormalize, + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + use_grouped_topk=use_grouped_topk, + num_expert_group=num_expert_group, + topk_group=topk_group, + ) + else: + routing_data, gather_idx, scatter_idx = aiter_routing( + gating_output, topk, sm_first=not renormalize + ) + + if on_gfx1250(): + gather_src = gather_idx.to(torch.long) // topk + hidden_states = hidden_states[gather_src] + gather_idx = None + + assert quant_config.w1_precision is not None + assert quant_config.w2_precision is not None + + w1_data = _aiter_raw(w1) + w2_data = _aiter_raw(w2) + w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale) + w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale) + + gammas = routing_data.gate_scal if routing_data else None + + swiglu_alpha = ( + quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0 + ) + # 7.0 is the SwigluOAI clamp, so only default to it for that activation; + # a plain-SILU model that configures no limit must not be clamped. + if quant_config.gemm1_clamp_limit is not None: + swiglu_limit = quant_config.gemm1_clamp_limit + elif activation == MoEActivation.SWIGLUOAI: + swiglu_limit = 7.0 + else: + swiglu_limit = None + + # SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults. + if activation == MoEActivation.SILU and on_gfx1250(): + return _aiter_w4a16_silu_via_a8w4( + hidden_states, + w1_data, + w2_data, + w1_wscale, + w2_wscale, + quant_config.w1_bias, + quant_config.w2_bias, + routing_data, + gather_idx, + scatter_idx, + gammas, + apply_router_weight_on_input, + # fused_clamp_act_mul encodes "no clamp" as 0, not None. + swiglu_limit if swiglu_limit is not None else 0.0, + unpadded_N_w1, + unpadded_K_w1, + unpadded_N_w2, + unpadded_K_w2, + ) + + # SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu. + swiglu_add_residual = activation != MoEActivation.SILU + + intermediate = moe_gemm_a16w4( + hidden_states, + w1_data, + None, + w1_wscale, + None, + None, + quant_config.w1_bias, + routing_data, + gather_indx=gather_idx, + gammas=gammas if apply_router_weight_on_input else None, + swizzle_mx_scale=None, + apply_swiglu=True, + alpha=swiglu_alpha, + limit=swiglu_limit, + swiglu_add_residual=swiglu_add_residual, + unpadded_N=unpadded_N_w1, + unpadded_K=unpadded_K_w1, + ) + + out = moe_gemm_a16w4( + intermediate, + w2_data, + None, + w2_wscale, + None, + None, + quant_config.w2_bias, + routing_data, + scatter_indx=scatter_idx, + gammas=None if apply_router_weight_on_input else gammas, + swizzle_mx_scale=None, + unpadded_N=unpadded_N_w2, + unpadded_K=unpadded_K_w2, + ) + + return out + + +class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + self.topk = moe_config.experts_per_token + self.renormalize = moe_config.routing_method in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepseekV4, + ) + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + if not rocm_aiter_ops.is_enabled(): + return False + from vllm.platforms.rocm import on_gfx942, on_gfx950, on_gfx1250 + + return on_gfx942() or on_gfx950() or on_gfx1250() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == (kMxfp4Static, None) + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU) + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return ( + not moe_parallel_config.use_all2all_kernels + and not moe_parallel_config.enable_eplb + and moe_parallel_config.dp_size <= 1 + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepseekV4, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + score_mode = ( + "sqrtsoftplus" + if self.moe_config.routing_method == RoutingMethodType.DeepseekV4 + else None + ) + return aiter_triton_kernel_w4a16_moe_forward( + hidden_states=hidden_states, + w1=w1, + w2=w2, + gating_output=router_logits, + topk=self.topk, + renormalize=self.renormalize, + activation=activation, + global_num_experts=global_num_experts, + expert_map=expert_map, + quant_config=self.quant_config, + apply_router_weight_on_input=apply_router_weight_on_input, + unpadded_N_w1=None, + unpadded_K_w1=None, + unpadded_N_w2=None, + unpadded_K_w2=None, + num_expert_group=num_expert_group, + topk_group=topk_group, + e_score_correction_bias=e_score_correction_bias, + routed_scaling_factor=routed_scaling_factor, + score_mode=score_mode, + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py index 8d68ba38f7..9d35774f4d 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py @@ -21,9 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( __all__ = [ "AiterW4A8ExpertsMonolithic", - "AiterW4A16ExpertsMonolithic", "aiter_triton_kernel_w4a8_moe_forward", - "aiter_triton_kernel_w4a16_moe_forward", ] @@ -371,363 +369,3 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): unpadded_N_w2=self.moe_config.hidden_dim_unpadded, unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded, ) - - -def _aiter_raw(t): - if t is None or isinstance(t, torch.Tensor): - return t - return t.storage.data if hasattr(t, "storage") else t - - -def _aiter_w4a16_silu_via_a8w4( - hidden_states: torch.Tensor, - w1_data, - w2_data, - w1_wscale, - w2_wscale, - w1_bias, - w2_bias, - routing_data, - gather_idx, - scatter_idx, - gammas, - apply_router_weight_on_input: bool, - swiglu_limit: float, - unpadded_N_w1, - unpadded_K_w1, - unpadded_N_w2, - unpadded_K_w2, -) -> torch.Tensor: - """ - MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation. - """ - from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul - from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4 - from aiter.ops.triton.quant import dynamic_mxfp8_quant - - from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( - should_use_cdna4_mx_scale_swizzle, - ) - - swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None - quant_dtype = torch.float8_e4m3fn - - g1_gammas = gammas if apply_router_weight_on_input else None - g2_gammas = None if apply_router_weight_on_input else gammas - - hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype) - raw_gate_up = moe_gemm_a8w4( - hidden_q, - w1_data, - a1_scale, - w1_wscale, - None, - None, - w1_bias, - routing_data, - gather_indx=gather_idx, - gammas=g1_gammas, - swizzle_mx_scale=swz, - out_dtype=torch.bfloat16, - apply_swiglu=False, - unpadded_N=unpadded_N_w1, - unpadded_K=unpadded_K_w1, - ) - if unpadded_N_w1 is not None: - raw_gate_up = raw_gate_up[:, :unpadded_N_w1] - - interim_fp8, a2_scale = fused_clamp_act_mul( - raw_gate_up, - swiglu_limit=swiglu_limit, - activation="silu", - dtype_quant=quant_dtype, - scale_dtype_fmt="ue8m0", - quant_block_size=32, - ) - - out = moe_gemm_a8w4( - interim_fp8, - w2_data, - a2_scale, - w2_wscale, - None, - None, - w2_bias, - routing_data, - scatter_indx=scatter_idx, - gammas=g2_gammas, - swizzle_mx_scale=swz, - unpadded_N=unpadded_N_w2, - unpadded_K=unpadded_K_w2, - ) - return out - - -def aiter_triton_kernel_w4a16_moe_forward( - hidden_states: torch.Tensor, - w1, - w2, - gating_output: torch.Tensor, - topk: int, - renormalize: bool, - activation: MoEActivation = MoEActivation.SWIGLUOAI, - quant_config: FusedMoEQuantConfig | None = None, - apply_router_weight_on_input: bool = False, - global_num_experts: int = -1, - expert_map: torch.Tensor | None = None, - unpadded_N_w1=None, - unpadded_K_w1=None, - unpadded_N_w2=None, - unpadded_K_w2=None, - num_expert_group: int | None = None, - topk_group: int | None = None, - e_score_correction_bias: torch.Tensor | None = None, - routed_scaling_factor: float | None = None, - score_mode: str | None = None, -): - assert quant_config is not None and rocm_aiter_ops.is_enabled() - from vllm.platforms.rocm import on_gfx1250 - - try: - from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4 - from aiter.ops.triton.moe.moe_routing import routing as _routing_mod - except ImportError: - from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4 - from aiter.ops.triton.moe_routing import routing as _routing_mod - - if on_gfx1250(): - _routing_mod.is_tdm_avail = lambda: False - aiter_routing = _routing_mod.routing - - # See context in #50859. - # TODO: Remove once https://github.com/ROCm/aiter/pull/4530 is merged, - # AITER released, and AITER pin in vLLM increased from 0.1.19. - gating_output = patch_gating_output(gating_output, global_num_experts) - - if score_mode is not None: - use_grouped_topk = num_expert_group is not None and num_expert_group > 1 - routing_data, gather_idx, scatter_idx = aiter_routing( - gating_output, - topk, - score_mode=score_mode, - bias=e_score_correction_bias, - renorm=renormalize, - routed_scaling_factor=( - routed_scaling_factor if routed_scaling_factor is not None else 1.0 - ), - use_grouped_topk=use_grouped_topk, - num_expert_group=num_expert_group, - topk_group=topk_group, - ) - else: - routing_data, gather_idx, scatter_idx = aiter_routing( - gating_output, topk, sm_first=not renormalize - ) - - if on_gfx1250(): - gather_src = gather_idx.to(torch.long) // topk - hidden_states = hidden_states[gather_src] - gather_idx = None - - assert quant_config.w1_precision is not None - assert quant_config.w2_precision is not None - - w1_data = _aiter_raw(w1) - w2_data = _aiter_raw(w2) - w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale) - w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale) - - gammas = routing_data.gate_scal if routing_data else None - - swiglu_alpha = ( - quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0 - ) - swiglu_limit = ( - quant_config.gemm1_clamp_limit - if quant_config.gemm1_clamp_limit is not None - else 7.0 - ) - - # SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults. - if activation == MoEActivation.SILU and on_gfx1250(): - return _aiter_w4a16_silu_via_a8w4( - hidden_states, - w1_data, - w2_data, - w1_wscale, - w2_wscale, - quant_config.w1_bias, - quant_config.w2_bias, - routing_data, - gather_idx, - scatter_idx, - gammas, - apply_router_weight_on_input, - swiglu_limit, - unpadded_N_w1, - unpadded_K_w1, - unpadded_N_w2, - unpadded_K_w2, - ) - - # SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu. - swiglu_add_residual = activation != MoEActivation.SILU - - intermediate = moe_gemm_a16w4( - hidden_states, - w1_data, - None, - w1_wscale, - None, - None, - quant_config.w1_bias, - routing_data, - gather_indx=gather_idx, - gammas=gammas if apply_router_weight_on_input else None, - swizzle_mx_scale=None, - apply_swiglu=True, - alpha=swiglu_alpha, - limit=swiglu_limit, - swiglu_add_residual=swiglu_add_residual, - unpadded_N=unpadded_N_w1, - unpadded_K=unpadded_K_w1, - ) - - out = moe_gemm_a16w4( - intermediate, - w2_data, - None, - w2_wscale, - None, - None, - quant_config.w2_bias, - routing_data, - scatter_indx=scatter_idx, - gammas=None if apply_router_weight_on_input else gammas, - swizzle_mx_scale=None, - unpadded_N=unpadded_N_w2, - unpadded_K=unpadded_K_w2, - ) - - return out - - -class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): - def __init__( - self, - moe_config: FusedMoEConfig, - quant_config: FusedMoEQuantConfig, - ): - super().__init__(moe_config, quant_config) - self.topk = moe_config.experts_per_token - self.renormalize = moe_config.routing_method in ( - RoutingMethodType.Renormalize, - RoutingMethodType.RenormalizeNaive, - RoutingMethodType.DeepseekV4, - ) - - @staticmethod - def activation_format() -> mk.FusedMoEActivationFormat: - return mk.FusedMoEActivationFormat.Standard - - @staticmethod - def _supports_current_device() -> bool: - if not rocm_aiter_ops.is_enabled(): - return False - from vllm.platforms.rocm import on_gfx950, on_gfx1250 - - return on_gfx950() or on_gfx1250() - - @staticmethod - def _supports_no_act_and_mul() -> bool: - return False - - @staticmethod - def _supports_quant_scheme( - weight_key: QuantKey | None, - activation_key: QuantKey | None, - ) -> bool: - return (weight_key, activation_key) == (kMxfp4Static, None) - - @staticmethod - def _supports_activation(activation: MoEActivation) -> bool: - return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU) - - @staticmethod - def _supports_parallel_config( - moe_parallel_config: FusedMoEParallelConfig, - ) -> bool: - return ( - not moe_parallel_config.use_all2all_kernels - and not moe_parallel_config.enable_eplb - and moe_parallel_config.dp_size <= 1 - ) - - @staticmethod - def _supports_routing_method( - routing_method: RoutingMethodType, - weight_key: QuantKey | None, - activation_key: QuantKey | None, - ) -> bool: - return routing_method in [ - RoutingMethodType.Renormalize, - RoutingMethodType.RenormalizeNaive, - RoutingMethodType.DeepseekV4, - ] - - @staticmethod - def _supports_router_logits_dtype( - router_logits_dtype: torch.dtype | None, - routing_method: RoutingMethodType, - ) -> bool: - return True - - @property - def expects_unquantized_inputs(self) -> bool: - return True - - def apply( - self, - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - router_logits: torch.Tensor, - activation: MoEActivation, - global_num_experts: int, - expert_map: torch.Tensor | None, - a1q_scale: torch.Tensor | None, - apply_router_weight_on_input: bool, - num_expert_group: int | None = None, - e_score_correction_bias: torch.Tensor | None = None, - routed_scaling_factor: float | None = None, - topk_group: int | None = None, - ) -> torch.Tensor: - assert self.moe_config.intermediate_size_per_partition_unpadded is not None - assert self.moe_config.hidden_dim_unpadded is not None - score_mode = ( - "sqrtsoftplus" - if self.moe_config.routing_method == RoutingMethodType.DeepseekV4 - else None - ) - return aiter_triton_kernel_w4a16_moe_forward( - hidden_states=hidden_states, - w1=w1, - w2=w2, - gating_output=router_logits, - topk=self.topk, - renormalize=self.renormalize, - activation=activation, - global_num_experts=global_num_experts, - expert_map=expert_map, - quant_config=self.quant_config, - apply_router_weight_on_input=apply_router_weight_on_input, - unpadded_N_w1=self.moe_config.intermediate_size_per_partition_unpadded * 2, - unpadded_K_w1=self.moe_config.hidden_dim_unpadded, - unpadded_N_w2=self.moe_config.hidden_dim_unpadded, - unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded, - num_expert_group=num_expert_group, - topk_group=topk_group, - e_score_correction_bias=e_score_correction_bias, - routed_scaling_factor=routed_scaling_factor, - score_mode=score_mode, - ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index f0de8522fa..77083c67ea 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -116,9 +116,12 @@ class Mxfp4MoeBackend(Enum): BATCHED_MARLIN = "BATCHED_MARLIN" MARLIN = "MARLIN" # ROCm AITER backends - AITER_MXFP4_BF16 = "AITER_MXFP4_BF16" # W4A16: CK kernel - # Keep the legacy name as an alias while the ROCm split backend rename settles. + AITER_MXFP4_BF16 = "AITER_MXFP4_BF16" # W4A16: CK kernel (gfx950) + # Legacy alias, resolves to the CK kernel. New code should name either + # AITER_MXFP4_BF16 (CK) or AITER_TRITON_MXFP4_BF16 (Triton) explicitly. AITER = "AITER_MXFP4_BF16" + # W4A16: aiter Triton moe_gemm_a16w4 kernel (gfx942/gfx950/gfx1250) + AITER_TRITON_MXFP4_BF16 = "AITER_TRITON_MXFP4_BF16" AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel # Triton @@ -143,6 +146,8 @@ TRTLLM_BACKENDS = ( TRITON_BACKENDS = ( Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.TRITON_UNFUSED, + # aiter Triton W4A16 shares the triton_kernels weight format + Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16, ) B12X_BACKENDS = ( @@ -232,14 +237,18 @@ def backend_to_kernel_cls( return [BatchedMarlinExperts] elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16: - from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import ( - AiterW4A16ExpertsMonolithic, - ) from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) - return [AiterExperts, AiterW4A16ExpertsMonolithic] + return [AiterExperts] + + elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16: + from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import ( + AiterW4A16ExpertsMonolithic, + ) + + return [AiterW4A16ExpertsMonolithic] elif backend == Mxfp4MoeBackend.AITER_MXFP4_FP8: from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import ( @@ -301,9 +310,11 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]: "marlin": [Mxfp4MoeBackend.MARLIN], "aiter": [ Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_MXFP4, ], + "aiter_triton_mxfp4_bf16": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16], "aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8], "aiter_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_MXFP4_MXFP4], "xpu": [Mxfp4MoeBackend.XPU], @@ -325,6 +336,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8, Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_MXFP4, Mxfp4MoeBackend.TRITON, @@ -1647,7 +1659,18 @@ def convert_weight_to_mxfp4_moe_kernel_format( ): from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig - if mxfp4_backend == Mxfp4MoeBackend.TRITON: + if mxfp4_backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16: + # AITER moe_gemm_a16w4 needs gate/up interleaved + def interleave_gate_up(w: torch.Tensor) -> torch.Tensor: + gate, up = w.chunk(2, dim=1) + return torch.stack((gate, up), dim=2).reshape(w.shape) + + w13_weight = interleave_gate_up(w13_weight) + w13_weight_scale = interleave_gate_up(w13_weight_scale) + + if w13_bias is not None: + w13_bias = interleave_gate_up(w13_bias.to(torch.float32)) + elif mxfp4_backend == Mxfp4MoeBackend.TRITON: def shuffle_weight(w: torch.Tensor) -> torch.Tensor: shape = w.shape @@ -1891,6 +1914,7 @@ def make_mxfp4_moe_quant_config( Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16, Mxfp4MoeBackend.CPU, ): return mxfp4_w4a16_moe_quant_config(