diff --git a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py index eaeca6a8a5..0f80ca5c55 100644 --- a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py +++ b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py @@ -5,9 +5,13 @@ import pytest import torch +from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( fused_marlin_moe, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import ( + TrtLlmMxint4ExpertsMonolithic, +) from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( grouped_topk, ) @@ -77,6 +81,14 @@ __all__ = [ ] +def test_trtllm_mxint4_activation_supports_vllm_gated_silu(): + assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SILU) + assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SWIGLUOAI) + assert not TrtLlmMxint4ExpertsMonolithic._supports_activation( + MoEActivation.RELU2_NO_MUL + ) + + def marlin_quantize_moe_weights( weights_bf16: torch.Tensor, group_size: int = 32 ) -> tuple[torch.Tensor, torch.Tensor]: diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py index a412a6936d..c6e5e70a14 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py @@ -68,8 +68,10 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): @staticmethod def _supports_activation(activation: MoEActivation) -> bool: - # FlashInfer MxInt4 uses a fused SwiGLU activation. - return activation == MoEActivation.SWIGLUOAI + # FlashInfer MxInt4 names the standard gated SiLU path "SwiGLU". + # In vLLM MoE configs that maps to SILU/silu_and_mul; SWIGLUOAI is + # kept as an alias for consistency with other FlashInfer backends. + return activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI) @staticmethod def _supports_parallel_config(