diff --git a/tests/kernels/moe/test_flashinfer_moe.py b/tests/kernels/moe/test_flashinfer_moe.py index 822f0f7d94..a98844d5f8 100644 --- a/tests/kernels/moe/test_flashinfer_moe.py +++ b/tests/kernels/moe/test_flashinfer_moe.py @@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, + FusedMoEQuantConfig, RoutingMethodType, ) from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( @@ -52,6 +53,74 @@ MNK_FACTORS = [ ] +@pytest.mark.parametrize( + "activation", + [MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE], +) +def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch): + from flashinfer.fused_moe.core import ActivationType + + moe_config = FusedMoEConfig( + num_experts=2, + experts_per_token=1, + hidden_dim=128, + intermediate_size=128, + num_local_experts=2, + num_logical_experts=2, + activation=activation, + device="cuda", + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + routing_method=RoutingMethodType.TopK, + ) + quant_config = FusedMoEQuantConfig.make( + gemm1_alpha=1.702, + gemm1_beta=1.0, + gemm1_clamp_limit=7.0, + ) + experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config) + + call_args = {} + + def fake_flashinfer_cutlass_fused_moe(**kwargs): + call_args.update(kwargs) + + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.experts." + "flashinfer_cutlass_moe.flashinfer_cutlass_fused_moe", + fake_flashinfer_cutlass_fused_moe, + ) + experts.apply( + output=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16), + hidden_states=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16), + w1=torch.empty((2, 256, 128), device="cuda", dtype=torch.bfloat16), + w2=torch.empty((2, 128, 128), device="cuda", dtype=torch.bfloat16), + topk_weights=torch.ones((1, 1), device="cuda", dtype=torch.float32), + topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64), + activation=activation, + global_num_experts=2, + expert_map=None, + a1q_scale=None, + a2_scale=None, + workspace13=None, + workspace2=None, + expert_tokens_meta=None, + apply_router_weight_on_input=False, + ) + + assert experts._supports_activation(activation) + assert call_args["activation_type"] == ActivationType.Swiglu + for name, value in ( + ("swiglu_alpha", 1.702), + ("swiglu_beta", 1.0), + ("swiglu_limit", 7.0), + ): + torch.testing.assert_close( + call_args[name], + torch.full((2,), value, device="cuda", dtype=torch.float32), + ) + + @pytest.mark.parametrize("m,n,k", MNK_FACTORS) @pytest.mark.parametrize("e", [40, 64, 256]) @pytest.mark.parametrize("topk", [1, 6, 8]) diff --git a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py index 2b8657723b..77afcf0d66 100644 --- a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py @@ -13,6 +13,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) +from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + activation_to_flashinfer_type, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kFp8Dynamic128Sym, @@ -93,29 +96,30 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): # - pass per-block weight scales to the kernel # - skip input activation quantization (kernel applies scaling) self.use_deepseek_fp8_block_scale = quant_config.is_block_quantized - self.gemm1_clamp_limit: torch.Tensor | None = None - if quant_config.gemm1_clamp_limit is not None: - self.gemm1_clamp_limit = torch.tensor( - [quant_config.gemm1_clamp_limit] * self.num_experts, + + def _per_expert(value: float | None) -> torch.Tensor | None: + if value is None: + return None + return torch.full( + (self.num_experts,), + float(value), dtype=torch.float32, device=self.device, ) + self.gemm1_clamp_limit = _per_expert(quant_config.gemm1_clamp_limit) + self.gemm1_alpha = _per_expert(quant_config.gemm1_alpha) + self.gemm1_beta = _per_expert(quant_config.gemm1_beta) + if quant_config.weight_quant_dtype == "mxfp4": # This value is used specifically for gpt-oss, # Need to revisit this for other models - self.gemm1_alpha = torch.tensor( - [1.702] * self.num_experts, dtype=torch.float32, device=self.device - ) - self.gemm1_beta = torch.tensor( - [1.0] * self.num_experts, dtype=torch.float32, device=self.device - ) + if self.gemm1_alpha is None: + self.gemm1_alpha = _per_expert(1.702) + if self.gemm1_beta is None: + self.gemm1_beta = _per_expert(1.0) if self.gemm1_clamp_limit is None: - self.gemm1_clamp_limit = torch.tensor( - [7.0] * self.num_experts, - dtype=torch.float32, - device=self.device, - ) + self.gemm1_clamp_limit = _per_expert(7.0) if quant_config.quant_dtype == "mxfp8": self.fake_input_scale = torch.ones( self.num_experts, @@ -190,6 +194,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): MoEActivation.GELU_TANH, MoEActivation.RELU2_NO_MUL, MoEActivation.SWIGLUOAI, + MoEActivation.SWIGLUOAI_UNINTERLEAVE, ] @staticmethod @@ -263,18 +268,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): expert_tokens_meta: mk.ExpertTokensMetadata | None, apply_router_weight_on_input: bool | None, ): - from flashinfer.fused_moe.core import ActivationType - - activation_str_to_value_map = { - MoEActivation.SILU: ActivationType.Swiglu, # This is the default - MoEActivation.GELU_TANH: ActivationType.Geglu, - MoEActivation.SWIGLUOAI: ActivationType.Swiglu, # gpt-oss alias - MoEActivation.RELU2_NO_MUL: ActivationType.Relu2, - } - assert activation in activation_str_to_value_map, ( - f"{activation=} missing from {activation_str_to_value_map.keys()=}" - ) - quant_scales = None fc1_expert_weights = None fc2_expert_weights = None @@ -282,9 +275,16 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): fc2_expert_biases = None swiglu_alpha = None swiglu_beta = None - swiglu_limit = ( - self.gemm1_clamp_limit if activation == MoEActivation.SILU else None - ) + swiglu_limit = None + if activation == MoEActivation.SILU: + swiglu_limit = self.gemm1_clamp_limit + elif activation in ( + MoEActivation.SWIGLUOAI, + MoEActivation.SWIGLUOAI_UNINTERLEAVE, + ): + swiglu_alpha = self.gemm1_alpha + swiglu_beta = self.gemm1_beta + swiglu_limit = self.gemm1_clamp_limit use_mxfp8_act_scaling = False use_w4_group_scaling = False # Select quantization metadata based on FP8 format/path @@ -392,7 +392,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): tp_rank=self.tp_rank, ep_size=self.ep_size, ep_rank=self.ep_rank, - activation_type=activation_str_to_value_map[activation], + activation_type=activation_to_flashinfer_type(activation), # Informs FlashInfer to use the block-scale decoding path when True use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale, use_mxfp8_act_scaling=use_mxfp8_act_scaling,