diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 3f296ceb85..70437d0ecf 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -150,9 +150,12 @@ steps: - csrc/quantization/fp4/ - csrc/attention/mla/ - csrc/quantization/cutlass_w8a8/moe/ - - vllm/model_executor/layers/fused_moe/cutlass_moe.py - - vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py - - vllm/model_executor/layers/fused_moe/flashinfer_a2a_prepare_finalize.py + - vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py + - vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py + - vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py + - vllm/model_executor/layers/fused_moe/oracle/nvfp4.py + - vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py + - vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py - vllm/v1/attention/backends/flashinfer.py - vllm/v1/attention/backends/mla/cutlass_mla.py @@ -185,6 +188,7 @@ steps: - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py - pytest -v -s tests/kernels/moe/test_flashinfer.py - pytest -v -s tests/kernels/moe/test_flashinfer_moe.py + - pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py # e2e - pytest -v -s tests/models/quantization/test_nvfp4.py diff --git a/tests/kernels/moe/test_trtllm_nvfp4_moe.py b/tests/kernels/moe/test_trtllm_nvfp4_moe.py index e595197405..ca10ebaddc 100644 --- a/tests/kernels/moe/test_trtllm_nvfp4_moe.py +++ b/tests/kernels/moe/test_trtllm_nvfp4_moe.py @@ -22,6 +22,8 @@ from tests.kernels.quantization.nvfp4_utils import ( from tests.kernels.utils import torch_moe from vllm import _custom_ops as ops from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config +from vllm.model_executor.custom_op import CustomOp, op_registry +from vllm.model_executor.layers.activation import SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import fused_topk from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.all2all_utils import ( @@ -58,15 +60,45 @@ MNK_FACTORS = [ (64, 704, 4096), ] +_SWIGLU_LIMIT = 0.1 +_LARGE_OUTPUT1_SCALE = 32768.0 +_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp" + +# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself +# valid as an ``activation=`` argument to ``torch_moe`` (which only looks +# up ``activation.custom_op_name`` in ``op_registry``), so no +# ``MoEActivation`` enum extension is needed. +if _CLAMP_OP_NAME not in op_registry: + + @CustomOp.register(_CLAMP_OP_NAME) + class _SiluAndMulWithClampTest(SiluAndMulWithClamp): + custom_op_name = _CLAMP_OP_NAME + + def __init__(self, *, compile_native: bool = True) -> None: + super().__init__(_SWIGLU_LIMIT, compile_native=compile_native) + + +SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME] + + +ACTIVATION_CASES = [ + pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"), + pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"), + pytest.param( + MoEActivation.RELU2_NO_MUL, + MoEActivation.RELU2_NO_MUL, + None, + id="relu2_no_mul", + ), + pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, id="gelu"), +] + @pytest.mark.parametrize("m,n,k", MNK_FACTORS) @pytest.mark.parametrize("e", [128]) @pytest.mark.parametrize("topk", [8]) @pytest.mark.parametrize("dtype", [torch.bfloat16]) -@pytest.mark.parametrize( - "activation", - [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL, MoEActivation.GELU], -) +@pytest.mark.parametrize("activation,torch_activation,swiglu_limit", ACTIVATION_CASES) @torch.inference_mode() def test_trtllm_fp4_moe_no_graph( m: int, @@ -76,6 +108,8 @@ def test_trtllm_fp4_moe_no_graph( topk: int, dtype: torch.dtype, activation: MoEActivation, + torch_activation: MoEActivation | type[SiluAndMulWithClamp], + swiglu_limit: float | None, workspace_init, ): # FlashInfer's trtllm_batched_gemm_runner has no precompiled tile @@ -113,6 +147,15 @@ def test_trtllm_fp4_moe_no_graph( # Match what oracle/nvfp4.py does for this backend. is_scale_swizzled=False, ) + quant_config.gemm1_clamp_limit = swiglu_limit + if swiglu_limit is not None: + assert quant_config.g1_alphas is not None + assert quant_config.a2_gscale is not None + assert torch.all(quant_config.a2_gscale == 1) + # With a2_gscale == 1, g1_alphas is the TRTLLM + # output1_scale_gate_scalar. Make it large enough to catch + # clamp/output-scale coupling in the FlashInfer kernel wrapper. + quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE) score = torch.randn((m, e), device="cuda", dtype=dtype) topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False) @@ -133,6 +176,23 @@ def test_trtllm_fp4_moe_no_graph( max_num_tokens=next_power_of_2(m), ) + trtllm_inner = TrtLlmNvFp4ExpertsModular( + moe_config=moe_config, quant_config=quant_config + ) + # Mimic the production weight-loader path so per-expert tensors that + # are normally precomputed in process_weights_after_loading (g1_scale_c + # and the rescaled gemm1_clamp_limit) get materialized. The test's + # synthetic quant_config has g1_alphas/g2_alphas already at their + # post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas + # (same tensor) and use input_scale=1 to make the in-place + # weight_scale_2 *= input_scale step a no-op. + fake_layer = torch.nn.Module() + fake_layer.w13_weight_scale_2 = quant_config.g1_alphas + fake_layer.w2_weight_scale_2 = quant_config.g2_alphas + fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas) + fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas) + trtllm_inner.process_weights_after_loading(fake_layer) + trtllm_experts = mk.FusedMoEKernel( maybe_make_prepare_finalize( moe=moe_config, @@ -140,7 +200,7 @@ def test_trtllm_fp4_moe_no_graph( allow_new_interface=True, use_monolithic=False, ), - TrtLlmNvFp4ExpertsModular(moe_config=moe_config, quant_config=quant_config), + trtllm_inner, inplace=False, ) @@ -195,7 +255,7 @@ def test_trtllm_fp4_moe_no_graph( ) torch_output = torch_moe( - a_in_dtype, w1_d, w2_d, score, topk, activation=activation + a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation ) torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1) @@ -203,5 +263,14 @@ def test_trtllm_fp4_moe_no_graph( if __name__ == "__main__": test_trtllm_fp4_moe_no_graph( - 64, 704, 4096, 128, 8, torch.bfloat16, MoEActivation.GELU, None + 64, + 704, + 4096, + 128, + 8, + torch.bfloat16, + MoEActivation.GELU, + MoEActivation.GELU, + None, + None, ) diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index d6f915fa34..eec7d4b87f 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -805,6 +805,7 @@ def nvfp4_moe_quant_config( w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, is_scale_swizzled: bool = True, + gemm1_clamp_limit: float | None = None, ) -> FusedMoEQuantConfig: """ Construct a quant config for mxfp4 activations and nvp4 weights. @@ -823,6 +824,7 @@ def nvfp4_moe_quant_config( per_out_ch_quant=False, block_shape=None, is_scale_swizzled=is_scale_swizzled, + gemm1_clamp_limit=gemm1_clamp_limit, ) @@ -1271,6 +1273,11 @@ class FusedMoEConfig: is_act_and_mul: bool = True is_lora_enabled: bool = False + # SwiGLU clamp limit. When set, backends that do not implement the clamp + # are filtered out by `FusedMoEExperts.is_supported_config` so the oracle + # cannot silently select one and drop the clamp. + swiglu_limit: float | None = None + # This flag is used to disable the inplace optimization # in MoE kernels. If this flag is True then the kernel # should not be using inplace. If the flag is false, the diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index 96202cad62..3ddcdef775 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -66,6 +66,17 @@ class TrtLlmNvFp4ExpertsBase: else: self.g1_scale_c = self.quant_config.a2_gscale.clone() + if moe_config.is_act_and_mul and quant_config.gemm1_clamp_limit is not None: + device = torch.accelerator.current_device_index() + self.gemm1_clamp_limit = torch.full( + (self.local_num_experts,), + quant_config.gemm1_clamp_limit, + dtype=torch.float32, + device=device, + ) + else: + self.gemm1_clamp_limit = None + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale) layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale) @@ -84,6 +95,20 @@ class TrtLlmNvFp4ExpertsBase: ) self.g1_scale_c = layer.g1_scale_c + # Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar) + # division so the TRTLLM kernel receives the raw-GEMM-space clamp + # directly. g1_alphas is set once here in process_weights_after_loading + # (via the in-place mul above) and never changes again, so this is a + # static, per-expert constant. Register on the layer so EPLB + # rearranges it alongside the other expert tensors. + if self.gemm1_clamp_limit is not None: + gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas + layer.register_parameter( + "gemm1_clamp_limit", + torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False), + ) + self.gemm1_clamp_limit = layer.gemm1_clamp_limit + @staticmethod def _supports_current_device() -> bool: """Supports only Blackwell-family GPUs.""" @@ -201,6 +226,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula # Pack topk ids and weights into format expected by the kernel. packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights) + output1_scale_gate_scalar = self.quant_config.g1_alphas # Invoke kernel. flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe( @@ -215,12 +241,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=None, + gemm1_clamp_limit=self.gemm1_clamp_limit, gemm2_weights=w2, gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn), gemm2_bias=None, output1_scale_scalar=self.g1_scale_c, - output1_scale_gate_scalar=self.quant_config.g1_alphas, + output1_scale_gate_scalar=output1_scale_gate_scalar, output2_scale_scalar=self.quant_config.g2_alphas, num_experts=global_num_experts, top_k=self.topk, @@ -313,6 +339,8 @@ class TrtLlmNvFp4ExpertsMonolithic( if e_score_correction_bias is not None: e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16) + output1_scale_gate_scalar = self.quant_config.g1_alphas + # Invoke kernel. # NOTE: Activation padding and output # truncation are handled by the MoE runner's @@ -328,12 +356,12 @@ class TrtLlmNvFp4ExpertsMonolithic( gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=None, + gemm1_clamp_limit=self.gemm1_clamp_limit, gemm2_weights=w2, gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn), gemm2_bias=None, output1_scale_scalar=self.g1_scale_c, - output1_scale_gate_scalar=self.quant_config.g1_alphas, + output1_scale_gate_scalar=output1_scale_gate_scalar, output2_scale_scalar=self.quant_config.g2_alphas, num_experts=global_num_experts, top_k=self.topk, diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index f8ffac8df1..776e86770e 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -335,6 +335,7 @@ class FusedMoE(PluggableLayer): activation=self.activation, device=vllm_config.device_config.device, routing_method=self.routing_method_type, + swiglu_limit=swiglu_limit, # TODO: in_dtype == out_dtype? disable_inplace=disable_inplace() or shared_experts is not None, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 3e8f38c414..5a2c64cecc 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -181,6 +181,15 @@ def select_nvfp4_moe_backend( NvFp4MoeBackend.EMULATION, ] + NVFP4_BACKENDS_WITH_CLAMP = { + NvFp4MoeBackend.FLASHINFER_TRTLLM, + } + + if config.swiglu_limit is not None: + AVAILABLE_BACKENDS = [ + b for b in AVAILABLE_BACKENDS if b in NVFP4_BACKENDS_WITH_CLAMP + ] + use_batched = config.moe_parallel_config.use_batched_activation_format activation_format = ( mk.FusedMoEActivationFormat.BatchedExperts @@ -234,6 +243,16 @@ def select_nvfp4_moe_backend( and requested_backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL ): requested_backend = NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED + if ( + config.swiglu_limit is not None + and requested_backend not in NVFP4_BACKENDS_WITH_CLAMP + ): + raise ValueError( + f"Model sets swiglu_limit={config.swiglu_limit}, but the " + f"explicitly requested moe_backend={runner_backend!r} does " + f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm' or " + f"'flashinfer_cutlass' instead." + ) return _return_or_raise( requested_backend, config, weight_key, activation_key, activation_format ) @@ -242,17 +261,32 @@ def select_nvfp4_moe_backend( if not envs.VLLM_USE_FLASHINFER_MOE_FP4: # If the user rejects FlashInfer remove those backends. for b in FLASHINFER_NVFP4_MOE_BACKENDS: - AVAILABLE_BACKENDS.remove(b) + if b in AVAILABLE_BACKENDS: + AVAILABLE_BACKENDS.remove(b) elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"): # If user is explicit about backend, validate it. backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()] + if ( + config.swiglu_limit is not None + and backend not in NVFP4_BACKENDS_WITH_CLAMP + ): + raise ValueError( + f"Model sets swiglu_limit={config.swiglu_limit}, but the " + f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND " + f"({backend.value}) does not apply the SwiGLU clamp." + ) return _return_or_raise( backend, config, weight_key, activation_key, activation_format ) else: # If the user is not explicit about the backend, try each. - for backend in FLASHINFER_NVFP4_MOE_BACKENDS: + fi_backends = [ + b + for b in FLASHINFER_NVFP4_MOE_BACKENDS + if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP + ] + for backend in fi_backends: for k_cls in backend_to_kernel_cls(backend): supported, reason = k_cls.is_supported_config( k_cls, @@ -439,6 +473,7 @@ def make_nvfp4_moe_quant_config( w2_scale_2: torch.Tensor, a13_scale: torch.Tensor, a2_scale: torch.Tensor, + swiglu_limit: float | None = None, ) -> FusedMoEQuantConfig: if backend == NvFp4MoeBackend.MARLIN: return nvfp4_w4a16_moe_quant_config( @@ -455,6 +490,7 @@ def make_nvfp4_moe_quant_config( a2_gscale=a2_scale, w1_scale=w13_scale, w2_scale=w2_scale, + gemm1_clamp_limit=swiglu_limit, ) # Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas. @@ -478,6 +514,7 @@ def make_nvfp4_moe_quant_config( NvFp4MoeBackend.FLASHINFER_CUTEDSL, ) ), + gemm1_clamp_limit=swiglu_limit, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index e1df4484f1..9a051c038f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -258,6 +258,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): w2_scale_2=layer.w2_weight_scale_2, a13_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, + swiglu_limit=getattr(layer, "swiglu_limit", None), ) def apply_monolithic( diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 4a3b7619b9..67cdab4f22 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -1601,6 +1601,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): w2_scale_2=layer.w2_weight_scale_2, a13_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, + swiglu_limit=getattr(layer, "swiglu_limit", None), ) @property diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py index 2d844b04ac..85a78883fd 100644 --- a/vllm/models/deepseek_v4/quant_config.py +++ b/vllm/models/deepseek_v4/quant_config.py @@ -2,6 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Quantization config for DeepSeek V4.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + from vllm.config import get_current_vllm_config from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod @@ -14,6 +18,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( _DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8") +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptNvFp4Config, + ) + class DeepseekV4FP8Config(Fp8Config): """FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch. @@ -38,6 +47,8 @@ class DeepseekV4FP8Config(Fp8Config): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._resolved_expert_dtype: str | None = None + self._resolved_moe_quant_algo: str | None = None + self._nvfp4_config: ModelOptNvFp4Config | None = None # ``is_scale_e8m0`` is a property that resolves on first read, # by which time the current vllm_config has been set. @@ -70,6 +81,36 @@ class DeepseekV4FP8Config(Fp8Config): # checkpoints (Flash-Base) store them as float32. return self.expert_dtype == "fp4" + def _resolve_moe_overrides(self) -> None: + if self._resolved_moe_quant_algo is not None: + return + try: + hf_config = get_current_vllm_config().model_config.hf_config + except Exception: + return + quant_cfg = getattr(hf_config, "quantization_config", None) or {} + algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None + self._resolved_moe_quant_algo = algo or "" + + @property + def moe_quant_algo(self) -> str: + self._resolve_moe_overrides() + return self._resolved_moe_quant_algo or "" + + def _get_nvfp4_config(self) -> ModelOptNvFp4Config: + if self._nvfp4_config is None: + from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptNvFp4Config, + ) + + self._nvfp4_config = ModelOptNvFp4Config( + is_checkpoint_nvfp4_serialized=True, + kv_cache_quant_algo=None, + exclude_modules=[], + group_size=16, + ) + return self._nvfp4_config + @classmethod def get_name(cls) -> QuantizationMethods: return "deepseek_v4_fp8" @@ -97,10 +138,21 @@ class DeepseekV4FP8Config(Fp8Config): ): return UnquantizedFusedMoEMethod(layer.moe_config) if self.expert_dtype == "fp4": + if self.moe_quant_algo == "NVFP4": + from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptNvFp4FusedMoE, + ) + + return ModelOptNvFp4FusedMoE( + quant_config=self._get_nvfp4_config(), + moe_config=layer.moe_config, + ) return Mxfp4MoEMethod(layer.moe_config) # expert_dtype == "fp8": fall through to Fp8Config which # returns Fp8MoEMethod with block-wise float32 scales. return super().get_quant_method(layer, prefix) def is_mxfp4_quant(self, prefix, layer): - return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4" + if not isinstance(layer, FusedMoE) or self.expert_dtype != "fp4": + return False + return self.moe_quant_algo != "NVFP4"