Add NVFP4 MOE support for Deepseek V4. (#42209)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-05-22 07:21:51 -07:00
committed by GitHub
parent a377631d21
commit fb21d8b4f9
9 changed files with 217 additions and 17 deletions
+7 -3
View File
@@ -150,9 +150,12 @@ steps:
- csrc/quantization/fp4/
- csrc/attention/mla/
- csrc/quantization/cutlass_w8a8/moe/
- vllm/model_executor/layers/fused_moe/cutlass_moe.py
- vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py
- vllm/model_executor/layers/fused_moe/flashinfer_a2a_prepare_finalize.py
- vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py
- vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
- vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
- vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
- vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py
- vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py
- vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
- vllm/v1/attention/backends/flashinfer.py
- vllm/v1/attention/backends/mla/cutlass_mla.py
@@ -185,6 +188,7 @@ steps:
- pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py
- pytest -v -s tests/kernels/moe/test_flashinfer.py
- pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
- pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
- pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
# e2e
- pytest -v -s tests/models/quantization/test_nvfp4.py
+76 -7
View File
@@ -22,6 +22,8 @@ from tests.kernels.quantization.nvfp4_utils import (
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.custom_op import CustomOp, op_registry
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -58,15 +60,45 @@ MNK_FACTORS = [
(64, 704, 4096),
]
_SWIGLU_LIMIT = 0.1
_LARGE_OUTPUT1_SCALE = 32768.0
_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp"
# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself
# valid as an ``activation=`` argument to ``torch_moe`` (which only looks
# up ``activation.custom_op_name`` in ``op_registry``), so no
# ``MoEActivation`` enum extension is needed.
if _CLAMP_OP_NAME not in op_registry:
@CustomOp.register(_CLAMP_OP_NAME)
class _SiluAndMulWithClampTest(SiluAndMulWithClamp):
custom_op_name = _CLAMP_OP_NAME
def __init__(self, *, compile_native: bool = True) -> None:
super().__init__(_SWIGLU_LIMIT, compile_native=compile_native)
SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
pytest.param(
MoEActivation.RELU2_NO_MUL,
MoEActivation.RELU2_NO_MUL,
None,
id="relu2_no_mul",
),
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, id="gelu"),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [128])
@pytest.mark.parametrize("topk", [8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize(
"activation",
[MoEActivation.SILU, MoEActivation.RELU2_NO_MUL, MoEActivation.GELU],
)
@pytest.mark.parametrize("activation,torch_activation,swiglu_limit", ACTIVATION_CASES)
@torch.inference_mode()
def test_trtllm_fp4_moe_no_graph(
m: int,
@@ -76,6 +108,8 @@ def test_trtllm_fp4_moe_no_graph(
topk: int,
dtype: torch.dtype,
activation: MoEActivation,
torch_activation: MoEActivation | type[SiluAndMulWithClamp],
swiglu_limit: float | None,
workspace_init,
):
# FlashInfer's trtllm_batched_gemm_runner has no precompiled tile
@@ -113,6 +147,15 @@ def test_trtllm_fp4_moe_no_graph(
# Match what oracle/nvfp4.py does for this backend.
is_scale_swizzled=False,
)
quant_config.gemm1_clamp_limit = swiglu_limit
if swiglu_limit is not None:
assert quant_config.g1_alphas is not None
assert quant_config.a2_gscale is not None
assert torch.all(quant_config.a2_gscale == 1)
# With a2_gscale == 1, g1_alphas is the TRTLLM
# output1_scale_gate_scalar. Make it large enough to catch
# clamp/output-scale coupling in the FlashInfer kernel wrapper.
quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
@@ -133,6 +176,23 @@ def test_trtllm_fp4_moe_no_graph(
max_num_tokens=next_power_of_2(m),
)
trtllm_inner = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
trtllm_inner.process_weights_after_loading(fake_layer)
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
@@ -140,7 +200,7 @@ def test_trtllm_fp4_moe_no_graph(
allow_new_interface=True,
use_monolithic=False,
),
TrtLlmNvFp4ExpertsModular(moe_config=moe_config, quant_config=quant_config),
trtllm_inner,
inplace=False,
)
@@ -195,7 +255,7 @@ def test_trtllm_fp4_moe_no_graph(
)
torch_output = torch_moe(
a_in_dtype, w1_d, w2_d, score, topk, activation=activation
a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation
)
torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1)
@@ -203,5 +263,14 @@ def test_trtllm_fp4_moe_no_graph(
if __name__ == "__main__":
test_trtllm_fp4_moe_no_graph(
64, 704, 4096, 128, 8, torch.bfloat16, MoEActivation.GELU, None
64,
704,
4096,
128,
8,
torch.bfloat16,
MoEActivation.GELU,
MoEActivation.GELU,
None,
None,
)
@@ -805,6 +805,7 @@ def nvfp4_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
is_scale_swizzled: bool = True,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for mxfp4 activations and nvp4 weights.
@@ -823,6 +824,7 @@ def nvfp4_moe_quant_config(
per_out_ch_quant=False,
block_shape=None,
is_scale_swizzled=is_scale_swizzled,
gemm1_clamp_limit=gemm1_clamp_limit,
)
@@ -1271,6 +1273,11 @@ class FusedMoEConfig:
is_act_and_mul: bool = True
is_lora_enabled: bool = False
# SwiGLU clamp limit. When set, backends that do not implement the clamp
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
# cannot silently select one and drop the clamp.
swiglu_limit: float | None = None
# This flag is used to disable the inplace optimization
# in MoE kernels. If this flag is True then the kernel
# should not be using inplace. If the flag is false, the
@@ -66,6 +66,17 @@ class TrtLlmNvFp4ExpertsBase:
else:
self.g1_scale_c = self.quant_config.a2_gscale.clone()
if moe_config.is_act_and_mul and quant_config.gemm1_clamp_limit is not None:
device = torch.accelerator.current_device_index()
self.gemm1_clamp_limit = torch.full(
(self.local_num_experts,),
quant_config.gemm1_clamp_limit,
dtype=torch.float32,
device=device,
)
else:
self.gemm1_clamp_limit = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
@@ -84,6 +95,20 @@ class TrtLlmNvFp4ExpertsBase:
)
self.g1_scale_c = layer.g1_scale_c
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
# directly. g1_alphas is set once here in process_weights_after_loading
# (via the in-place mul above) and never changes again, so this is a
# static, per-expert constant. Register on the layer so EPLB
# rearranges it alongside the other expert tensors.
if self.gemm1_clamp_limit is not None:
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
layer.register_parameter(
"gemm1_clamp_limit",
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
)
self.gemm1_clamp_limit = layer.gemm1_clamp_limit
@staticmethod
def _supports_current_device() -> bool:
"""Supports only Blackwell-family GPUs."""
@@ -201,6 +226,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
# Pack topk ids and weights into format expected by the kernel.
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
@@ -215,12 +241,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
gemm1_bias=None,
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
gemm2_bias=None,
output1_scale_scalar=self.g1_scale_c,
output1_scale_gate_scalar=self.quant_config.g1_alphas,
output1_scale_gate_scalar=output1_scale_gate_scalar,
output2_scale_scalar=self.quant_config.g2_alphas,
num_experts=global_num_experts,
top_k=self.topk,
@@ -313,6 +339,8 @@ class TrtLlmNvFp4ExpertsMonolithic(
if e_score_correction_bias is not None:
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
@@ -328,12 +356,12 @@ class TrtLlmNvFp4ExpertsMonolithic(
gemm1_bias=None,
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
gemm2_bias=None,
output1_scale_scalar=self.g1_scale_c,
output1_scale_gate_scalar=self.quant_config.g1_alphas,
output1_scale_gate_scalar=output1_scale_gate_scalar,
output2_scale_scalar=self.quant_config.g2_alphas,
num_experts=global_num_experts,
top_k=self.topk,
@@ -335,6 +335,7 @@ class FusedMoE(PluggableLayer):
activation=self.activation,
device=vllm_config.device_config.device,
routing_method=self.routing_method_type,
swiglu_limit=swiglu_limit,
# TODO: in_dtype == out_dtype?
disable_inplace=disable_inplace() or shared_experts is not None,
)
@@ -181,6 +181,15 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.EMULATION,
]
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.FLASHINFER_TRTLLM,
}
if config.swiglu_limit is not None:
AVAILABLE_BACKENDS = [
b for b in AVAILABLE_BACKENDS if b in NVFP4_BACKENDS_WITH_CLAMP
]
use_batched = config.moe_parallel_config.use_batched_activation_format
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
@@ -234,6 +243,16 @@ def select_nvfp4_moe_backend(
and requested_backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL
):
requested_backend = NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED
if (
config.swiglu_limit is not None
and requested_backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"explicitly requested moe_backend={runner_backend!r} does "
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm' or "
f"'flashinfer_cutlass' instead."
)
return _return_or_raise(
requested_backend, config, weight_key, activation_key, activation_format
)
@@ -242,17 +261,32 @@ def select_nvfp4_moe_backend(
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
# If the user rejects FlashInfer remove those backends.
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
if b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
if (
config.swiglu_limit is not None
and backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
f"({backend.value}) does not apply the SwiGLU clamp."
)
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try each.
for backend in FLASHINFER_NVFP4_MOE_BACKENDS:
fi_backends = [
b
for b in FLASHINFER_NVFP4_MOE_BACKENDS
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
]
for backend in fi_backends:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
@@ -439,6 +473,7 @@ def make_nvfp4_moe_quant_config(
w2_scale_2: torch.Tensor,
a13_scale: torch.Tensor,
a2_scale: torch.Tensor,
swiglu_limit: float | None = None,
) -> FusedMoEQuantConfig:
if backend == NvFp4MoeBackend.MARLIN:
return nvfp4_w4a16_moe_quant_config(
@@ -455,6 +490,7 @@ def make_nvfp4_moe_quant_config(
a2_gscale=a2_scale,
w1_scale=w13_scale,
w2_scale=w2_scale,
gemm1_clamp_limit=swiglu_limit,
)
# Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas.
@@ -478,6 +514,7 @@ def make_nvfp4_moe_quant_config(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
)
),
gemm1_clamp_limit=swiglu_limit,
)
@@ -258,6 +258,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
def apply_monolithic(
@@ -1601,6 +1601,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
@property
+53 -1
View File
@@ -2,6 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Quantization config for DeepSeek V4."""
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
@@ -14,6 +18,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
class DeepseekV4FP8Config(Fp8Config):
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
@@ -38,6 +47,8 @@ class DeepseekV4FP8Config(Fp8Config):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._resolved_expert_dtype: str | None = None
self._resolved_moe_quant_algo: str | None = None
self._nvfp4_config: ModelOptNvFp4Config | None = None
# ``is_scale_e8m0`` is a property that resolves on first read,
# by which time the current vllm_config has been set.
@@ -70,6 +81,36 @@ class DeepseekV4FP8Config(Fp8Config):
# checkpoints (Flash-Base) store them as float32.
return self.expert_dtype == "fp4"
def _resolve_moe_overrides(self) -> None:
if self._resolved_moe_quant_algo is not None:
return
try:
hf_config = get_current_vllm_config().model_config.hf_config
except Exception:
return
quant_cfg = getattr(hf_config, "quantization_config", None) or {}
algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None
self._resolved_moe_quant_algo = algo or ""
@property
def moe_quant_algo(self) -> str:
self._resolve_moe_overrides()
return self._resolved_moe_quant_algo or ""
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
if self._nvfp4_config is None:
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
self._nvfp4_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
group_size=16,
)
return self._nvfp4_config
@classmethod
def get_name(cls) -> QuantizationMethods:
return "deepseek_v4_fp8"
@@ -97,10 +138,21 @@ class DeepseekV4FP8Config(Fp8Config):
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.expert_dtype == "fp4":
if self.moe_quant_algo == "NVFP4":
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4FusedMoE,
)
return ModelOptNvFp4FusedMoE(
quant_config=self._get_nvfp4_config(),
moe_config=layer.moe_config,
)
return Mxfp4MoEMethod(layer.moe_config)
# expert_dtype == "fp8": fall through to Fp8Config which
# returns Fp8MoEMethod with block-wise float32 scales.
return super().get_quant_method(layer, prefix)
def is_mxfp4_quant(self, prefix, layer):
return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4"
if not isinstance(layer, FusedMoE) or self.expert_dtype != "fp4":
return False
return self.moe_quant_algo != "NVFP4"