Add NVFP4 MOE support for Deepseek V4. (#42209)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-05-22 07:21:51 -07:00
committed by GitHub
parent a377631d21
commit fb21d8b4f9
9 changed files with 217 additions and 17 deletions
@@ -805,6 +805,7 @@ def nvfp4_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
is_scale_swizzled: bool = True,
gemm1_clamp_limit: float | None = None,
) -> FusedMoEQuantConfig:
"""
Construct a quant config for mxfp4 activations and nvp4 weights.
@@ -823,6 +824,7 @@ def nvfp4_moe_quant_config(
per_out_ch_quant=False,
block_shape=None,
is_scale_swizzled=is_scale_swizzled,
gemm1_clamp_limit=gemm1_clamp_limit,
)
@@ -1271,6 +1273,11 @@ class FusedMoEConfig:
is_act_and_mul: bool = True
is_lora_enabled: bool = False
# SwiGLU clamp limit. When set, backends that do not implement the clamp
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
# cannot silently select one and drop the clamp.
swiglu_limit: float | None = None
# This flag is used to disable the inplace optimization
# in MoE kernels. If this flag is True then the kernel
# should not be using inplace. If the flag is false, the
@@ -66,6 +66,17 @@ class TrtLlmNvFp4ExpertsBase:
else:
self.g1_scale_c = self.quant_config.a2_gscale.clone()
if moe_config.is_act_and_mul and quant_config.gemm1_clamp_limit is not None:
device = torch.accelerator.current_device_index()
self.gemm1_clamp_limit = torch.full(
(self.local_num_experts,),
quant_config.gemm1_clamp_limit,
dtype=torch.float32,
device=device,
)
else:
self.gemm1_clamp_limit = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
@@ -84,6 +95,20 @@ class TrtLlmNvFp4ExpertsBase:
)
self.g1_scale_c = layer.g1_scale_c
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
# directly. g1_alphas is set once here in process_weights_after_loading
# (via the in-place mul above) and never changes again, so this is a
# static, per-expert constant. Register on the layer so EPLB
# rearranges it alongside the other expert tensors.
if self.gemm1_clamp_limit is not None:
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
layer.register_parameter(
"gemm1_clamp_limit",
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
)
self.gemm1_clamp_limit = layer.gemm1_clamp_limit
@staticmethod
def _supports_current_device() -> bool:
"""Supports only Blackwell-family GPUs."""
@@ -201,6 +226,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
# Pack topk ids and weights into format expected by the kernel.
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
@@ -215,12 +241,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
gemm1_bias=None,
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
gemm2_bias=None,
output1_scale_scalar=self.g1_scale_c,
output1_scale_gate_scalar=self.quant_config.g1_alphas,
output1_scale_gate_scalar=output1_scale_gate_scalar,
output2_scale_scalar=self.quant_config.g2_alphas,
num_experts=global_num_experts,
top_k=self.topk,
@@ -313,6 +339,8 @@ class TrtLlmNvFp4ExpertsMonolithic(
if e_score_correction_bias is not None:
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
@@ -328,12 +356,12 @@ class TrtLlmNvFp4ExpertsMonolithic(
gemm1_bias=None,
gemm1_alpha=None,
gemm1_beta=None,
gemm1_clamp_limit=None,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
gemm2_bias=None,
output1_scale_scalar=self.g1_scale_c,
output1_scale_gate_scalar=self.quant_config.g1_alphas,
output1_scale_gate_scalar=output1_scale_gate_scalar,
output2_scale_scalar=self.quant_config.g2_alphas,
num_experts=global_num_experts,
top_k=self.topk,
@@ -335,6 +335,7 @@ class FusedMoE(PluggableLayer):
activation=self.activation,
device=vllm_config.device_config.device,
routing_method=self.routing_method_type,
swiglu_limit=swiglu_limit,
# TODO: in_dtype == out_dtype?
disable_inplace=disable_inplace() or shared_experts is not None,
)
@@ -181,6 +181,15 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.EMULATION,
]
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.FLASHINFER_TRTLLM,
}
if config.swiglu_limit is not None:
AVAILABLE_BACKENDS = [
b for b in AVAILABLE_BACKENDS if b in NVFP4_BACKENDS_WITH_CLAMP
]
use_batched = config.moe_parallel_config.use_batched_activation_format
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
@@ -234,6 +243,16 @@ def select_nvfp4_moe_backend(
and requested_backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL
):
requested_backend = NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED
if (
config.swiglu_limit is not None
and requested_backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"explicitly requested moe_backend={runner_backend!r} does "
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm' or "
f"'flashinfer_cutlass' instead."
)
return _return_or_raise(
requested_backend, config, weight_key, activation_key, activation_format
)
@@ -242,17 +261,32 @@ def select_nvfp4_moe_backend(
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
# If the user rejects FlashInfer remove those backends.
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
if b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
if (
config.swiglu_limit is not None
and backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
f"({backend.value}) does not apply the SwiGLU clamp."
)
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try each.
for backend in FLASHINFER_NVFP4_MOE_BACKENDS:
fi_backends = [
b
for b in FLASHINFER_NVFP4_MOE_BACKENDS
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
]
for backend in fi_backends:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
@@ -439,6 +473,7 @@ def make_nvfp4_moe_quant_config(
w2_scale_2: torch.Tensor,
a13_scale: torch.Tensor,
a2_scale: torch.Tensor,
swiglu_limit: float | None = None,
) -> FusedMoEQuantConfig:
if backend == NvFp4MoeBackend.MARLIN:
return nvfp4_w4a16_moe_quant_config(
@@ -455,6 +490,7 @@ def make_nvfp4_moe_quant_config(
a2_gscale=a2_scale,
w1_scale=w13_scale,
w2_scale=w2_scale,
gemm1_clamp_limit=swiglu_limit,
)
# Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas.
@@ -478,6 +514,7 @@ def make_nvfp4_moe_quant_config(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
)
),
gemm1_clamp_limit=swiglu_limit,
)
@@ -258,6 +258,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
def apply_monolithic(
@@ -1601,6 +1601,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
@property