Add NVFP4 MOE support for Deepseek V4. (#42209)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -805,6 +805,7 @@ def nvfp4_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and nvp4 weights.
|
||||
@@ -823,6 +824,7 @@ def nvfp4_moe_quant_config(
|
||||
per_out_ch_quant=False,
|
||||
block_shape=None,
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -1271,6 +1273,11 @@ class FusedMoEConfig:
|
||||
is_act_and_mul: bool = True
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
# SwiGLU clamp limit. When set, backends that do not implement the clamp
|
||||
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
|
||||
# cannot silently select one and drop the clamp.
|
||||
swiglu_limit: float | None = None
|
||||
|
||||
# This flag is used to disable the inplace optimization
|
||||
# in MoE kernels. If this flag is True then the kernel
|
||||
# should not be using inplace. If the flag is false, the
|
||||
|
||||
@@ -66,6 +66,17 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
else:
|
||||
self.g1_scale_c = self.quant_config.a2_gscale.clone()
|
||||
|
||||
if moe_config.is_act_and_mul and quant_config.gemm1_clamp_limit is not None:
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.gemm1_clamp_limit = torch.full(
|
||||
(self.local_num_experts,),
|
||||
quant_config.gemm1_clamp_limit,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
|
||||
@@ -84,6 +95,20 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
)
|
||||
self.g1_scale_c = layer.g1_scale_c
|
||||
|
||||
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
|
||||
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
|
||||
# directly. g1_alphas is set once here in process_weights_after_loading
|
||||
# (via the in-place mul above) and never changes again, so this is a
|
||||
# static, per-expert constant. Register on the layer so EPLB
|
||||
# rearranges it alongside the other expert tensors.
|
||||
if self.gemm1_clamp_limit is not None:
|
||||
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_clamp_limit",
|
||||
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
|
||||
)
|
||||
self.gemm1_clamp_limit = layer.gemm1_clamp_limit
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
"""Supports only Blackwell-family GPUs."""
|
||||
@@ -201,6 +226,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
# Pack topk ids and weights into format expected by the kernel.
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
# Invoke kernel.
|
||||
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
|
||||
@@ -215,12 +241,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
gemm1_bias=None,
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
|
||||
gemm2_bias=None,
|
||||
output1_scale_scalar=self.g1_scale_c,
|
||||
output1_scale_gate_scalar=self.quant_config.g1_alphas,
|
||||
output1_scale_gate_scalar=output1_scale_gate_scalar,
|
||||
output2_scale_scalar=self.quant_config.g2_alphas,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
@@ -313,6 +339,8 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
if e_score_correction_bias is not None:
|
||||
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
|
||||
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
@@ -328,12 +356,12 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
gemm1_bias=None,
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
|
||||
gemm2_bias=None,
|
||||
output1_scale_scalar=self.g1_scale_c,
|
||||
output1_scale_gate_scalar=self.quant_config.g1_alphas,
|
||||
output1_scale_gate_scalar=output1_scale_gate_scalar,
|
||||
output2_scale_scalar=self.quant_config.g2_alphas,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
|
||||
@@ -335,6 +335,7 @@ class FusedMoE(PluggableLayer):
|
||||
activation=self.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=self.routing_method_type,
|
||||
swiglu_limit=swiglu_limit,
|
||||
# TODO: in_dtype == out_dtype?
|
||||
disable_inplace=disable_inplace() or shared_experts is not None,
|
||||
)
|
||||
|
||||
@@ -181,6 +181,15 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
]
|
||||
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
}
|
||||
|
||||
if config.swiglu_limit is not None:
|
||||
AVAILABLE_BACKENDS = [
|
||||
b for b in AVAILABLE_BACKENDS if b in NVFP4_BACKENDS_WITH_CLAMP
|
||||
]
|
||||
|
||||
use_batched = config.moe_parallel_config.use_batched_activation_format
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
@@ -234,6 +243,16 @@ def select_nvfp4_moe_backend(
|
||||
and requested_backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL
|
||||
):
|
||||
requested_backend = NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED
|
||||
if (
|
||||
config.swiglu_limit is not None
|
||||
and requested_backend not in NVFP4_BACKENDS_WITH_CLAMP
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"explicitly requested moe_backend={runner_backend!r} does "
|
||||
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm' or "
|
||||
f"'flashinfer_cutlass' instead."
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
@@ -242,17 +261,32 @@ def select_nvfp4_moe_backend(
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
|
||||
# If the user rejects FlashInfer remove those backends.
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(b)
|
||||
if b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(b)
|
||||
|
||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
||||
# If user is explicit about backend, validate it.
|
||||
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
|
||||
if (
|
||||
config.swiglu_limit is not None
|
||||
and backend not in NVFP4_BACKENDS_WITH_CLAMP
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
|
||||
f"({backend.value}) does not apply the SwiGLU clamp."
|
||||
)
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
else:
|
||||
# If the user is not explicit about the backend, try each.
|
||||
for backend in FLASHINFER_NVFP4_MOE_BACKENDS:
|
||||
fi_backends = [
|
||||
b
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS
|
||||
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
|
||||
]
|
||||
for backend in fi_backends:
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls,
|
||||
@@ -439,6 +473,7 @@ def make_nvfp4_moe_quant_config(
|
||||
w2_scale_2: torch.Tensor,
|
||||
a13_scale: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
swiglu_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
if backend == NvFp4MoeBackend.MARLIN:
|
||||
return nvfp4_w4a16_moe_quant_config(
|
||||
@@ -455,6 +490,7 @@ def make_nvfp4_moe_quant_config(
|
||||
a2_gscale=a2_scale,
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
# Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas.
|
||||
@@ -478,6 +514,7 @@ def make_nvfp4_moe_quant_config(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
)
|
||||
),
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+1
@@ -258,6 +258,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -1601,6 +1601,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user