[NVFP4 MoE/Deepseek V4] Marlin: wire SwiGLU clamp + allow it for clamped models on non-Blackwell (#45836)
Signed-off-by: Mike G <180722391+mikekg@users.noreply.github.com>
This commit is contained in:
@@ -861,6 +861,7 @@ def nvfp4_w4a16_moe_quant_config(
|
||||
g2_alphas: torch.Tensor,
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-but activations and nvp4 weights.
|
||||
@@ -872,6 +873,7 @@ def nvfp4_w4a16_moe_quant_config(
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
weight_dtype="nvfp4",
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -174,6 +174,7 @@ def select_nvfp4_moe_backend(
|
||||
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
}
|
||||
|
||||
if config.swiglu_limit is not None:
|
||||
@@ -423,6 +424,7 @@ def make_nvfp4_moe_quant_config(
|
||||
g2_alphas=w2_scale_2,
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif backend == NvFp4MoeBackend.EMULATION:
|
||||
return nvfp4_moe_quant_config(
|
||||
|
||||
Reference in New Issue
Block a user