[Bugfix] Fix DSV4 Base model swiglu limit issue in FP8 path (#42855)

Signed-off-by: Chengze Fan <chengze@meta.com>
Signed-off-by: Chengze Fan <fancz2002@gmail.com>
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
Chengze Fan
2026-05-21 19:43:01 -07:00
committed by GitHub
co-authored by Lu Fang
parent ba369b7eb5
commit 2998a047aa
3 changed files with 38 additions and 4 deletions
@@ -97,6 +97,13 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
self.max_capture_size = (
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
)
self.gemm1_clamp_limit: torch.Tensor | None = None
if quant_config.gemm1_clamp_limit is not None:
self.gemm1_clamp_limit = torch.tensor(
[quant_config.gemm1_clamp_limit] * self.num_experts,
dtype=torch.float32,
device=self.device,
)
if quant_config.weight_quant_dtype == "mxfp4":
# This value is used specifically for gpt-oss,
@@ -107,9 +114,12 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
self.gemm1_beta = torch.tensor(
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_clamp_limit = torch.tensor(
[7.0] * self.num_experts, dtype=torch.float32, device=self.device
)
if self.gemm1_clamp_limit is None:
self.gemm1_clamp_limit = torch.tensor(
[7.0] * self.num_experts,
dtype=torch.float32,
device=self.device,
)
if quant_config.quant_dtype == "mxfp8":
self.fake_input_scale = torch.ones(
self.num_experts,
@@ -277,7 +287,9 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
fc2_expert_biases = None
swiglu_alpha = None
swiglu_beta = None
swiglu_limit = None
swiglu_limit = (
self.gemm1_clamp_limit if activation == MoEActivation.SILU else None
)
use_mxfp8_act_scaling = False
use_w4_group_scaling = False
# Select quantization metadata based on FP8 format/path
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
moe_kernel_quantize_input,
normalize_batched_scales_shape,
swiglu_limit_func,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -805,6 +806,16 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
# Let PrepareAndFinalize::finalize() decide the impl.
return TopKWeightAndReduceDelegate()
def activation(
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
return
super().activation(activation, output, input)
def workspace_shapes(
self,
M: int,
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
moe_kernel_quantize_input,
swiglu_limit_func,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
is_deep_gemm_e8m0_used,
@@ -129,6 +130,16 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def activation(
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
) -> None:
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
return
super().activation(activation, output, input)
def workspace_shapes(
self,
M: int,