[Bugfix] Fix DSV4 Base model swiglu limit issue in FP8 path (#42855)
Signed-off-by: Chengze Fan <chengze@meta.com> Signed-off-by: Chengze Fan <fancz2002@gmail.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
@@ -97,6 +97,13 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
self.max_capture_size = (
|
||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
self.gemm1_clamp_limit: torch.Tensor | None = None
|
||||
if quant_config.gemm1_clamp_limit is not None:
|
||||
self.gemm1_clamp_limit = torch.tensor(
|
||||
[quant_config.gemm1_clamp_limit] * self.num_experts,
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if quant_config.weight_quant_dtype == "mxfp4":
|
||||
# This value is used specifically for gpt-oss,
|
||||
@@ -107,9 +114,12 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
self.gemm1_beta = torch.tensor(
|
||||
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
|
||||
)
|
||||
self.gemm1_clamp_limit = torch.tensor(
|
||||
[7.0] * self.num_experts, dtype=torch.float32, device=self.device
|
||||
)
|
||||
if self.gemm1_clamp_limit is None:
|
||||
self.gemm1_clamp_limit = torch.tensor(
|
||||
[7.0] * self.num_experts,
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
if quant_config.quant_dtype == "mxfp8":
|
||||
self.fake_input_scale = torch.ones(
|
||||
self.num_experts,
|
||||
@@ -277,7 +287,9 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
fc2_expert_biases = None
|
||||
swiglu_alpha = None
|
||||
swiglu_beta = None
|
||||
swiglu_limit = None
|
||||
swiglu_limit = (
|
||||
self.gemm1_clamp_limit if activation == MoEActivation.SILU else None
|
||||
)
|
||||
use_mxfp8_act_scaling = False
|
||||
use_w4_group_scaling = False
|
||||
# Select quantization metadata based on FP8 format/path
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
moe_kernel_quantize_input,
|
||||
normalize_batched_scales_shape,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -805,6 +806,16 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
# Let PrepareAndFinalize::finalize() decide the impl.
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def activation(
|
||||
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
|
||||
return
|
||||
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
moe_kernel_quantize_input,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
is_deep_gemm_e8m0_used,
|
||||
@@ -129,6 +130,16 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def activation(
|
||||
self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
|
||||
return
|
||||
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
|
||||
Reference in New Issue
Block a user