[Bugfix] fix swiglu limit issue for humming backend + deepseek v4 (#42541)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
parent
67f58ce23f
commit
6859ca7615
@@ -33,7 +33,10 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceDelegate,
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
@@ -422,6 +425,18 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
return supported, None if supported else reason
|
||||
|
||||
def apply_activation(
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
) -> None:
|
||||
swiglu_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and swiglu_limit is not None:
|
||||
swiglu_limit_func(output=output, input=input, swiglu_limit=swiglu_limit)
|
||||
else:
|
||||
self.activation(activation=activation, input=input, output=output)
|
||||
|
||||
|
||||
class HummingIndexedExperts(HummingExpertsBase):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -514,7 +529,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
**moe_kwargs1,
|
||||
)
|
||||
|
||||
self.activation(
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
@@ -604,7 +619,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
self.activation(
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
@@ -691,7 +706,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
self.activation(
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
|
||||
@@ -1567,7 +1567,12 @@ def make_mxfp4_moe_quant_config(
|
||||
)
|
||||
|
||||
assert isinstance(layer, FusedMoE)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
else:
|
||||
return ocp_mx_moe_quant_config(
|
||||
quant_dtype="mxfp4",
|
||||
|
||||
@@ -164,7 +164,12 @@ def prepare_humming_moe_layer(layer: RoutedExperts, quant_config: dict):
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
|
||||
def get_humming_moe_quant_config(layer: RoutedExperts):
|
||||
def get_humming_moe_quant_config(
|
||||
layer: RoutedExperts,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
):
|
||||
input_schema = layer.input_schemas["w13"]
|
||||
weight_schema = layer.weight_schemas["w13"]
|
||||
|
||||
@@ -211,4 +216,7 @@ def get_humming_moe_quant_config(layer: RoutedExperts):
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
_w2=w2_quant_desc,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user