[DSV4] Add silu clamp limit to shared expert (#40950)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -151,6 +151,46 @@ class SiluAndMul(CustomOp):
|
||||
return self.forward_cuda(x)
|
||||
|
||||
|
||||
@CustomOp.register("silu_and_mul_with_clamp")
|
||||
class SiluAndMulWithClamp(CustomOp):
|
||||
"""SwiGLU activation with input clamping (used by some MoE shared experts).
|
||||
|
||||
Computes:
|
||||
gate = clamp(x[..., :d], max=swiglu_limit)
|
||||
up = clamp(x[..., d:], min=-swiglu_limit, max=swiglu_limit)
|
||||
out = silu(gate) * up
|
||||
where d = x.shape[-1] // 2.
|
||||
|
||||
Shapes:
|
||||
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
|
||||
return: (num_tokens, d) or (batch_size, seq_len, d)
|
||||
"""
|
||||
|
||||
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.swiglu_limit = float(swiglu_limit)
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
self.op = torch.ops._C.silu_and_mul_with_clamp
|
||||
elif current_platform.is_cpu():
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
gate = torch.clamp(x[..., :d], max=self.swiglu_limit)
|
||||
up = torch.clamp(x[..., d:], min=-self.swiglu_limit, max=self.swiglu_limit)
|
||||
return F.silu(gate) * up
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
self.op(out, x, self.swiglu_limit)
|
||||
return out
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_cuda(x)
|
||||
|
||||
|
||||
# --8<-- [start:mul_and_silu]
|
||||
@CustomOp.register("mul_and_silu")
|
||||
class MulAndSilu(CustomOp):
|
||||
|
||||
@@ -45,7 +45,7 @@ def _gelu_and_mul(
|
||||
# Uses static methods or standalone functions to avoid instantiating CustomOp
|
||||
# classes, which would call get_current_vllm_config() before config is set.
|
||||
_CPU_MOE_ACT_FN: dict[MoEActivation, Callable[[torch.Tensor], torch.Tensor]] = {
|
||||
MoEActivation.SILU: SiluAndMul.forward_native,
|
||||
MoEActivation.SILU: lambda x: SiluAndMul(compile_native=False).forward_native(x),
|
||||
MoEActivation.SWIGLUOAI: _swigluoai_forward_native,
|
||||
MoEActivation.GELU: _gelu_and_mul,
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.deepseek_v4_attention import (
|
||||
DeepseekV4Indexer,
|
||||
DeepseekV4MLAModules,
|
||||
@@ -34,7 +35,10 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
QuantizationMethods,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -46,7 +50,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MLP
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -63,6 +66,57 @@ from .utils import (
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
swiglu_limit: float | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
is_sequence_parallel: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# If is_sequence_parallel, the input and output tensors are sharded
|
||||
# across the ranks within the tp_group. In this case the weights are
|
||||
# replicated and no collective ops are needed.
|
||||
# Otherwise we use standard TP with an allreduce at the end.
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
if swiglu_limit is not None:
|
||||
self.act_fn = SiluAndMulWithClamp(swiglu_limit)
|
||||
else:
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class DeepseekV4FP8Config(Fp8Config):
|
||||
"""FP8 config that routes MoE layers to MXFP4 quantization.
|
||||
|
||||
@@ -672,10 +726,11 @@ class DeepseekV4MoE(nn.Module):
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
|
||||
self.shared_experts = DeepseekV2MLP(
|
||||
self.shared_experts = DeepseekV4MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=self.use_mega_moe,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
|
||||
Reference in New Issue
Block a user