[Kernel] Applies routed_scaling_factor internally (#47408)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -43,6 +43,7 @@ def vllm_topk_sigmoid(
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool = False,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
ops.topk_sigmoid(
|
||||
topk_weights,
|
||||
@@ -51,6 +52,7 @@ def vllm_topk_sigmoid(
|
||||
gating_output,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
@@ -216,9 +218,8 @@ def fused_topk_bias(
|
||||
gating_output,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
return topk_weights, topk_ids
|
||||
elif scoring_func == "sqrtsoftplus":
|
||||
return vllm_topk_softplus_sqrt(
|
||||
|
||||
@@ -280,6 +280,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
parallel_config: ParallelConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -372,9 +373,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
topk_group=getattr(config, "topk_group", 1),
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=getattr(config, "scoring_func", "softmax"),
|
||||
# aiter applies routed_scaling_factor internally
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scale_to_output=not self.is_rocm_aiter_moe_enabled,
|
||||
apply_routed_scale_to_output=apply_routed_scale_to_output,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
@@ -1246,6 +1246,8 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
parallel_config=parallel_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
# aiter applies routed_scaling_factor internally
|
||||
apply_routed_scale_to_output=not rocm_aiter_ops.is_fused_moe_enabled(),
|
||||
)
|
||||
else:
|
||||
self.mlp = DeepseekV2MLP(
|
||||
|
||||
Reference in New Issue
Block a user