K3: remove the add operation for megamoe path (#51146)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -242,9 +242,23 @@ class KimiRoutedOutputTransform(nn.Module):
|
|||||||
self.norm = norm
|
self.norm = norm
|
||||||
self.up_proj = up_proj
|
self.up_proj = up_proj
|
||||||
|
|
||||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
def forward(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
residual: torch.Tensor | None = None,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Project the routed latent back to the hidden dim.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
hidden_states: Routed expert output in latent space.
|
||||||
|
residual: Optional tensor of the up-projection's output shape to
|
||||||
|
accumulate into. It is consumed in the GEMM's beta-add
|
||||||
|
epilogue, so adding it costs no extra kernel.
|
||||||
|
"""
|
||||||
if self.norm is not None:
|
if self.norm is not None:
|
||||||
hidden_states = self.norm(hidden_states)
|
hidden_states = self.norm(hidden_states)
|
||||||
|
if residual is not None:
|
||||||
|
return residual.addmm_(hidden_states, self.up_proj.weight.t())
|
||||||
hidden_states, _ = self.up_proj(hidden_states)
|
hidden_states, _ = self.up_proj(hidden_states)
|
||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
@@ -523,8 +537,8 @@ class KimiMoE(nn.Module):
|
|||||||
if self.num_shared_experts is not None:
|
if self.num_shared_experts is not None:
|
||||||
shared_intermediate_size = moe_intermediate_size * self.num_shared_experts
|
shared_intermediate_size = moe_intermediate_size * self.num_shared_experts
|
||||||
self.shared_experts = KimiMLP(
|
self.shared_experts = KimiMLP(
|
||||||
hidden_size=config.hidden_size, # 7618
|
hidden_size=config.hidden_size,
|
||||||
intermediate_size=shared_intermediate_size, # 3072*2
|
intermediate_size=shared_intermediate_size,
|
||||||
hidden_act=config.hidden_act,
|
hidden_act=config.hidden_act,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
reduce_results=False,
|
reduce_results=False,
|
||||||
@@ -721,11 +735,16 @@ class KimiMoE(nn.Module):
|
|||||||
topk_ids,
|
topk_ids,
|
||||||
activation_clamp=None,
|
activation_clamp=None,
|
||||||
)
|
)
|
||||||
final_hidden_states = self.routed_output_transform(final_hidden_states)
|
# The shared output is folded into the up-projection GEMM's beta-add
|
||||||
if self.shared_experts is not None:
|
# epilogue, so combining the two branches costs no extra kernel.
|
||||||
final_hidden_states = final_hidden_states + self.shared_experts(
|
shared_output = (
|
||||||
hidden_states
|
self.shared_experts(hidden_states)
|
||||||
)
|
if self.shared_experts is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
final_hidden_states = self.routed_output_transform(
|
||||||
|
final_hidden_states, residual=shared_output
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
# Routed experts consume the down-projected latent; shared experts
|
# Routed experts consume the down-projected latent; shared experts
|
||||||
# (inside MoERunner) get the original hidden states via
|
# (inside MoERunner) get the original hidden states via
|
||||||
|
|||||||
Reference in New Issue
Block a user