[MoE Refactor] Move the shared/fused expert output sum into MoERunnerBase (#35949)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -359,7 +359,6 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
top_k=self.top_k,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
reduce_results=False,
|
||||
renormalize=self.norm_expert_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
@@ -388,24 +387,11 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
self.gate.weight.float(),
|
||||
).to(hidden_states.dtype)
|
||||
|
||||
final_hidden = self.experts(
|
||||
expert_output = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
shared_output, expert_output = final_hidden
|
||||
else:
|
||||
shared_output, expert_output = None, final_hidden
|
||||
|
||||
if shared_output is not None:
|
||||
expert_output = expert_output + shared_output
|
||||
|
||||
if self.tp_size > 1:
|
||||
expert_output = self.experts.maybe_all_reduce_tensor_model_parallel(
|
||||
expert_output
|
||||
)
|
||||
|
||||
return expert_output.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user