Add bias handling to CPUFusedMOE kernel (#26289)
Signed-off-by: Crefeda Rodrigues <crefeda.rodrigues@arm.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Crefeda Rodrigues <65665931+cfRod@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Sharif Inamdar <Sharif.Inamdar@arm.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
Isotr0py
Sharif Inamdar
Isotr0py
parent
b2ea5ba677
commit
c02058c222
@@ -276,6 +276,9 @@ class CPUFusedMOE:
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
has_w13_bias = hasattr(layer, "w13_bias")
|
||||
has_w2_bias = hasattr(layer, "w2_bias")
|
||||
|
||||
for i, num_tokens in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + num_tokens
|
||||
if num_tokens == 0:
|
||||
@@ -283,11 +286,15 @@ class CPUFusedMOE:
|
||||
tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
|
||||
|
||||
layer_w13_weight = layer.w13_weight[i]
|
||||
layer_w13_bias = layer.w13_bias[i] if has_w13_bias else None
|
||||
layer_w2_weight = layer.w2_weight[i]
|
||||
layer_w2_bias = layer.w2_bias[i] if has_w2_bias else None
|
||||
|
||||
gate_up = F.linear(tokens_for_this_expert, layer_w13_weight)
|
||||
gate_up = F.linear(
|
||||
tokens_for_this_expert, layer_w13_weight, bias=layer_w13_bias
|
||||
)
|
||||
gate_up = silu_and_mul(gate_up)
|
||||
expert_out = F.linear(gate_up, layer_w2_weight)
|
||||
expert_out = F.linear(gate_up, layer_w2_weight, bias=layer_w2_bias)
|
||||
outputs.append(expert_out)
|
||||
start_idx = end_idx
|
||||
|
||||
|
||||
Reference in New Issue
Block a user