[Performance][Model] Avoid transient Inkling result allocations (performance, and OOM prevention on smaller memory configurations) (#49487)
Signed-off-by: Michael Gschwind <mgschwind@nvidia.com> Co-authored-by: Michael Gschwind <mgschwind@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Michael Gschwind
OpenAI Codex
parent
4080263bb2
commit
9a698f3255
@@ -58,6 +58,6 @@ class InklingDenseMLP(nn.Module):
|
||||
x = silu_and_mul_triton(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
if self.global_scale is not None:
|
||||
x = x * self.global_scale
|
||||
x.mul_(self.global_scale)
|
||||
# TP-partial output: the layer's reduce-scatter fallback consumes it.
|
||||
return x
|
||||
|
||||
@@ -528,7 +528,7 @@ class InklingMoE(nn.Module):
|
||||
)
|
||||
self._routed_sel = None
|
||||
|
||||
return out + sink_out
|
||||
return out.add_(sink_out)
|
||||
|
||||
# -- weight loading ----------------------------------------------------
|
||||
|
||||
|
||||
Reference in New Issue
Block a user