Fix PP in Gemma4 (#40786)
Signed-off-by: Rohit kumar Singh <rksingh@habana.ai> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6d7d4da99e
commit
5371d6fb40
@@ -1144,11 +1144,6 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
),
|
||||
"residual": torch.zeros(
|
||||
(batch_size, hidden_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
if ple_dim and ple_dim > 0:
|
||||
tensors["per_layer_inputs"] = torch.zeros(
|
||||
@@ -1312,13 +1307,12 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
per_layer_inputs = self.project_per_layer_inputs(
|
||||
hidden_states, per_layer_embeds
|
||||
)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
per_layer_inputs = intermediate_tensors.get("per_layer_inputs")
|
||||
|
||||
if per_layer_inputs is not None:
|
||||
per_layer_inputs = intermediate_tensors["per_layer_inputs"]
|
||||
residual = None
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
for layer_idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
@@ -1342,13 +1336,12 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
aux_hidden_states, layer_idx + 1, hidden_states, residual
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
"per_layer_inputs": per_layer_inputs,
|
||||
}
|
||||
)
|
||||
tensors: dict[str, torch.Tensor] = {
|
||||
"hidden_states": hidden_states,
|
||||
}
|
||||
if per_layer_inputs is not None:
|
||||
tensors["per_layer_inputs"] = per_layer_inputs
|
||||
return IntermediateTensors(tensors)
|
||||
# Gemma4 incorporates residual into hidden_states directly
|
||||
# Apply norm without residual fusion when possible.
|
||||
if residual is None:
|
||||
|
||||
Reference in New Issue
Block a user