[Mypy] Fix mypy typing for P models (#54169)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
5093e4844a
commit
784cac7c42
@@ -218,10 +218,14 @@ class Plamo3AttentionMixer(nn.Module):
|
||||
|
||||
q_shape = q.shape
|
||||
q = q.reshape(q_shape[:-1] + (q_shape[-1] // self.head_dim, self.head_dim))
|
||||
q = self.q_norm.forward_native(q).reshape(q_shape)
|
||||
normalized_q = self.q_norm.forward_native(q)
|
||||
assert isinstance(normalized_q, torch.Tensor)
|
||||
q = normalized_q.reshape(q_shape)
|
||||
k_shape = k.shape
|
||||
k = k.reshape(k_shape[:-1] + (k_shape[-1] // self.head_dim, self.head_dim))
|
||||
k = self.k_norm.forward_native(k).reshape(k_shape)
|
||||
normalized_k = self.k_norm.forward_native(k)
|
||||
assert isinstance(normalized_k, torch.Tensor)
|
||||
k = normalized_k.reshape(k_shape)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
Reference in New Issue
Block a user