Reduce memory usage for granite_speech. (#42933)
Signed-off-by: Yihuki <wangbovbvb@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0c942c69d6
commit
5c1aec3dc0
@@ -389,10 +389,8 @@ class GraniteSpeechConformerAttention(nn.Module):
|
||||
# shaw's relative positional embedding
|
||||
dist = attention_dists.to(hidden_states.device)
|
||||
rel_pos_emb = self.rel_pos_emb(dist)
|
||||
rel_pos_emb_expanded = rel_pos_emb.view([1, 1, 1] + list(rel_pos_emb.shape))
|
||||
pos_attn = (
|
||||
torch.sum(query_states.unsqueeze(-2) * rel_pos_emb_expanded, dim=-1)
|
||||
* self.scale
|
||||
torch.einsum("bnhid,ijd->bnhij", query_states, rel_pos_emb) * self.scale
|
||||
)
|
||||
|
||||
if remainder > 0:
|
||||
|
||||
Reference in New Issue
Block a user