From 5c1aec3dc0600cb816a0389e55ef1f1c17893380 Mon Sep 17 00:00:00 2001 From: Yihuki Date: Mon, 25 May 2026 14:12:57 +0800 Subject: [PATCH] Reduce memory usage for granite_speech. (#42933) Signed-off-by: Yihuki Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/granite_speech.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index d2a1cb5755..5b4959dc20 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -389,10 +389,8 @@ class GraniteSpeechConformerAttention(nn.Module): # shaw's relative positional embedding dist = attention_dists.to(hidden_states.device) rel_pos_emb = self.rel_pos_emb(dist) - rel_pos_emb_expanded = rel_pos_emb.view([1, 1, 1] + list(rel_pos_emb.shape)) pos_attn = ( - torch.sum(query_states.unsqueeze(-2) * rel_pos_emb_expanded, dim=-1) - * self.scale + torch.einsum("bnhid,ijd->bnhij", query_states, rel_pos_emb) * self.scale ) if remainder > 0: