[Perf] Accumulate Conformer attention scores with baddbmm (#55062)

Signed-off-by: levius <2114377220@qq.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Sun
2026-09-03 04:17:52 -07:00
committed by GitHub
co-authored by Codex
parent 758c79e1c5
commit 848ab131bc
2 changed files with 124 additions and 3 deletions
@@ -206,12 +206,22 @@ class RelPosMultiHeadAttention(EncoderMultiHeadAttention):
q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2)
q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2)
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
matrix_bd = self._rel_shift(matrix_bd)
attn_scores = matrix_ac + matrix_bd
attn_scores.mul_(self.scale)
batch_size, num_heads, query_len, key_len = matrix_bd.shape
attn_scores = torch.baddbmm(
matrix_bd.reshape(-1, query_len, key_len),
q_with_bias_u.reshape(-1, query_len, self.d_k),
k.transpose(-2, -1).reshape(-1, self.d_k, key_len),
beta=self.scale,
alpha=self.scale,
).view(
batch_size,
num_heads,
query_len,
key_len,
)
output, attn = self.forward_attention(attn_scores, v, mask=mask)
output = self.forward_output(output, residual, sz_b, len_q)