[Perf] Fuse CohereASR relative attention score accumulation (#55190)

Signed-off-by: levius <2114377220@qq.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Sun
2026-09-18 06:18:41 +00:00
committed by GitHub
co-authored by Codex
parent 39e33db7f3
commit 42a33039eb
2 changed files with 69 additions and 5 deletions
+57 -1
View File
@@ -4,9 +4,65 @@
from types import SimpleNamespace
import pytest
import torch
from vllm.config import SpeechToTextConfig
from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration
from vllm.model_executor.models.cohere_asr import (
CohereAsrForConditionalGeneration,
RelPositionMultiHeadAttention,
)
class _ReferenceRelPositionMultiHeadAttention(RelPositionMultiHeadAttention):
"""Relative attention implementation before score accumulation is fused."""
def forward(self, query, key, value, mask, pos_emb=None):
q, k, v = self.forward_qkv(query, key, value)
q = q.transpose(1, 2)
assert pos_emb is not None
n_batch_pos = pos_emb.size(0)
p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
p = p.transpose(1, 2)
q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2)
q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2)
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
matrix_bd = self.rel_shift(matrix_bd)
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)]
scores = (matrix_ac + matrix_bd) / self.s_d_k
return self.forward_attention(v, scores, mask)
@pytest.mark.parametrize(("query_len", "key_len"), [(2, 2), (3, 5)])
def test_relative_attention_fuses_score_accumulation(monkeypatch, query_len, key_len):
torch.manual_seed(7)
batch, heads, model_dim = 2, 4, 32
optimized = RelPositionMultiHeadAttention(heads, model_dim, None, None)
reference = _ReferenceRelPositionMultiHeadAttention(heads, model_dim, None, None)
reference.load_state_dict(optimized.state_dict())
query = torch.randn(batch, query_len, model_dim)
key = torch.randn(batch, key_len, model_dim)
value = torch.randn(batch, key_len, model_dim)
pos_emb = torch.randn(1, query_len + key_len - 1, model_dim)
expected = reference(query, key, value, None, pos_emb)
original_baddbmm = torch.baddbmm
calls = 0
def tracked_baddbmm(*args, **kwargs):
nonlocal calls
calls += 1
return original_baddbmm(*args, **kwargs)
monkeypatch.setattr(torch, "baddbmm", tracked_baddbmm)
actual = optimized(query, key, value, None, pos_emb)
torch.testing.assert_close(actual, expected)
assert calls == 1
@pytest.mark.parametrize(
+12 -4
View File
@@ -1206,10 +1206,18 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
matrix_bd = self.rel_shift(matrix_bd)
# drops extra elements in the matrix_bd to match the matrix_ac's size
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)]
scores = (matrix_ac + matrix_bd) / self.s_d_k # (batch, head, time1, time2)
# drops extra elements in matrix_bd to match the key sequence length
matrix_bd = matrix_bd[:, :, :, : k.size(-2)]
batch, heads, query_len, key_len = matrix_bd.shape
scale = 1.0 / self.s_d_k
scores = torch.baddbmm(
matrix_bd.flatten(0, 1),
q_with_bias_u.flatten(0, 1),
k.transpose(-2, -1).flatten(0, 1),
beta=scale,
alpha=scale,
).view(batch, heads, query_len, key_len)
return self.forward_attention(v, scores, mask)