diff --git a/tests/models/multimodal/test_cohere_asr.py b/tests/models/multimodal/test_cohere_asr.py index 85a9b59dd5..950542f900 100644 --- a/tests/models/multimodal/test_cohere_asr.py +++ b/tests/models/multimodal/test_cohere_asr.py @@ -4,9 +4,65 @@ from types import SimpleNamespace import pytest +import torch from vllm.config import SpeechToTextConfig -from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration +from vllm.model_executor.models.cohere_asr import ( + CohereAsrForConditionalGeneration, + RelPositionMultiHeadAttention, +) + + +class _ReferenceRelPositionMultiHeadAttention(RelPositionMultiHeadAttention): + """Relative attention implementation before score accumulation is fused.""" + + def forward(self, query, key, value, mask, pos_emb=None): + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) + + assert pos_emb is not None + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) + + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] + scores = (matrix_ac + matrix_bd) / self.s_d_k + return self.forward_attention(v, scores, mask) + + +@pytest.mark.parametrize(("query_len", "key_len"), [(2, 2), (3, 5)]) +def test_relative_attention_fuses_score_accumulation(monkeypatch, query_len, key_len): + torch.manual_seed(7) + batch, heads, model_dim = 2, 4, 32 + optimized = RelPositionMultiHeadAttention(heads, model_dim, None, None) + reference = _ReferenceRelPositionMultiHeadAttention(heads, model_dim, None, None) + reference.load_state_dict(optimized.state_dict()) + + query = torch.randn(batch, query_len, model_dim) + key = torch.randn(batch, key_len, model_dim) + value = torch.randn(batch, key_len, model_dim) + pos_emb = torch.randn(1, query_len + key_len - 1, model_dim) + expected = reference(query, key, value, None, pos_emb) + + original_baddbmm = torch.baddbmm + calls = 0 + + def tracked_baddbmm(*args, **kwargs): + nonlocal calls + calls += 1 + return original_baddbmm(*args, **kwargs) + + monkeypatch.setattr(torch, "baddbmm", tracked_baddbmm) + actual = optimized(query, key, value, None, pos_emb) + + torch.testing.assert_close(actual, expected) + assert calls == 1 @pytest.mark.parametrize( diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 844e8c5e0e..33b4aa7b0d 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -1206,10 +1206,18 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention): matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) matrix_bd = self.rel_shift(matrix_bd) - # drops extra elements in the matrix_bd to match the matrix_ac's size - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] - scores = (matrix_ac + matrix_bd) / self.s_d_k # (batch, head, time1, time2) + # drops extra elements in matrix_bd to match the key sequence length + matrix_bd = matrix_bd[:, :, :, : k.size(-2)] + batch, heads, query_len, key_len = matrix_bd.shape + scale = 1.0 / self.s_d_k + scores = torch.baddbmm( + matrix_bd.flatten(0, 1), + q_with_bias_u.flatten(0, 1), + k.transpose(-2, -1).flatten(0, 1), + beta=scale, + alpha=scale, + ).view(batch, heads, query_len, key_len) + return self.forward_attention(v, scores, mask)