[Perf] Fuse CohereASR relative attention score accumulation (#55190)
Signed-off-by: levius <2114377220@qq.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -4,9 +4,65 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import SpeechToTextConfig
|
||||
from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration
|
||||
from vllm.model_executor.models.cohere_asr import (
|
||||
CohereAsrForConditionalGeneration,
|
||||
RelPositionMultiHeadAttention,
|
||||
)
|
||||
|
||||
|
||||
class _ReferenceRelPositionMultiHeadAttention(RelPositionMultiHeadAttention):
|
||||
"""Relative attention implementation before score accumulation is fused."""
|
||||
|
||||
def forward(self, query, key, value, mask, pos_emb=None):
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
q = q.transpose(1, 2)
|
||||
|
||||
assert pos_emb is not None
|
||||
n_batch_pos = pos_emb.size(0)
|
||||
p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
|
||||
p = p.transpose(1, 2)
|
||||
|
||||
q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2)
|
||||
q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2)
|
||||
|
||||
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
|
||||
matrix_bd = self.rel_shift(matrix_bd)
|
||||
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
|
||||
matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)]
|
||||
scores = (matrix_ac + matrix_bd) / self.s_d_k
|
||||
return self.forward_attention(v, scores, mask)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("query_len", "key_len"), [(2, 2), (3, 5)])
|
||||
def test_relative_attention_fuses_score_accumulation(monkeypatch, query_len, key_len):
|
||||
torch.manual_seed(7)
|
||||
batch, heads, model_dim = 2, 4, 32
|
||||
optimized = RelPositionMultiHeadAttention(heads, model_dim, None, None)
|
||||
reference = _ReferenceRelPositionMultiHeadAttention(heads, model_dim, None, None)
|
||||
reference.load_state_dict(optimized.state_dict())
|
||||
|
||||
query = torch.randn(batch, query_len, model_dim)
|
||||
key = torch.randn(batch, key_len, model_dim)
|
||||
value = torch.randn(batch, key_len, model_dim)
|
||||
pos_emb = torch.randn(1, query_len + key_len - 1, model_dim)
|
||||
expected = reference(query, key, value, None, pos_emb)
|
||||
|
||||
original_baddbmm = torch.baddbmm
|
||||
calls = 0
|
||||
|
||||
def tracked_baddbmm(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original_baddbmm(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(torch, "baddbmm", tracked_baddbmm)
|
||||
actual = optimized(query, key, value, None, pos_emb)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -1206,10 +1206,18 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
|
||||
matrix_bd = self.rel_shift(matrix_bd)
|
||||
|
||||
# drops extra elements in the matrix_bd to match the matrix_ac's size
|
||||
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
|
||||
matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)]
|
||||
scores = (matrix_ac + matrix_bd) / self.s_d_k # (batch, head, time1, time2)
|
||||
# drops extra elements in matrix_bd to match the key sequence length
|
||||
matrix_bd = matrix_bd[:, :, :, : k.size(-2)]
|
||||
batch, heads, query_len, key_len = matrix_bd.shape
|
||||
scale = 1.0 / self.s_d_k
|
||||
scores = torch.baddbmm(
|
||||
matrix_bd.flatten(0, 1),
|
||||
q_with_bias_u.flatten(0, 1),
|
||||
k.transpose(-2, -1).flatten(0, 1),
|
||||
beta=scale,
|
||||
alpha=scale,
|
||||
).view(batch, heads, query_len, key_len)
|
||||
|
||||
return self.forward_attention(v, scores, mask)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user