[Perf] Optimize MLA attention _v_up_proj bmm by removing additional copy (#42561)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -989,19 +989,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True, YQ=out
|
||||
)
|
||||
else:
|
||||
# Convert from (B, N * V) to (N, B, V)
|
||||
out = out.transpose(0, 1)
|
||||
|
||||
# Multiply (N, B, L) x (N, L, V) -> (N, B, V)
|
||||
torch.bmm(x, self.W_UV, out=out) # Reuse "out" to make it "hot"
|
||||
|
||||
# Convert from (N, B, V) to (B, N * V)
|
||||
out_new = out.transpose(0, 1).reshape(-1, self.num_heads * self.v_head_dim)
|
||||
|
||||
# Adjust output buffer shape back to the original (B, N * V)
|
||||
N, B, V = out.shape
|
||||
out.resize_((B, N * V))
|
||||
out.copy_(out_new) # Copy result
|
||||
# Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V)
|
||||
torch.bmm(x, self.W_UV, out=out.transpose(0, 1))
|
||||
|
||||
|
||||
def unified_mla_kv_cache_update(
|
||||
|
||||
Reference in New Issue
Block a user