[Attention] use diff kv backend for mimo v2 flash (#40045)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -597,6 +597,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
|
||||
@@ -46,6 +46,9 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.flash_attn_diffkv import (
|
||||
FlashAttentionDiffKVBackend,
|
||||
)
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsPP
|
||||
from .utils import (
|
||||
@@ -287,6 +290,15 @@ class MiMoV2Attention(nn.Module):
|
||||
)
|
||||
|
||||
sliding_window = sliding_window_size if sliding_window_size > -1 else None
|
||||
|
||||
# Use DiffKV backend when V has a different head dim than K
|
||||
if self.v_head_dim != self.head_dim:
|
||||
FlashAttentionDiffKVBackend.set_head_size_v(self.v_head_dim)
|
||||
attn_backend = FlashAttentionDiffKVBackend
|
||||
logger.info_once("Using FlashAttentionDiffKVBackend for attention.")
|
||||
else:
|
||||
attn_backend = None
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -298,6 +310,8 @@ class MiMoV2Attention(nn.Module):
|
||||
attn_type=AttentionType.DECODER,
|
||||
prefix=f"{prefix}.attn",
|
||||
sinks=self.attention_sink_bias,
|
||||
attn_backend=attn_backend,
|
||||
head_size_v=self.v_head_dim,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -313,16 +327,8 @@ class MiMoV2Attention(nn.Module):
|
||||
if self.v_scale is not None:
|
||||
v = v * self.v_scale
|
||||
|
||||
v = v.view(-1, self.num_kv_heads, self.v_head_dim)
|
||||
v = torch.nn.functional.pad(v, [0, self.head_dim - self.v_head_dim], value=0)
|
||||
v = v.view(-1, self.num_kv_heads * self.head_dim)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
attn_output = attn_output.view(-1, self.num_heads, self.head_dim)[
|
||||
..., : self.v_head_dim
|
||||
].reshape(-1, self.num_heads * self.v_head_dim)
|
||||
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
Reference in New Issue
Block a user