[Attention] use diff kv backend for mimo v2 flash (#40045)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu
2026-04-24 11:25:55 +00:00
committed by GitHub
parent 2ec18f5df4
commit e8ee2a78db
8 changed files with 112 additions and 24 deletions
@@ -597,6 +597,7 @@ class Attention(nn.Module, AttentionLayerBase):
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
+14 -8
View File
@@ -46,6 +46,9 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.flash_attn_diffkv import (
FlashAttentionDiffKVBackend,
)
from .interfaces import MixtureOfExperts, SupportsPP
from .utils import (
@@ -287,6 +290,15 @@ class MiMoV2Attention(nn.Module):
)
sliding_window = sliding_window_size if sliding_window_size > -1 else None
# Use DiffKV backend when V has a different head dim than K
if self.v_head_dim != self.head_dim:
FlashAttentionDiffKVBackend.set_head_size_v(self.v_head_dim)
attn_backend = FlashAttentionDiffKVBackend
logger.info_once("Using FlashAttentionDiffKVBackend for attention.")
else:
attn_backend = None
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -298,6 +310,8 @@ class MiMoV2Attention(nn.Module):
attn_type=AttentionType.DECODER,
prefix=f"{prefix}.attn",
sinks=self.attention_sink_bias,
attn_backend=attn_backend,
head_size_v=self.v_head_dim,
)
def forward(
@@ -313,16 +327,8 @@ class MiMoV2Attention(nn.Module):
if self.v_scale is not None:
v = v * self.v_scale
v = v.view(-1, self.num_kv_heads, self.v_head_dim)
v = torch.nn.functional.pad(v, [0, self.head_dim - self.v_head_dim], value=0)
v = v.view(-1, self.num_kv_heads * self.head_dim)
attn_output = self.attn(q, k, v)
attn_output = attn_output.view(-1, self.num_heads, self.head_dim)[
..., : self.v_head_dim
].reshape(-1, self.num_heads * self.v_head_dim)
output, _ = self.o_proj(attn_output)
return output