[Attention] add triton diff-kv backend for mimo (#41797)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu
2026-06-11 11:36:31 -04:00
committed by GitHub
parent 4085ff7cb4
commit f81daf8880
8 changed files with 1041 additions and 9 deletions
+21 -7
View File
@@ -47,9 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.flash_attn_diffkv import (
FlashAttentionDiffKVBackend,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import MixtureOfExperts, SupportsPP
from .utils import (
@@ -292,11 +290,27 @@ class MiMoV2Attention(nn.Module):
sliding_window = sliding_window_size if sliding_window_size > -1 else None
# Use DiffKV backend when V has a different head dim than K
# Use DiffKV backend when V has a different head dim than K.
# Auto-pick FA-DiffKV when FA3/4 is usable on this device, else fall
# back to TRITON_ATTN_DIFFKV. Users can force a choice via
# `--attention-backend <FLASH_ATTN_DIFFKV|TRITON_ATTN_DIFFKV>`.
if self.v_head_dim != self.head_dim:
FlashAttentionDiffKVBackend.set_head_size_v(self.v_head_dim)
attn_backend = FlashAttentionDiffKVBackend
logger.info_once("Using FlashAttentionDiffKVBackend for attention.")
requested = get_current_vllm_config().attention_config.backend
if requested is not None and requested.name.endswith("_DIFFKV"):
backend_enum = requested
else:
fa_backend = AttentionBackendEnum.FLASH_ATTN_DIFFKV.get_class()
if fa_backend.is_supported_on_current_device(
head_size=self.head_dim,
head_size_v=self.v_head_dim,
has_sinks=self.attention_sink_bias is not None,
):
backend_enum = AttentionBackendEnum.FLASH_ATTN_DIFFKV
else:
backend_enum = AttentionBackendEnum.TRITON_ATTN_DIFFKV
attn_backend = backend_enum.get_class()
attn_backend.set_head_size_v(self.v_head_dim)
logger.info_once("Using %s for attention.", attn_backend.get_name())
else:
attn_backend = None