[Bugfix][MiMo] Apply vision attention sinks in the window attention path (#49815)

Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
Almog Tavor
2026-08-10 22:36:42 +00:00
committed by GitHub
parent 98a4144a41
commit c3cac8c63d
3 changed files with 132 additions and 12 deletions
+27 -10
View File
@@ -193,7 +193,7 @@ class MiMoVisionAttention(nn.Module):
# Rotary embeddings applied separately to Q and K
self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
# Sink attention weights (loaded but not used in vLLM flash_attn)
# Per-head sink logits, applied in the window attention path.
# The checkpoint stores these only for non-full-attention blocks
self.use_sink = use_sink
if use_sink:
@@ -214,21 +214,38 @@ class MiMoVisionAttention(nn.Module):
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
) -> torch.Tensor:
"""Window attention via flash_attn_varlen_func with window_size."""
from vllm.vllm_flash_attn import flash_attn_varlen_func
"""Window attention with the per-head sink applied to key 0.
The reference adds ``sinks[h]`` to the logit of each sequence's first
key, which the Triton prefill kernel supports directly, so the softmax
normalizes over the biased scores in one pass.
"""
from vllm.v1.attention.ops.triton_prefill_attention import (
context_attention_fwd,
)
w = self.visual_token_window_size
output = flash_attn_varlen_func(
output = torch.empty_like(q)
head_start = self.tp_rank * self.num_heads_per_partition
sinks = (
self.sinks[head_start : head_start + self.num_heads_per_partition]
if self.sinks is not None
else None
)
context_attention_fwd(
q,
k,
v,
cu_seqlens_q=cu_seqlens,
cu_seqlens_k=cu_seqlens,
max_seqlen_q=max_seqlen,
max_seqlen_k=max_seqlen,
output,
b_start_loc=cu_seqlens[:-1],
b_seq_len=cu_seqlens[1:] - cu_seqlens[:-1],
max_input_len=max_seqlen,
is_causal=False,
softmax_scale=self.scale,
causal=False,
window_size=[w, w],
sliding_window_q=w,
sliding_window_k=w,
sinks=sinks,
sinks_bias_key0=True,
)
return output