[Bugfix][MiMo] Apply vision attention sinks in the window attention path (#49815)
Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
@@ -193,7 +193,7 @@ class MiMoVisionAttention(nn.Module):
|
||||
# Rotary embeddings applied separately to Q and K
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
|
||||
|
||||
# Sink attention weights (loaded but not used in vLLM flash_attn)
|
||||
# Per-head sink logits, applied in the window attention path.
|
||||
# The checkpoint stores these only for non-full-attention blocks
|
||||
self.use_sink = use_sink
|
||||
if use_sink:
|
||||
@@ -214,21 +214,38 @@ class MiMoVisionAttention(nn.Module):
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Window attention via flash_attn_varlen_func with window_size."""
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
"""Window attention with the per-head sink applied to key 0.
|
||||
|
||||
The reference adds ``sinks[h]`` to the logit of each sequence's first
|
||||
key, which the Triton prefill kernel supports directly, so the softmax
|
||||
normalizes over the biased scores in one pass.
|
||||
"""
|
||||
from vllm.v1.attention.ops.triton_prefill_attention import (
|
||||
context_attention_fwd,
|
||||
)
|
||||
|
||||
w = self.visual_token_window_size
|
||||
output = flash_attn_varlen_func(
|
||||
output = torch.empty_like(q)
|
||||
head_start = self.tp_rank * self.num_heads_per_partition
|
||||
sinks = (
|
||||
self.sinks[head_start : head_start + self.num_heads_per_partition]
|
||||
if self.sinks is not None
|
||||
else None
|
||||
)
|
||||
context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_seqlen,
|
||||
max_seqlen_k=max_seqlen,
|
||||
output,
|
||||
b_start_loc=cu_seqlens[:-1],
|
||||
b_seq_len=cu_seqlens[1:] - cu_seqlens[:-1],
|
||||
max_input_len=max_seqlen,
|
||||
is_causal=False,
|
||||
softmax_scale=self.scale,
|
||||
causal=False,
|
||||
window_size=[w, w],
|
||||
sliding_window_q=w,
|
||||
sliding_window_k=w,
|
||||
sinks=sinks,
|
||||
sinks_bias_key0=True,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
Reference in New Issue
Block a user