From 34b560b725b401f33477464163244b4744d1c40f Mon Sep 17 00:00:00 2001 From: Luciano Martins <22145370+lucianommartins@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:46:40 -0300 Subject: [PATCH] [Bugfix][Gemma4] Fix FA4 mm_prefix mask: add sliding window and absolute q_idx (#47332) Signed-off-by: Luciano Martins Co-authored-by: Luciano Martins --- vllm/v1/attention/backends/flash_attn.py | 136 +++++++++++++++-------- 1 file changed, 88 insertions(+), 48 deletions(-) diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index a7cc6fb08e..db5b0dda36 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -892,21 +892,28 @@ class FlashAttentionImpl(AttentionImpl): and self.vllm_flash_attn_version == 4 ): max_ranges = mm_prefix_ranges.shape[1] - # Gemma4: clamp the bidirectional block to the sliding - # window (HF (causal OR blockwise) AND sliding_window on - # local layers). Other mm-prefix models pass sliding_window=0 - # -> unclamped, behavior unchanged. sliding_window_size[0]+1 - # is the window (self.sliding_window = (sw-1, 0) on sliding - # layers); causal mm_prefix never hits the symmetric path. + # Sliding window value in Triton convention + # (1 + window_size[0]). Global-attention layers + # store (-1, -1) → sw stays None / 0. + sw_val = ( + 1 + sliding_window_size[0] + if sliding_window_size is not None + and sliding_window_size[0] >= 0 + else None + ) + # Gemma4: also clamp the bidirectional block to the + # sliding window when the layer opts in + # (mm_prefix_clamp_sliding_window flag from PR #47217). mm_clamp_sw = 0 if ( getattr(layer, "mm_prefix_clamp_sliding_window", False) - and sliding_window_size is not None - and sliding_window_size[0] >= 0 + and sw_val is not None ): - mm_clamp_sw = sliding_window_size[0] + 1 + mm_clamp_sw = sw_val mm_mask_mod = _make_mm_prefix_mask_mod( - max_ranges, sliding_window=mm_clamp_sw + max_ranges, + sliding_window=mm_clamp_sw, + sliding_window_left=sw_val, ) mm_aux = [mm_prefix_ranges] @@ -1204,23 +1211,26 @@ class FlashAttentionImpl(AttentionImpl): return output -def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0): - """Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix). +def _make_mm_prefix_mask_mod( + max_ranges: int, + sliding_window: int = 0, + sliding_window_left: int | None = None, +): + """Build a CuTE-DSL mask_mod implementing + ``(causal AND sliding_window) OR mm_prefix``. - Returns a @cute.jit callable that evaluates: - keep = (kv_idx <= q_idx) OR - (q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end] - [AND (q_idx - kv_idx) < sliding_window]) - for each mm_prefix range stored in aux_tensors[0]. + The FA4 kernel passes *local* ``q_idx`` (0-based within the current + prefill chunk) while ``kv_idx`` is absolute (0-based over the full + KV cache). We recover the absolute Q position via + ``q_abs = q_idx + seqlen_k - seqlen_q`` (the context-length offset) + so that causal, sliding-window, and mm_prefix range comparisons all + use consistent absolute positions. This matches the Triton + reference path (``compute_kv_seq_mask``). - When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is - clamped to the sliding window, matching HF's - (causal OR blockwise) AND sliding_window (== sliding_window_overlay - kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0 - (default) leaves the block unclamped, so other mm-prefix models are - unchanged. q_idx/kv_idx are local offsets, but their difference is - frame-invariant and image bidirectional attention only matters during - prefill (where local == absolute). + ``sliding_window_left`` enforces the sliding window on the causal + term (None = full causal, no window). ``sliding_window`` clamps the + bidirectional block to the window (0 = unclamped; >0 = Gemma4 local + layers via ``mm_prefix_clamp_sliding_window``). """ import cutlass import cutlass.cute as cute @@ -1230,29 +1240,59 @@ def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0): scalar_to_ssa, ) - @cute.jit - def mm_prefix_mask_mod( - batch_idx: cute.TensorSSA, - head_idx: cute.TensorSSA, - q_idx: cute.TensorSSA, - kv_idx: cute.TensorSSA, - seqlen_info, - aux_tensors, - ): - keep = kv_idx <= q_idx - ranges = aux_tensors[0] - b = batch_idx[0] - for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined] - r_start = scalar_to_ssa(ranges[b, i, 0], Int32) - r_end = scalar_to_ssa(ranges[b, i, 1], Int32) - valid = r_start < r_end - q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid - k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid - mm = q_in & k_in - if sliding_window > 0: - mm = mm & ((q_idx - kv_idx) < sliding_window) - keep = keep | mm - return keep + if sliding_window_left is not None: + + @cute.jit + def mm_prefix_mask_mod( + batch_idx: cute.TensorSSA, + head_idx: cute.TensorSSA, + q_idx: cute.TensorSSA, + kv_idx: cute.TensorSSA, + seqlen_info, + aux_tensors, + ): + ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32) + q_abs = q_idx + ctx_off + sw = scalar_to_ssa(Int32(sliding_window_left), Int32) + keep = (kv_idx <= q_abs) & ((q_abs - kv_idx) < sw) + ranges = aux_tensors[0] + b = batch_idx[0] + for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined] + r_start = scalar_to_ssa(ranges[b, i, 0], Int32) + r_end = scalar_to_ssa(ranges[b, i, 1], Int32) + valid = r_start < r_end + q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid + k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid + mm = q_in & k_in + if sliding_window > 0: + mm = mm & ((q_abs - kv_idx) < sw) + keep = keep | mm + return keep + + else: + + @cute.jit + def mm_prefix_mask_mod( + batch_idx: cute.TensorSSA, + head_idx: cute.TensorSSA, + q_idx: cute.TensorSSA, + kv_idx: cute.TensorSSA, + seqlen_info, + aux_tensors, + ): + ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32) + q_abs = q_idx + ctx_off + keep = kv_idx <= q_abs + ranges = aux_tensors[0] + b = batch_idx[0] + for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined] + r_start = scalar_to_ssa(ranges[b, i, 0], Int32) + r_end = scalar_to_ssa(ranges[b, i, 1], Int32) + valid = r_start < r_end + q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid + k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid + keep = keep | (q_in & k_in) + return keep mm_prefix_mask_mod.use_fast_sampling = True return mm_prefix_mask_mod