[Bugfix][Gemma4] Fix FA4 mm_prefix mask: add sliding window and absolute q_idx (#47332)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-07-04 17:46:40 -07:00
committed by GitHub
co-authored by Luciano Martins
parent 91b5647300
commit 34b560b725
+88 -48
View File
@@ -892,21 +892,28 @@ class FlashAttentionImpl(AttentionImpl):
and self.vllm_flash_attn_version == 4
):
max_ranges = mm_prefix_ranges.shape[1]
# Gemma4: clamp the bidirectional block to the sliding
# window (HF (causal OR blockwise) AND sliding_window on
# local layers). Other mm-prefix models pass sliding_window=0
# -> unclamped, behavior unchanged. sliding_window_size[0]+1
# is the window (self.sliding_window = (sw-1, 0) on sliding
# layers); causal mm_prefix never hits the symmetric path.
# Sliding window value in Triton convention
# (1 + window_size[0]). Global-attention layers
# store (-1, -1) → sw stays None / 0.
sw_val = (
1 + sliding_window_size[0]
if sliding_window_size is not None
and sliding_window_size[0] >= 0
else None
)
# Gemma4: also clamp the bidirectional block to the
# sliding window when the layer opts in
# (mm_prefix_clamp_sliding_window flag from PR #47217).
mm_clamp_sw = 0
if (
getattr(layer, "mm_prefix_clamp_sliding_window", False)
and sliding_window_size is not None
and sliding_window_size[0] >= 0
and sw_val is not None
):
mm_clamp_sw = sliding_window_size[0] + 1
mm_clamp_sw = sw_val
mm_mask_mod = _make_mm_prefix_mask_mod(
max_ranges, sliding_window=mm_clamp_sw
max_ranges,
sliding_window=mm_clamp_sw,
sliding_window_left=sw_val,
)
mm_aux = [mm_prefix_ranges]
@@ -1204,23 +1211,26 @@ class FlashAttentionImpl(AttentionImpl):
return output
def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
"""Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix).
def _make_mm_prefix_mask_mod(
max_ranges: int,
sliding_window: int = 0,
sliding_window_left: int | None = None,
):
"""Build a CuTE-DSL mask_mod implementing
``(causal AND sliding_window) OR mm_prefix``.
Returns a @cute.jit callable that evaluates:
keep = (kv_idx <= q_idx) OR
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end]
[AND (q_idx - kv_idx) < sliding_window])
for each mm_prefix range stored in aux_tensors[0].
The FA4 kernel passes *local* ``q_idx`` (0-based within the current
prefill chunk) while ``kv_idx`` is absolute (0-based over the full
KV cache). We recover the absolute Q position via
``q_abs = q_idx + seqlen_k - seqlen_q`` (the context-length offset)
so that causal, sliding-window, and mm_prefix range comparisons all
use consistent absolute positions. This matches the Triton
reference path (``compute_kv_seq_mask``).
When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is
clamped to the sliding window, matching HF's
(causal OR blockwise) AND sliding_window (== sliding_window_overlay
kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0
(default) leaves the block unclamped, so other mm-prefix models are
unchanged. q_idx/kv_idx are local offsets, but their difference is
frame-invariant and image bidirectional attention only matters during
prefill (where local == absolute).
``sliding_window_left`` enforces the sliding window on the causal
term (None = full causal, no window). ``sliding_window`` clamps the
bidirectional block to the window (0 = unclamped; >0 = Gemma4 local
layers via ``mm_prefix_clamp_sliding_window``).
"""
import cutlass
import cutlass.cute as cute
@@ -1230,29 +1240,59 @@ def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
scalar_to_ssa,
)
@cute.jit
def mm_prefix_mask_mod(
batch_idx: cute.TensorSSA,
head_idx: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors,
):
keep = kv_idx <= q_idx
ranges = aux_tensors[0]
b = batch_idx[0]
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
valid = r_start < r_end
q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
mm = q_in & k_in
if sliding_window > 0:
mm = mm & ((q_idx - kv_idx) < sliding_window)
keep = keep | mm
return keep
if sliding_window_left is not None:
@cute.jit
def mm_prefix_mask_mod(
batch_idx: cute.TensorSSA,
head_idx: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors,
):
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
q_abs = q_idx + ctx_off
sw = scalar_to_ssa(Int32(sliding_window_left), Int32)
keep = (kv_idx <= q_abs) & ((q_abs - kv_idx) < sw)
ranges = aux_tensors[0]
b = batch_idx[0]
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
valid = r_start < r_end
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
mm = q_in & k_in
if sliding_window > 0:
mm = mm & ((q_abs - kv_idx) < sw)
keep = keep | mm
return keep
else:
@cute.jit
def mm_prefix_mask_mod(
batch_idx: cute.TensorSSA,
head_idx: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors,
):
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
q_abs = q_idx + ctx_off
keep = kv_idx <= q_abs
ranges = aux_tensors[0]
b = batch_idx[0]
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
valid = r_start < r_end
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
keep = keep | (q_in & k_in)
return keep
mm_prefix_mask_mod.use_fast_sampling = True
return mm_prefix_mask_mod