[Bugfix][Gemma4] Fix FA4 mm_prefix mask: add sliding window and absolute q_idx (#47332)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
91b5647300
commit
34b560b725
@@ -892,21 +892,28 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
and self.vllm_flash_attn_version == 4
|
||||
):
|
||||
max_ranges = mm_prefix_ranges.shape[1]
|
||||
# Gemma4: clamp the bidirectional block to the sliding
|
||||
# window (HF (causal OR blockwise) AND sliding_window on
|
||||
# local layers). Other mm-prefix models pass sliding_window=0
|
||||
# -> unclamped, behavior unchanged. sliding_window_size[0]+1
|
||||
# is the window (self.sliding_window = (sw-1, 0) on sliding
|
||||
# layers); causal mm_prefix never hits the symmetric path.
|
||||
# Sliding window value in Triton convention
|
||||
# (1 + window_size[0]). Global-attention layers
|
||||
# store (-1, -1) → sw stays None / 0.
|
||||
sw_val = (
|
||||
1 + sliding_window_size[0]
|
||||
if sliding_window_size is not None
|
||||
and sliding_window_size[0] >= 0
|
||||
else None
|
||||
)
|
||||
# Gemma4: also clamp the bidirectional block to the
|
||||
# sliding window when the layer opts in
|
||||
# (mm_prefix_clamp_sliding_window flag from PR #47217).
|
||||
mm_clamp_sw = 0
|
||||
if (
|
||||
getattr(layer, "mm_prefix_clamp_sliding_window", False)
|
||||
and sliding_window_size is not None
|
||||
and sliding_window_size[0] >= 0
|
||||
and sw_val is not None
|
||||
):
|
||||
mm_clamp_sw = sliding_window_size[0] + 1
|
||||
mm_clamp_sw = sw_val
|
||||
mm_mask_mod = _make_mm_prefix_mask_mod(
|
||||
max_ranges, sliding_window=mm_clamp_sw
|
||||
max_ranges,
|
||||
sliding_window=mm_clamp_sw,
|
||||
sliding_window_left=sw_val,
|
||||
)
|
||||
mm_aux = [mm_prefix_ranges]
|
||||
|
||||
@@ -1204,23 +1211,26 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
return output
|
||||
|
||||
|
||||
def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
|
||||
"""Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix).
|
||||
def _make_mm_prefix_mask_mod(
|
||||
max_ranges: int,
|
||||
sliding_window: int = 0,
|
||||
sliding_window_left: int | None = None,
|
||||
):
|
||||
"""Build a CuTE-DSL mask_mod implementing
|
||||
``(causal AND sliding_window) OR mm_prefix``.
|
||||
|
||||
Returns a @cute.jit callable that evaluates:
|
||||
keep = (kv_idx <= q_idx) OR
|
||||
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end]
|
||||
[AND (q_idx - kv_idx) < sliding_window])
|
||||
for each mm_prefix range stored in aux_tensors[0].
|
||||
The FA4 kernel passes *local* ``q_idx`` (0-based within the current
|
||||
prefill chunk) while ``kv_idx`` is absolute (0-based over the full
|
||||
KV cache). We recover the absolute Q position via
|
||||
``q_abs = q_idx + seqlen_k - seqlen_q`` (the context-length offset)
|
||||
so that causal, sliding-window, and mm_prefix range comparisons all
|
||||
use consistent absolute positions. This matches the Triton
|
||||
reference path (``compute_kv_seq_mask``).
|
||||
|
||||
When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is
|
||||
clamped to the sliding window, matching HF's
|
||||
(causal OR blockwise) AND sliding_window (== sliding_window_overlay
|
||||
kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0
|
||||
(default) leaves the block unclamped, so other mm-prefix models are
|
||||
unchanged. q_idx/kv_idx are local offsets, but their difference is
|
||||
frame-invariant and image bidirectional attention only matters during
|
||||
prefill (where local == absolute).
|
||||
``sliding_window_left`` enforces the sliding window on the causal
|
||||
term (None = full causal, no window). ``sliding_window`` clamps the
|
||||
bidirectional block to the window (0 = unclamped; >0 = Gemma4 local
|
||||
layers via ``mm_prefix_clamp_sliding_window``).
|
||||
"""
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
@@ -1230,29 +1240,59 @@ def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
|
||||
scalar_to_ssa,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def mm_prefix_mask_mod(
|
||||
batch_idx: cute.TensorSSA,
|
||||
head_idx: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors,
|
||||
):
|
||||
keep = kv_idx <= q_idx
|
||||
ranges = aux_tensors[0]
|
||||
b = batch_idx[0]
|
||||
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
|
||||
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
|
||||
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
|
||||
valid = r_start < r_end
|
||||
q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
mm = q_in & k_in
|
||||
if sliding_window > 0:
|
||||
mm = mm & ((q_idx - kv_idx) < sliding_window)
|
||||
keep = keep | mm
|
||||
return keep
|
||||
if sliding_window_left is not None:
|
||||
|
||||
@cute.jit
|
||||
def mm_prefix_mask_mod(
|
||||
batch_idx: cute.TensorSSA,
|
||||
head_idx: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors,
|
||||
):
|
||||
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
|
||||
q_abs = q_idx + ctx_off
|
||||
sw = scalar_to_ssa(Int32(sliding_window_left), Int32)
|
||||
keep = (kv_idx <= q_abs) & ((q_abs - kv_idx) < sw)
|
||||
ranges = aux_tensors[0]
|
||||
b = batch_idx[0]
|
||||
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
|
||||
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
|
||||
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
|
||||
valid = r_start < r_end
|
||||
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
mm = q_in & k_in
|
||||
if sliding_window > 0:
|
||||
mm = mm & ((q_abs - kv_idx) < sw)
|
||||
keep = keep | mm
|
||||
return keep
|
||||
|
||||
else:
|
||||
|
||||
@cute.jit
|
||||
def mm_prefix_mask_mod(
|
||||
batch_idx: cute.TensorSSA,
|
||||
head_idx: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors,
|
||||
):
|
||||
ctx_off = scalar_to_ssa(seqlen_info.seqlen_k - seqlen_info.seqlen_q, Int32)
|
||||
q_abs = q_idx + ctx_off
|
||||
keep = kv_idx <= q_abs
|
||||
ranges = aux_tensors[0]
|
||||
b = batch_idx[0]
|
||||
for i in cutlass.range_constexpr(max_ranges): # type: ignore[attr-defined]
|
||||
r_start = scalar_to_ssa(ranges[b, i, 0], Int32)
|
||||
r_end = scalar_to_ssa(ranges[b, i, 1], Int32)
|
||||
valid = r_start < r_end
|
||||
q_in = (q_abs >= r_start) & (q_abs <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
keep = keep | (q_in & k_in)
|
||||
return keep
|
||||
|
||||
mm_prefix_mask_mod.use_fast_sampling = True
|
||||
return mm_prefix_mask_mod
|
||||
|
||||
Reference in New Issue
Block a user