[Bugfix][Gemma4] Keep image bidirectional attention within the sliding window (#47217)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-07-02 19:57:41 -07:00
committed by GitHub
co-authored by Luciano Martins
parent 41de1380c2
commit 979f5511d7
8 changed files with 89 additions and 9 deletions
@@ -216,6 +216,7 @@ class Attention(nn.Module, AttentionLayerBase):
prefix: str = "",
attn_type: str = AttentionType.DECODER,
kv_sharing_target_layer_name: str | None = None,
mm_prefix_clamp_sliding_window: bool = False,
attn_backend: type[AttentionBackend] | None = None,
head_size_v: int | None = None,
**extra_impl_args,
@@ -420,6 +421,9 @@ class Attention(nn.Module, AttentionLayerBase):
compilation_config.static_forward_context,
)
self.kv_sharing_target_layer_name = kv_sharing_target_layer_name
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window
# (read by the Triton backend impl). Default False for all other models.
self.mm_prefix_clamp_sliding_window = mm_prefix_clamp_sliding_window
# use a placeholder kv cache tensor during init, which will be replaced
# by bind_kv_cache
+7
View File
@@ -499,6 +499,13 @@ class Gemma4Attention(nn.Module):
logits_soft_cap=attn_logits_soft_cap,
per_layer_sliding_window=sliding_window,
kv_sharing_target_layer_name=kv_sharing_target_layer_name,
# Gemma4 vision bidi: on sliding layers the bidirectional image
# block must stay within the sliding window, matching HF's
# (causal OR blockwise) AND sliding_window. Without this the image
# span (~1100 soft tokens at max_soft_tokens=1120) exceeds the 1024
# window; the runner keeps the full range and the kernel bounds it
# per-query here.
mm_prefix_clamp_sliding_window=self.is_sliding,
prefix=f"{prefix}.attn",
)
+7
View File
@@ -978,6 +978,13 @@ class Gemma4ForConditionalGeneration(
SupportsLoRA,
SupportsEagle3,
):
# Gemma4 clamps mm_prefix bidirectional ranges to the sliding window
# in-kernel (HF's (causal OR blockwise) AND sliding_window). The model
# runner reads this to keep image bidirectional ranges that exceed the
# window instead of dropping them (which would make image attention
# causal-only for images larger than the sliding window).
mm_prefix_clamp_sliding_window: bool = True
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
+32 -4
View File
@@ -892,7 +892,22 @@ class FlashAttentionImpl(AttentionImpl):
and self.vllm_flash_attn_version == 4
):
max_ranges = mm_prefix_ranges.shape[1]
mm_mask_mod = _make_mm_prefix_mask_mod(max_ranges)
# Gemma4: clamp the bidirectional block to the sliding
# window (HF (causal OR blockwise) AND sliding_window on
# local layers). Other mm-prefix models pass sliding_window=0
# -> unclamped, behavior unchanged. sliding_window_size[0]+1
# is the window (self.sliding_window = (sw-1, 0) on sliding
# layers); causal mm_prefix never hits the symmetric path.
mm_clamp_sw = 0
if (
getattr(layer, "mm_prefix_clamp_sliding_window", False)
and sliding_window_size is not None
and sliding_window_size[0] >= 0
):
mm_clamp_sw = sliding_window_size[0] + 1
mm_mask_mod = _make_mm_prefix_mask_mod(
max_ranges, sliding_window=mm_clamp_sw
)
mm_aux = [mm_prefix_ranges]
# R-SWA: use CuTE-DSL mask_mod on FA4 for exact token-level
@@ -1189,13 +1204,23 @@ class FlashAttentionImpl(AttentionImpl):
return output
def _make_mm_prefix_mask_mod(max_ranges: int):
def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
"""Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix).
Returns a @cute.jit callable that evaluates:
keep = (kv_idx <= q_idx) OR
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end])
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end]
[AND (q_idx - kv_idx) < sliding_window])
for each mm_prefix range stored in aux_tensors[0].
When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is
clamped to the sliding window, matching HF's
(causal OR blockwise) AND sliding_window (== sliding_window_overlay
kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0
(default) leaves the block unclamped, so other mm-prefix models are
unchanged. q_idx/kv_idx are local offsets, but their difference is
frame-invariant and image bidirectional attention only matters during
prefill (where local == absolute).
"""
import cutlass
import cutlass.cute as cute
@@ -1223,7 +1248,10 @@ def _make_mm_prefix_mask_mod(max_ranges: int):
valid = r_start < r_end
q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
keep = keep | (q_in & k_in)
mm = q_in & k_in
if sliding_window > 0:
mm = mm & ((q_idx - kv_idx) < sliding_window)
keep = keep | mm
return keep
mm_prefix_mask_mod.use_fast_sampling = True
@@ -676,6 +676,9 @@ class TritonAttentionImpl(AttentionImpl):
v_scale_cache=v_scale_cache,
chunk_lookback=self.chunk_lookback,
use_td=self.use_td,
mm_prefix_clamp_sliding_window=getattr(
layer, "mm_prefix_clamp_sliding_window", False
),
)
return output
@@ -283,6 +283,7 @@ def compute_kv_seq_mask(
per_seq_causal_ptr=None,
CHUNK_LOOKBACK: tl.constexpr = -1,
CHUNK_SIZE: tl.constexpr = -1,
MM_PREFIX_CLAMP_SW: tl.constexpr = False,
):
"""Build the KV mask for one tile.
@@ -333,7 +334,14 @@ def compute_kv_seq_mask(
seq_mask = seq_mask & sw_left
# PrefixLM: extend mask with bidirectional ranges for multimodal tokens.
# Applied AFTER sliding window so mm_prefix ranges override SW restriction.
# Default (MM_PREFIX_CLAMP_SW=False): applied AFTER sliding window so
# mm_prefix ranges override the SW restriction -> (causal AND SW) OR mm.
# Gemma4 (MM_PREFIX_CLAMP_SW=True): the bidirectional image block must stay
# within the sliding window, matching HF's (causal OR blockwise) AND
# sliding_window. We AND each range with the SW past-bound
# (query_abs_pos - seq_offset) < SLIDING_WINDOW (== HF sliding_window_overlay
# kv > q - sw; future kv passes trivially). Inert for full-attention layers
# (SLIDING_WINDOW <= 0).
if USE_MM_PREFIX:
for i in range(MAX_MM_RANGES):
range_start = tl.load(
@@ -351,7 +359,10 @@ def compute_kv_seq_mask(
& (seq_offset[None, :] <= range_end)
& is_valid
)
seq_mask |= q_in_range & k_in_range
mm_mask = q_in_range & k_in_range
if MM_PREFIX_CLAMP_SW and SLIDING_WINDOW > 0:
mm_mask = mm_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW)
seq_mask |= mm_mask
return seq_mask
@@ -275,6 +275,10 @@ def kernel_unified_attention(
USE_TD: tl.constexpr = False,
USE_TD_QO: tl.constexpr = False,
Q_IS_FP8: tl.constexpr = False,
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window
# instead of letting them override it. Default False preserves the
# original (causal AND SW) OR mm_prefix behavior for all other models.
MM_PREFIX_CLAMP_SW: tl.constexpr = False,
):
USE_PER_TOKEN_HEAD_SCALES: tl.constexpr = KV_QUANT_MODE >= 2
USE_FP8_Q_DESCALE: tl.constexpr = KV_QUANT_MODE == 1 and Q_IS_FP8
@@ -509,6 +513,7 @@ def kernel_unified_attention(
per_seq_causal_ptr,
CHUNK_LOOKBACK,
CHUNK_SIZE,
MM_PREFIX_CLAMP_SW,
)
# S : (BLOCK_M, TILE_SIZE)
@@ -819,6 +824,9 @@ def unified_attention(
# The non-TD branch is dead-code-eliminated at Triton compile time so
# disabling this flag costs nothing.
use_td: bool = False,
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window.
# Default False keeps the original behavior for every other model.
mm_prefix_clamp_sliding_window: bool = False,
):
# Resolve causal: bool or per-seq tensor.
use_per_seq_causal = isinstance(causal, torch.Tensor)
@@ -1130,6 +1138,7 @@ def unified_attention(
CHUNK_SIZE=chunk_size,
USE_TD=use_td,
USE_TD_QO=use_td_qo,
MM_PREFIX_CLAMP_SW=mm_prefix_clamp_sliding_window,
**launch_kwargs,
)
+14 -3
View File
@@ -2330,13 +2330,20 @@ class GPUModelRunner(
# Compute mm_prefix bidirectional ranges before building
# attention metadata so builders handle them during build().
# Ranges exceeding sliding_window are skipped to prevent
# early tokens from attending across the entire image span.
# By default, ranges exceeding sliding_window are skipped to prevent
# early tokens from attending across the entire image span. Models that
# clamp mm_prefix to the sliding window *in-kernel* (e.g. Gemma4, which
# needs HF's (causal OR blockwise) AND sliding_window on sliding layers)
# opt out of the skip so the bidirectional range survives for images
# larger than the window; the kernel then bounds it per-query.
req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None
if self.is_mm_prefix_lm:
req_doc_ranges = {}
hf_text_config = self.model_config.hf_text_config
_bidi_sw = getattr(hf_text_config, "sliding_window", None)
_clamps_in_kernel = getattr(
self.model, "mm_prefix_clamp_sliding_window", False
)
for req_id in self.input_batch.req_ids:
image_doc_ranges = []
req_state = self.requests[req_id]
@@ -2346,7 +2353,11 @@ class GPUModelRunner(
pos_info = mm_feature.mm_position
img_doc_range = pos_info.extract_embeds_range()
for r in img_doc_range:
if _bidi_sw is not None and (r[1] - r[0] + 1) > _bidi_sw:
if (
not _clamps_in_kernel
and _bidi_sw is not None
and (r[1] - r[0] + 1) > _bidi_sw
):
continue
image_doc_ranges.append(r)
req_idx = self.input_batch.req_id_to_index[req_id]