[Bugfix][Gemma4] Keep image bidirectional attention within the sliding window (#47217)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
41de1380c2
commit
979f5511d7
@@ -216,6 +216,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
kv_sharing_target_layer_name: str | None = None,
|
||||
mm_prefix_clamp_sliding_window: bool = False,
|
||||
attn_backend: type[AttentionBackend] | None = None,
|
||||
head_size_v: int | None = None,
|
||||
**extra_impl_args,
|
||||
@@ -420,6 +421,9 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context,
|
||||
)
|
||||
self.kv_sharing_target_layer_name = kv_sharing_target_layer_name
|
||||
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window
|
||||
# (read by the Triton backend impl). Default False for all other models.
|
||||
self.mm_prefix_clamp_sliding_window = mm_prefix_clamp_sliding_window
|
||||
|
||||
# use a placeholder kv cache tensor during init, which will be replaced
|
||||
# by bind_kv_cache
|
||||
|
||||
@@ -499,6 +499,13 @@ class Gemma4Attention(nn.Module):
|
||||
logits_soft_cap=attn_logits_soft_cap,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
kv_sharing_target_layer_name=kv_sharing_target_layer_name,
|
||||
# Gemma4 vision bidi: on sliding layers the bidirectional image
|
||||
# block must stay within the sliding window, matching HF's
|
||||
# (causal OR blockwise) AND sliding_window. Without this the image
|
||||
# span (~1100 soft tokens at max_soft_tokens=1120) exceeds the 1024
|
||||
# window; the runner keeps the full range and the kernel bounds it
|
||||
# per-query here.
|
||||
mm_prefix_clamp_sliding_window=self.is_sliding,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
|
||||
@@ -978,6 +978,13 @@ class Gemma4ForConditionalGeneration(
|
||||
SupportsLoRA,
|
||||
SupportsEagle3,
|
||||
):
|
||||
# Gemma4 clamps mm_prefix bidirectional ranges to the sliding window
|
||||
# in-kernel (HF's (causal OR blockwise) AND sliding_window). The model
|
||||
# runner reads this to keep image bidirectional ranges that exceed the
|
||||
# window instead of dropping them (which would make image attention
|
||||
# causal-only for images larger than the sliding window).
|
||||
mm_prefix_clamp_sliding_window: bool = True
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
|
||||
@@ -892,7 +892,22 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
and self.vllm_flash_attn_version == 4
|
||||
):
|
||||
max_ranges = mm_prefix_ranges.shape[1]
|
||||
mm_mask_mod = _make_mm_prefix_mask_mod(max_ranges)
|
||||
# Gemma4: clamp the bidirectional block to the sliding
|
||||
# window (HF (causal OR blockwise) AND sliding_window on
|
||||
# local layers). Other mm-prefix models pass sliding_window=0
|
||||
# -> unclamped, behavior unchanged. sliding_window_size[0]+1
|
||||
# is the window (self.sliding_window = (sw-1, 0) on sliding
|
||||
# layers); causal mm_prefix never hits the symmetric path.
|
||||
mm_clamp_sw = 0
|
||||
if (
|
||||
getattr(layer, "mm_prefix_clamp_sliding_window", False)
|
||||
and sliding_window_size is not None
|
||||
and sliding_window_size[0] >= 0
|
||||
):
|
||||
mm_clamp_sw = sliding_window_size[0] + 1
|
||||
mm_mask_mod = _make_mm_prefix_mask_mod(
|
||||
max_ranges, sliding_window=mm_clamp_sw
|
||||
)
|
||||
mm_aux = [mm_prefix_ranges]
|
||||
|
||||
# R-SWA: use CuTE-DSL mask_mod on FA4 for exact token-level
|
||||
@@ -1189,13 +1204,23 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
return output
|
||||
|
||||
|
||||
def _make_mm_prefix_mask_mod(max_ranges: int):
|
||||
def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0):
|
||||
"""Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix).
|
||||
|
||||
Returns a @cute.jit callable that evaluates:
|
||||
keep = (kv_idx <= q_idx) OR
|
||||
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end])
|
||||
(q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end]
|
||||
[AND (q_idx - kv_idx) < sliding_window])
|
||||
for each mm_prefix range stored in aux_tensors[0].
|
||||
|
||||
When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is
|
||||
clamped to the sliding window, matching HF's
|
||||
(causal OR blockwise) AND sliding_window (== sliding_window_overlay
|
||||
kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0
|
||||
(default) leaves the block unclamped, so other mm-prefix models are
|
||||
unchanged. q_idx/kv_idx are local offsets, but their difference is
|
||||
frame-invariant and image bidirectional attention only matters during
|
||||
prefill (where local == absolute).
|
||||
"""
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
@@ -1223,7 +1248,10 @@ def _make_mm_prefix_mask_mod(max_ranges: int):
|
||||
valid = r_start < r_end
|
||||
q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid
|
||||
k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid
|
||||
keep = keep | (q_in & k_in)
|
||||
mm = q_in & k_in
|
||||
if sliding_window > 0:
|
||||
mm = mm & ((q_idx - kv_idx) < sliding_window)
|
||||
keep = keep | mm
|
||||
return keep
|
||||
|
||||
mm_prefix_mask_mod.use_fast_sampling = True
|
||||
|
||||
@@ -676,6 +676,9 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
v_scale_cache=v_scale_cache,
|
||||
chunk_lookback=self.chunk_lookback,
|
||||
use_td=self.use_td,
|
||||
mm_prefix_clamp_sliding_window=getattr(
|
||||
layer, "mm_prefix_clamp_sliding_window", False
|
||||
),
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -283,6 +283,7 @@ def compute_kv_seq_mask(
|
||||
per_seq_causal_ptr=None,
|
||||
CHUNK_LOOKBACK: tl.constexpr = -1,
|
||||
CHUNK_SIZE: tl.constexpr = -1,
|
||||
MM_PREFIX_CLAMP_SW: tl.constexpr = False,
|
||||
):
|
||||
"""Build the KV mask for one tile.
|
||||
|
||||
@@ -333,7 +334,14 @@ def compute_kv_seq_mask(
|
||||
seq_mask = seq_mask & sw_left
|
||||
|
||||
# PrefixLM: extend mask with bidirectional ranges for multimodal tokens.
|
||||
# Applied AFTER sliding window so mm_prefix ranges override SW restriction.
|
||||
# Default (MM_PREFIX_CLAMP_SW=False): applied AFTER sliding window so
|
||||
# mm_prefix ranges override the SW restriction -> (causal AND SW) OR mm.
|
||||
# Gemma4 (MM_PREFIX_CLAMP_SW=True): the bidirectional image block must stay
|
||||
# within the sliding window, matching HF's (causal OR blockwise) AND
|
||||
# sliding_window. We AND each range with the SW past-bound
|
||||
# (query_abs_pos - seq_offset) < SLIDING_WINDOW (== HF sliding_window_overlay
|
||||
# kv > q - sw; future kv passes trivially). Inert for full-attention layers
|
||||
# (SLIDING_WINDOW <= 0).
|
||||
if USE_MM_PREFIX:
|
||||
for i in range(MAX_MM_RANGES):
|
||||
range_start = tl.load(
|
||||
@@ -351,7 +359,10 @@ def compute_kv_seq_mask(
|
||||
& (seq_offset[None, :] <= range_end)
|
||||
& is_valid
|
||||
)
|
||||
seq_mask |= q_in_range & k_in_range
|
||||
mm_mask = q_in_range & k_in_range
|
||||
if MM_PREFIX_CLAMP_SW and SLIDING_WINDOW > 0:
|
||||
mm_mask = mm_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW)
|
||||
seq_mask |= mm_mask
|
||||
return seq_mask
|
||||
|
||||
|
||||
|
||||
@@ -275,6 +275,10 @@ def kernel_unified_attention(
|
||||
USE_TD: tl.constexpr = False,
|
||||
USE_TD_QO: tl.constexpr = False,
|
||||
Q_IS_FP8: tl.constexpr = False,
|
||||
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window
|
||||
# instead of letting them override it. Default False preserves the
|
||||
# original (causal AND SW) OR mm_prefix behavior for all other models.
|
||||
MM_PREFIX_CLAMP_SW: tl.constexpr = False,
|
||||
):
|
||||
USE_PER_TOKEN_HEAD_SCALES: tl.constexpr = KV_QUANT_MODE >= 2
|
||||
USE_FP8_Q_DESCALE: tl.constexpr = KV_QUANT_MODE == 1 and Q_IS_FP8
|
||||
@@ -509,6 +513,7 @@ def kernel_unified_attention(
|
||||
per_seq_causal_ptr,
|
||||
CHUNK_LOOKBACK,
|
||||
CHUNK_SIZE,
|
||||
MM_PREFIX_CLAMP_SW,
|
||||
)
|
||||
|
||||
# S : (BLOCK_M, TILE_SIZE)
|
||||
@@ -819,6 +824,9 @@ def unified_attention(
|
||||
# The non-TD branch is dead-code-eliminated at Triton compile time so
|
||||
# disabling this flag costs nothing.
|
||||
use_td: bool = False,
|
||||
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window.
|
||||
# Default False keeps the original behavior for every other model.
|
||||
mm_prefix_clamp_sliding_window: bool = False,
|
||||
):
|
||||
# Resolve causal: bool or per-seq tensor.
|
||||
use_per_seq_causal = isinstance(causal, torch.Tensor)
|
||||
@@ -1130,6 +1138,7 @@ def unified_attention(
|
||||
CHUNK_SIZE=chunk_size,
|
||||
USE_TD=use_td,
|
||||
USE_TD_QO=use_td_qo,
|
||||
MM_PREFIX_CLAMP_SW=mm_prefix_clamp_sliding_window,
|
||||
**launch_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -2330,13 +2330,20 @@ class GPUModelRunner(
|
||||
|
||||
# Compute mm_prefix bidirectional ranges before building
|
||||
# attention metadata so builders handle them during build().
|
||||
# Ranges exceeding sliding_window are skipped to prevent
|
||||
# early tokens from attending across the entire image span.
|
||||
# By default, ranges exceeding sliding_window are skipped to prevent
|
||||
# early tokens from attending across the entire image span. Models that
|
||||
# clamp mm_prefix to the sliding window *in-kernel* (e.g. Gemma4, which
|
||||
# needs HF's (causal OR blockwise) AND sliding_window on sliding layers)
|
||||
# opt out of the skip so the bidirectional range survives for images
|
||||
# larger than the window; the kernel then bounds it per-query.
|
||||
req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None
|
||||
if self.is_mm_prefix_lm:
|
||||
req_doc_ranges = {}
|
||||
hf_text_config = self.model_config.hf_text_config
|
||||
_bidi_sw = getattr(hf_text_config, "sliding_window", None)
|
||||
_clamps_in_kernel = getattr(
|
||||
self.model, "mm_prefix_clamp_sliding_window", False
|
||||
)
|
||||
for req_id in self.input_batch.req_ids:
|
||||
image_doc_ranges = []
|
||||
req_state = self.requests[req_id]
|
||||
@@ -2346,7 +2353,11 @@ class GPUModelRunner(
|
||||
pos_info = mm_feature.mm_position
|
||||
img_doc_range = pos_info.extract_embeds_range()
|
||||
for r in img_doc_range:
|
||||
if _bidi_sw is not None and (r[1] - r[0] + 1) > _bidi_sw:
|
||||
if (
|
||||
not _clamps_in_kernel
|
||||
and _bidi_sw is not None
|
||||
and (r[1] - r[0] + 1) > _bidi_sw
|
||||
):
|
||||
continue
|
||||
image_doc_ranges.append(r)
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
|
||||
Reference in New Issue
Block a user