diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 32562f0d9d..195d1af3be 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -216,6 +216,7 @@ class Attention(nn.Module, AttentionLayerBase): prefix: str = "", attn_type: str = AttentionType.DECODER, kv_sharing_target_layer_name: str | None = None, + mm_prefix_clamp_sliding_window: bool = False, attn_backend: type[AttentionBackend] | None = None, head_size_v: int | None = None, **extra_impl_args, @@ -420,6 +421,9 @@ class Attention(nn.Module, AttentionLayerBase): compilation_config.static_forward_context, ) self.kv_sharing_target_layer_name = kv_sharing_target_layer_name + # Gemma4: clamp mm_prefix bidirectional ranges by the sliding window + # (read by the Triton backend impl). Default False for all other models. + self.mm_prefix_clamp_sliding_window = mm_prefix_clamp_sliding_window # use a placeholder kv cache tensor during init, which will be replaced # by bind_kv_cache diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 9cf86a5ba8..63d97fbd1b 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -499,6 +499,13 @@ class Gemma4Attention(nn.Module): logits_soft_cap=attn_logits_soft_cap, per_layer_sliding_window=sliding_window, kv_sharing_target_layer_name=kv_sharing_target_layer_name, + # Gemma4 vision bidi: on sliding layers the bidirectional image + # block must stay within the sliding window, matching HF's + # (causal OR blockwise) AND sliding_window. Without this the image + # span (~1100 soft tokens at max_soft_tokens=1120) exceeds the 1024 + # window; the runner keeps the full range and the kernel bounds it + # per-query here. + mm_prefix_clamp_sliding_window=self.is_sliding, prefix=f"{prefix}.attn", ) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 5e405c4462..d733a181a0 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -978,6 +978,13 @@ class Gemma4ForConditionalGeneration( SupportsLoRA, SupportsEagle3, ): + # Gemma4 clamps mm_prefix bidirectional ranges to the sliding window + # in-kernel (HF's (causal OR blockwise) AND sliding_window). The model + # runner reads this to keep image bidirectional ranges that exceed the + # window instead of dropping them (which would make image attention + # causal-only for images larger than the sliding window). + mm_prefix_clamp_sliding_window: bool = True + packed_modules_mapping = { "qkv_proj": [ "q_proj", diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 7823fd48b5..a7cc6fb08e 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -892,7 +892,22 @@ class FlashAttentionImpl(AttentionImpl): and self.vllm_flash_attn_version == 4 ): max_ranges = mm_prefix_ranges.shape[1] - mm_mask_mod = _make_mm_prefix_mask_mod(max_ranges) + # Gemma4: clamp the bidirectional block to the sliding + # window (HF (causal OR blockwise) AND sliding_window on + # local layers). Other mm-prefix models pass sliding_window=0 + # -> unclamped, behavior unchanged. sliding_window_size[0]+1 + # is the window (self.sliding_window = (sw-1, 0) on sliding + # layers); causal mm_prefix never hits the symmetric path. + mm_clamp_sw = 0 + if ( + getattr(layer, "mm_prefix_clamp_sliding_window", False) + and sliding_window_size is not None + and sliding_window_size[0] >= 0 + ): + mm_clamp_sw = sliding_window_size[0] + 1 + mm_mask_mod = _make_mm_prefix_mask_mod( + max_ranges, sliding_window=mm_clamp_sw + ) mm_aux = [mm_prefix_ranges] # R-SWA: use CuTE-DSL mask_mod on FA4 for exact token-level @@ -1189,13 +1204,23 @@ class FlashAttentionImpl(AttentionImpl): return output -def _make_mm_prefix_mask_mod(max_ranges: int): +def _make_mm_prefix_mask_mod(max_ranges: int, sliding_window: int = 0): """Build a CuTE-DSL mask_mod implementing (causal OR mm_prefix). Returns a @cute.jit callable that evaluates: keep = (kv_idx <= q_idx) OR - (q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end]) + (q_idx in [r_start,r_end] AND kv_idx in [r_start,r_end] + [AND (q_idx - kv_idx) < sliding_window]) for each mm_prefix range stored in aux_tensors[0]. + + When sliding_window > 0 (Gemma4 sliding layers), the bidirectional block is + clamped to the sliding window, matching HF's + (causal OR blockwise) AND sliding_window (== sliding_window_overlay + kv > q - sw; future kv passes since q - kv < 0). sliding_window <= 0 + (default) leaves the block unclamped, so other mm-prefix models are + unchanged. q_idx/kv_idx are local offsets, but their difference is + frame-invariant and image bidirectional attention only matters during + prefill (where local == absolute). """ import cutlass import cutlass.cute as cute @@ -1223,7 +1248,10 @@ def _make_mm_prefix_mask_mod(max_ranges: int): valid = r_start < r_end q_in = (q_idx >= r_start) & (q_idx <= r_end) & valid k_in = (kv_idx >= r_start) & (kv_idx <= r_end) & valid - keep = keep | (q_in & k_in) + mm = q_in & k_in + if sliding_window > 0: + mm = mm & ((q_idx - kv_idx) < sliding_window) + keep = keep | mm return keep mm_prefix_mask_mod.use_fast_sampling = True diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 628d7a1b1c..b0030d27d9 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -676,6 +676,9 @@ class TritonAttentionImpl(AttentionImpl): v_scale_cache=v_scale_cache, chunk_lookback=self.chunk_lookback, use_td=self.use_td, + mm_prefix_clamp_sliding_window=getattr( + layer, "mm_prefix_clamp_sliding_window", False + ), ) return output diff --git a/vllm/v1/attention/ops/triton_attention_helpers.py b/vllm/v1/attention/ops/triton_attention_helpers.py index b90a1ac39b..e753d53cdf 100644 --- a/vllm/v1/attention/ops/triton_attention_helpers.py +++ b/vllm/v1/attention/ops/triton_attention_helpers.py @@ -283,6 +283,7 @@ def compute_kv_seq_mask( per_seq_causal_ptr=None, CHUNK_LOOKBACK: tl.constexpr = -1, CHUNK_SIZE: tl.constexpr = -1, + MM_PREFIX_CLAMP_SW: tl.constexpr = False, ): """Build the KV mask for one tile. @@ -333,7 +334,14 @@ def compute_kv_seq_mask( seq_mask = seq_mask & sw_left # PrefixLM: extend mask with bidirectional ranges for multimodal tokens. - # Applied AFTER sliding window so mm_prefix ranges override SW restriction. + # Default (MM_PREFIX_CLAMP_SW=False): applied AFTER sliding window so + # mm_prefix ranges override the SW restriction -> (causal AND SW) OR mm. + # Gemma4 (MM_PREFIX_CLAMP_SW=True): the bidirectional image block must stay + # within the sliding window, matching HF's (causal OR blockwise) AND + # sliding_window. We AND each range with the SW past-bound + # (query_abs_pos - seq_offset) < SLIDING_WINDOW (== HF sliding_window_overlay + # kv > q - sw; future kv passes trivially). Inert for full-attention layers + # (SLIDING_WINDOW <= 0). if USE_MM_PREFIX: for i in range(MAX_MM_RANGES): range_start = tl.load( @@ -351,7 +359,10 @@ def compute_kv_seq_mask( & (seq_offset[None, :] <= range_end) & is_valid ) - seq_mask |= q_in_range & k_in_range + mm_mask = q_in_range & k_in_range + if MM_PREFIX_CLAMP_SW and SLIDING_WINDOW > 0: + mm_mask = mm_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW) + seq_mask |= mm_mask return seq_mask diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index b6ba669a08..a1c9abde88 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -275,6 +275,10 @@ def kernel_unified_attention( USE_TD: tl.constexpr = False, USE_TD_QO: tl.constexpr = False, Q_IS_FP8: tl.constexpr = False, + # Gemma4: clamp mm_prefix bidirectional ranges by the sliding window + # instead of letting them override it. Default False preserves the + # original (causal AND SW) OR mm_prefix behavior for all other models. + MM_PREFIX_CLAMP_SW: tl.constexpr = False, ): USE_PER_TOKEN_HEAD_SCALES: tl.constexpr = KV_QUANT_MODE >= 2 USE_FP8_Q_DESCALE: tl.constexpr = KV_QUANT_MODE == 1 and Q_IS_FP8 @@ -509,6 +513,7 @@ def kernel_unified_attention( per_seq_causal_ptr, CHUNK_LOOKBACK, CHUNK_SIZE, + MM_PREFIX_CLAMP_SW, ) # S : (BLOCK_M, TILE_SIZE) @@ -819,6 +824,9 @@ def unified_attention( # The non-TD branch is dead-code-eliminated at Triton compile time so # disabling this flag costs nothing. use_td: bool = False, + # Gemma4: clamp mm_prefix bidirectional ranges by the sliding window. + # Default False keeps the original behavior for every other model. + mm_prefix_clamp_sliding_window: bool = False, ): # Resolve causal: bool or per-seq tensor. use_per_seq_causal = isinstance(causal, torch.Tensor) @@ -1130,6 +1138,7 @@ def unified_attention( CHUNK_SIZE=chunk_size, USE_TD=use_td, USE_TD_QO=use_td_qo, + MM_PREFIX_CLAMP_SW=mm_prefix_clamp_sliding_window, **launch_kwargs, ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index e8d472e9eb..d67a055259 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2330,13 +2330,20 @@ class GPUModelRunner( # Compute mm_prefix bidirectional ranges before building # attention metadata so builders handle them during build(). - # Ranges exceeding sliding_window are skipped to prevent - # early tokens from attending across the entire image span. + # By default, ranges exceeding sliding_window are skipped to prevent + # early tokens from attending across the entire image span. Models that + # clamp mm_prefix to the sliding window *in-kernel* (e.g. Gemma4, which + # needs HF's (causal OR blockwise) AND sliding_window on sliding layers) + # opt out of the skip so the bidirectional range survives for images + # larger than the window; the kernel then bounds it per-query. req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None if self.is_mm_prefix_lm: req_doc_ranges = {} hf_text_config = self.model_config.hf_text_config _bidi_sw = getattr(hf_text_config, "sliding_window", None) + _clamps_in_kernel = getattr( + self.model, "mm_prefix_clamp_sliding_window", False + ) for req_id in self.input_batch.req_ids: image_doc_ranges = [] req_state = self.requests[req_id] @@ -2346,7 +2353,11 @@ class GPUModelRunner( pos_info = mm_feature.mm_position img_doc_range = pos_info.extract_embeds_range() for r in img_doc_range: - if _bidi_sw is not None and (r[1] - r[0] + 1) > _bidi_sw: + if ( + not _clamps_in_kernel + and _bidi_sw is not None + and (r[1] - r[0] + 1) > _bidi_sw + ): continue image_doc_ranges.append(r) req_idx = self.input_batch.req_id_to_index[req_id]