[Model] Gemma4: add bidirectional vision attention for sliding layers with window guard (#40534)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Signed-off-by: Luciano Martins <lucianomartins@google.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Luciano Martins
2026-04-24 08:27:46 +00:00
committed by GitHub
co-authored by Luciano Martins Isotr0py Isotr0py
parent 4c34b2f6fc
commit 512f522192
2 changed files with 73 additions and 1 deletions
+59
View File
@@ -969,6 +969,16 @@ class Gemma4ForConditionalGeneration(
self.language_model.make_empty_intermediate_tensors
)
# --- Precompute full-attention layer indices for bidi clearing ---
self._full_attn_layer_idxs: frozenset[int] = frozenset()
text_config = config.text_config
if getattr(text_config, "use_bidirectional_attention", None) == "vision":
layer_types = getattr(text_config, "layer_types", None)
if layer_types:
self._full_attn_layer_idxs = frozenset(
i for i, lt in enumerate(layer_types) if lt != "sliding_attention"
)
# --- MixtureOfExperts delegation to language_model ---
self.expert_weights = self.language_model.expert_weights
self.moe_layers = self.language_model.moe_layers
@@ -1310,6 +1320,12 @@ class Gemma4ForConditionalGeneration(
else None
)
# Gemma4 bidi: clear mm_prefix_range for full_attention layers.
# Must run here (outside @support_torch_compile boundary) because
# _run_decoder_layers is inside a compiled graph where Python
# side effects are eliminated.
self._clear_mm_prefix_for_full_attn_layers()
hidden_states = self.language_model.model(
input_ids,
positions,
@@ -1327,6 +1343,49 @@ class Gemma4ForConditionalGeneration(
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
# ------------------------------------------------------------------ #
# Bidirectional attention helpers
# ------------------------------------------------------------------ #
def _clear_mm_prefix_for_full_attn_layers(self) -> None:
"""Clear mm_prefix_range for non-sliding layers.
Gemma4 with use_bidirectional_attention='vision' applies
bidirectional attention only to sliding_attention layers.
Full attention layers use plain causal masking.
Uses _full_attn_layer_idxs (precomputed in __init__) for O(1)
lookup instead of per-call regex parsing.
"""
if not self._full_attn_layer_idxs:
return
from vllm.forward_context import get_forward_context
attn_metadata = get_forward_context().attn_metadata
if attn_metadata is None:
return
def _process(metadata_dict: dict) -> None:
for layer_name, metadata in metadata_dict.items():
if ".layers." not in layer_name:
continue
try:
layer_idx = int(layer_name.split(".layers.")[1].split(".")[0])
except (ValueError, IndexError):
continue
if layer_idx in self._full_attn_layer_idxs:
if hasattr(metadata, "mm_prefix_range"):
metadata.mm_prefix_range = None
if hasattr(metadata, "mm_prefix_range_tensor"):
metadata.mm_prefix_range_tensor = None
if isinstance(attn_metadata, list):
for ub_metadata in attn_metadata:
_process(ub_metadata)
elif isinstance(attn_metadata, dict):
_process(attn_metadata)
# ------------------------------------------------------------------ #
# Weight loading
# ------------------------------------------------------------------ #