[Bugfix][Gemma4] Keep image bidirectional attention within the sliding window (#47217)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
41de1380c2
commit
979f5511d7
@@ -216,6 +216,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
kv_sharing_target_layer_name: str | None = None,
|
||||
mm_prefix_clamp_sliding_window: bool = False,
|
||||
attn_backend: type[AttentionBackend] | None = None,
|
||||
head_size_v: int | None = None,
|
||||
**extra_impl_args,
|
||||
@@ -420,6 +421,9 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context,
|
||||
)
|
||||
self.kv_sharing_target_layer_name = kv_sharing_target_layer_name
|
||||
# Gemma4: clamp mm_prefix bidirectional ranges by the sliding window
|
||||
# (read by the Triton backend impl). Default False for all other models.
|
||||
self.mm_prefix_clamp_sliding_window = mm_prefix_clamp_sliding_window
|
||||
|
||||
# use a placeholder kv cache tensor during init, which will be replaced
|
||||
# by bind_kv_cache
|
||||
|
||||
@@ -499,6 +499,13 @@ class Gemma4Attention(nn.Module):
|
||||
logits_soft_cap=attn_logits_soft_cap,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
kv_sharing_target_layer_name=kv_sharing_target_layer_name,
|
||||
# Gemma4 vision bidi: on sliding layers the bidirectional image
|
||||
# block must stay within the sliding window, matching HF's
|
||||
# (causal OR blockwise) AND sliding_window. Without this the image
|
||||
# span (~1100 soft tokens at max_soft_tokens=1120) exceeds the 1024
|
||||
# window; the runner keeps the full range and the kernel bounds it
|
||||
# per-query here.
|
||||
mm_prefix_clamp_sliding_window=self.is_sliding,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
|
||||
@@ -978,6 +978,13 @@ class Gemma4ForConditionalGeneration(
|
||||
SupportsLoRA,
|
||||
SupportsEagle3,
|
||||
):
|
||||
# Gemma4 clamps mm_prefix bidirectional ranges to the sliding window
|
||||
# in-kernel (HF's (causal OR blockwise) AND sliding_window). The model
|
||||
# runner reads this to keep image bidirectional ranges that exceed the
|
||||
# window instead of dropping them (which would make image attention
|
||||
# causal-only for images larger than the sliding window).
|
||||
mm_prefix_clamp_sliding_window: bool = True
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
|
||||
Reference in New Issue
Block a user