[CI/Build] Ensure compatability with Transformers v4.53 (#20541)

Signed-off-by: Isotr0py <2037008807@qq.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2025-07-11 20:53:07 -07:00
committed by GitHub
parent 11c0198615
commit 01cae37713
13 changed files with 74 additions and 38 deletions
+6 -3
View File
@@ -149,14 +149,17 @@ class Gemma3Attention(nn.Module):
# TODO(woosuk): Add reference to the original HF implementation.
layer_idx = extract_layer_index(prefix)
self.is_sliding = (getattr(
config, "interleaved_sliding_window", None) is not None and bool(
(layer_idx + 1) % config.sliding_window_pattern))
config, "interleaved_sliding_window", None) is not None and (bool(
(layer_idx + 1) % config.sliding_window_pattern))) or (
getattr(config, "layer_types", None) is not None
and config.layer_types[layer_idx] == "sliding_attention")
# Initialize the rotary embedding.
if self.is_sliding:
# Local attention. Override the values in config.json.
self.rope_theta = config.rope_local_base_freq
self.rope_scaling = {"rope_type": "default"}
self.sliding_window = config.interleaved_sliding_window
self.sliding_window = (config.interleaved_sliding_window
or config.sliding_window)
else:
# Global attention. Use the values in config.json.
self.rope_theta = config.rope_theta