Refactor sliding window configuration to Transformers best practice (#21927)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -182,21 +182,13 @@ class CohereAttention(nn.Module):
|
||||
)
|
||||
|
||||
# Model v2 has interleaved sliding windows, v1 does not
|
||||
interleaved_sliding_window = getattr(config,
|
||||
"interleaved_sliding_window",
|
||||
None)
|
||||
self.v1 = interleaved_sliding_window is None
|
||||
self.v1 = isinstance(config, CohereConfig)
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
layer_has_sliding_window = (
|
||||
getattr(config, "sliding_window_pattern", False) and
|
||||
(layer_idx + 1) % self.config.sliding_window_pattern
|
||||
!= 0) or (getattr(config, "layer_types", False)
|
||||
and config.layer_types[layer_idx] == "sliding_attention")
|
||||
|
||||
self.sliding_window = (interleaved_sliding_window
|
||||
or config.sliding_window
|
||||
if layer_has_sliding_window else None)
|
||||
self.sliding_window = None
|
||||
if not self.v1:
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
if config.layer_types[layer_idx] == "sliding_attention":
|
||||
self.sliding_window = config.sliding_window
|
||||
|
||||
self.attn = Attention(self.num_heads,
|
||||
self.head_dim,
|
||||
|
||||
@@ -159,25 +159,12 @@ class Exaone4Attention(nn.Module):
|
||||
if quant_config is not None and quant_config.get_name() == "gguf":
|
||||
is_neox_style = False
|
||||
|
||||
self.apply_all_layers = False # apply rotary embeddings to every layer.
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
interleaved_sliding_window = getattr(config,
|
||||
"interleaved_sliding_window",
|
||||
4096)
|
||||
sliding_window_pattern = getattr(config, "sliding_window_pattern",
|
||||
"LLLG")
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
self.sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
if sliding_window_pattern:
|
||||
layer_has_sliding_window = (
|
||||
layer_idx + 1) % sliding_window_pattern.__len__() != 0
|
||||
else:
|
||||
layer_has_sliding_window = False
|
||||
self.apply_all_layers = True
|
||||
|
||||
if layer_has_sliding_window:
|
||||
self.sliding_window = interleaved_sliding_window
|
||||
else:
|
||||
self.sliding_window = None
|
||||
# apply rotary embeddings to every layer
|
||||
self.apply_all_layers = not is_sliding
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
|
||||
@@ -144,13 +144,10 @@ class Gemma2Attention(nn.Module):
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
# reference:
|
||||
# https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
use_sliding_window = (layer_idx % 2 == 0 and getattr(
|
||||
config, "interleaved_sliding_window", None) is not None)
|
||||
sliding_window = config.interleaved_sliding_window if \
|
||||
use_sliding_window else None
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
self.attn = Attention(self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
|
||||
@@ -146,25 +146,19 @@ class Gemma3Attention(nn.Module):
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
# TODO(woosuk): Add reference to the original HF implementation.
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
self.is_sliding = (getattr(
|
||||
config, "interleaved_sliding_window", None) is not None and (bool(
|
||||
(layer_idx + 1) % config.sliding_window_pattern))) or (
|
||||
getattr(config, "layer_types", None) is not None
|
||||
and config.layer_types[layer_idx] == "sliding_attention")
|
||||
self.is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
sliding_window = config.sliding_window if self.is_sliding else None
|
||||
|
||||
# Initialize the rotary embedding.
|
||||
if self.is_sliding:
|
||||
# Local attention. Override the values in config.json.
|
||||
self.rope_theta = config.rope_local_base_freq
|
||||
self.rope_scaling = {"rope_type": "default"}
|
||||
self.sliding_window = (config.interleaved_sliding_window
|
||||
or config.sliding_window)
|
||||
else:
|
||||
# Global attention. Use the values in config.json.
|
||||
self.rope_theta = config.rope_theta
|
||||
self.rope_scaling = config.rope_scaling
|
||||
self.sliding_window = None
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
rotary_dim=self.head_dim,
|
||||
@@ -182,7 +176,7 @@ class Gemma3Attention(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
logits_soft_cap=attn_logits_soft_cap,
|
||||
per_layer_sliding_window=self.sliding_window,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
prefix=f"{prefix}.attn")
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -502,8 +502,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.sliding_window = getattr(config.text_config,
|
||||
"interleaved_sliding_window", None)
|
||||
|
||||
self.vision_tower = SiglipVisionModel(config.vision_config,
|
||||
quant_config,
|
||||
@@ -690,11 +688,11 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
|
||||
global_attn_mask = torch.where(img_mask == 2, 0, global_attn_mask)
|
||||
global_attn_masks.append(global_attn_mask)
|
||||
|
||||
if self.sliding_window is not None:
|
||||
if (sliding_window := self.config.sliding_window) is not None:
|
||||
# Create a local causal mask with sliding window (1024).
|
||||
local_attn_mask = torch.ones_like(global_attn_mask)
|
||||
local_attn_mask = torch.tril(local_attn_mask,
|
||||
diagonal=-self.sliding_window)
|
||||
diagonal=-sliding_window)
|
||||
local_attn_mask = torch.where(local_attn_mask == 0,
|
||||
global_attn_mask, float("-inf"))
|
||||
local_attn_masks.append(local_attn_mask)
|
||||
|
||||
@@ -313,17 +313,16 @@ class Gemma3nAttention(nn.Module):
|
||||
has_weight=False)
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
self.sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
is_sliding_window = (
|
||||
getattr(config, "interleaved_sliding_window", None) is not None
|
||||
and config.layer_types[layer_idx] == "sliding_attention")
|
||||
|
||||
if is_sliding_window:
|
||||
self.sliding_window = config.interleaved_sliding_window
|
||||
# Initialize the rotary embedding.
|
||||
if is_sliding:
|
||||
# Local attention. Override the values in config.json.
|
||||
rope_theta = config.rope_local_base_freq
|
||||
rope_scaling = {"rope_type": "default"}
|
||||
else:
|
||||
self.sliding_window = None
|
||||
# Global attention. Use the values in config.json.
|
||||
rope_theta = config.rope_theta
|
||||
rope_scaling = config.rope_scaling
|
||||
|
||||
|
||||
@@ -248,9 +248,7 @@ class GritLM(LlamaForCausalLM, SupportsV0Only):
|
||||
|
||||
vllm_config.cache_config.sliding_window = None
|
||||
|
||||
for attr in ("sliding_window", "interleaved_sliding_window"):
|
||||
if hasattr(hf_config, attr):
|
||||
delattr(hf_config, attr)
|
||||
hf_config.sliding_window = None
|
||||
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
|
||||
|
||||
|
||||
@@ -167,18 +167,11 @@ class LlamaAttention(nn.Module):
|
||||
rope_scaling=rope_scaling,
|
||||
quant_config=quant_config)
|
||||
|
||||
if hasattr(config, "interleaved_sliding_window"):
|
||||
interleaved_sliding_window = config.interleaved_sliding_window
|
||||
if isinstance(interleaved_sliding_window, int):
|
||||
sliding_window = interleaved_sliding_window
|
||||
elif isinstance(interleaved_sliding_window, list):
|
||||
sw_idx = layer_idx % len(interleaved_sliding_window)
|
||||
sliding_window = interleaved_sliding_window[sw_idx]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{type(interleaved_sliding_window)} is not supported.")
|
||||
else:
|
||||
sliding_window = None
|
||||
sliding_window = None
|
||||
if layer_types := getattr(config, "layer_types", None):
|
||||
is_sliding = layer_types[layer_idx] == "sliding_attention"
|
||||
if is_sliding:
|
||||
sliding_window = config.sliding_window
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
|
||||
@@ -116,13 +116,8 @@ class SambaYAttention(nn.Module):
|
||||
self.Wqkv = nn.Linear(self.hidden_size, op_size, bias=True)
|
||||
|
||||
# disable sliding window for the second half of the model
|
||||
sliding_window = config.interleaved_sliding_window[layer_idx]
|
||||
if layer_idx >= config.num_hidden_layers // 2:
|
||||
assert sliding_window is None, \
|
||||
"sliding_window must be none for the second decoder"
|
||||
else:
|
||||
assert sliding_window is not None, \
|
||||
"sliding_window must be set for the first decoder"
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
assert self.num_heads % 2 == 0, 'num_heads should be even'
|
||||
assert self.num_key_value_heads % 2 == 0, 'num_heads should be even'
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader, maybe_remap_kv_scale_name)
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import is_interleaved
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
|
||||
@@ -285,8 +286,7 @@ class Qwen2Model(nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
# TODO (@robertgshaw2): see if this can be moved out
|
||||
if (cache_config.sliding_window is not None
|
||||
and hasattr(config, "max_window_layers")):
|
||||
if is_interleaved(vllm_config.model_config.hf_text_config):
|
||||
assert config.max_window_layers == config.num_hidden_layers, (
|
||||
"Sliding window for some but all layers is not supported. "
|
||||
"This model uses sliding window but `max_window_layers` = {} "
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
# limitations under the License.
|
||||
"""Wrapper around `transformers` models"""
|
||||
from collections.abc import Iterable, Mapping
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import regex as re
|
||||
@@ -382,33 +382,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
)
|
||||
|
||||
|
||||
class ConfigOverride:
|
||||
"""Context manager to temporarily override config attributes."""
|
||||
|
||||
def __init__(self, config: PretrainedConfig, **kwargs):
|
||||
self.config = config
|
||||
self.kwargs = kwargs
|
||||
self.kwargs_original = {}
|
||||
self.kwargs_delete = set()
|
||||
|
||||
def __enter__(self):
|
||||
"""Override config attributes."""
|
||||
for key, value in self.kwargs.items():
|
||||
if not hasattr(self.config, key):
|
||||
self.kwargs_delete.add(key)
|
||||
self.kwargs_original[key] = getattr(self.config, key, None)
|
||||
setattr(self.config, key, value)
|
||||
return self.config
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
"""Restore original config attributes."""
|
||||
for key, value in self.kwargs_original.items():
|
||||
if key in self.kwargs_delete:
|
||||
delattr(self.config, key)
|
||||
else:
|
||||
setattr(self.config, key, value)
|
||||
|
||||
|
||||
class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
embedding_padding_modules = ["lm_head"]
|
||||
embedding_modules = ["embed_tokens"
|
||||
@@ -434,21 +407,11 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
# To be updated in child classes for use in `load_weights`
|
||||
self.skip_prefixes: Optional[list[str]] = None
|
||||
|
||||
# vLLM handles interleaved sliding window attention by creating a new
|
||||
# interleaved_sliding_window attribute and deleting the sliding_window
|
||||
# attribute. This breaks the constructors in Transformers so we
|
||||
# temporarily add the attribute back to construct the model.
|
||||
config_override = nullcontext()
|
||||
if hasattr(self.config, "interleaved_sliding_window"):
|
||||
config_override = ConfigOverride(
|
||||
self.config,
|
||||
sliding_window=self.config.interleaved_sliding_window)
|
||||
|
||||
# Set correct attn and init on "meta" to delay allocating GPU tensors
|
||||
# TODO: @raushan, use the public `model.set_attn_implementation()`
|
||||
# method once its checks are fixed in Transformers.
|
||||
self.text_config._attn_implementation = "vllm"
|
||||
with init_on_device_without_buffers("meta"), config_override:
|
||||
with init_on_device_without_buffers("meta"):
|
||||
self.model: PreTrainedModel = AutoModel.from_config(
|
||||
self.config,
|
||||
torch_dtype=self.model_config.dtype,
|
||||
@@ -575,11 +538,10 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
attention_instances = {}
|
||||
for i in range(start, end):
|
||||
# Handle interleaved sliding window attention
|
||||
sliding_window = None
|
||||
if (hasattr(self.config, "interleaved_sliding_window")
|
||||
and hasattr(self.config, "sliding_window_pattern")
|
||||
and ((i + 1) % self.config.sliding_window_pattern > 0)):
|
||||
sliding_window = self.config.interleaved_sliding_window
|
||||
per_layer_sliding_window = None
|
||||
if (hasattr(self.config, "layer_types")
|
||||
and self.config.layer_types[i] == "sliding_attention"):
|
||||
per_layer_sliding_window = self.config.sliding_window
|
||||
|
||||
attention_instances[i] = Attention(
|
||||
num_heads=num_heads,
|
||||
@@ -590,7 +552,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
num_kv_heads=num_kv_heads,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
per_layer_sliding_window=per_layer_sliding_window,
|
||||
prefix=f"{i}.attn")
|
||||
return attention_instances
|
||||
|
||||
|
||||
Reference in New Issue
Block a user