[CI] Bump Transformers version to 5.17.0 (#56108)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 5
parent
f8b5c11468
commit
ec4a3a5370
@@ -76,7 +76,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..config import Qwen4ExpConfig
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
@@ -214,8 +214,11 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type == "full_attention":
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
elif layer_type in ATTENTION_LAYER_TYPES:
|
||||
use_qsa = (
|
||||
layer_type == QSA_LAYER_TYPE
|
||||
or getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config,
|
||||
@@ -311,7 +314,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type == "full_attention":
|
||||
elif self.layer_type in ATTENTION_LAYER_TYPES:
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -402,8 +405,11 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
if layer_type == QSA_LAYER_TYPE
|
||||
or (
|
||||
layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
|
||||
@@ -19,6 +19,12 @@ _QSA_CONFIG_FIELDS = (
|
||||
"indexer_compress_ratio",
|
||||
)
|
||||
|
||||
# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
|
||||
# Older checkpoints label them `full_attention` and mark QSA by setting the
|
||||
# indexer fields for the whole model.
|
||||
QSA_LAYER_TYPE = "qwen_sparse_attention"
|
||||
ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
|
||||
|
||||
|
||||
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
|
||||
model_type = "qwen4_exp"
|
||||
@@ -134,6 +140,11 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
def _validate_qsa_config(self) -> None:
|
||||
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
|
||||
if all(value is None for value in configured.values()):
|
||||
if QSA_LAYER_TYPE in (self.layer_types or ()):
|
||||
raise ValueError(
|
||||
f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
|
||||
f"{list(_QSA_CONFIG_FIELDS)}"
|
||||
)
|
||||
return
|
||||
|
||||
missing = [name for name, value in configured.items() if value is None]
|
||||
@@ -165,7 +176,7 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
@property
|
||||
def layers_block_type(self) -> list[str]:
|
||||
return [
|
||||
"attention" if layer_type == "full_attention" else layer_type
|
||||
"attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
|
||||
for layer_type in self.layer_types
|
||||
]
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..config import Qwen4ExpConfig
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
@@ -215,8 +215,11 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type == "full_attention":
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
elif layer_type in ATTENTION_LAYER_TYPES:
|
||||
use_qsa = (
|
||||
layer_type == QSA_LAYER_TYPE
|
||||
or getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config,
|
||||
@@ -314,7 +317,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type == "full_attention":
|
||||
elif self.layer_type in ATTENTION_LAYER_TYPES:
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -394,8 +397,11 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
if layer_type == QSA_LAYER_TYPE
|
||||
or (
|
||||
layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user