[CI] Bump Transformers version to 5.17.0 (#56108)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 06:34:16 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Claude Opus 5
parent f8b5c11468
commit ec4a3a5370
17 changed files with 148 additions and 171 deletions
+12 -6
View File
@@ -76,7 +76,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..config import Qwen4ExpConfig
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ple_layer import Qwen4ExpPLELayer
@@ -214,8 +214,11 @@ class Qwen4ExpDecoderLayer(nn.Module):
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
)
elif layer_type == "full_attention":
use_qsa = getattr(config, "indexer_n_heads", None) is not None
elif layer_type in ATTENTION_LAYER_TYPES:
use_qsa = (
layer_type == QSA_LAYER_TYPE
or getattr(config, "indexer_n_heads", None) is not None
)
if not use_qsa:
self.self_attn = Qwen3NextAttention(
config,
@@ -311,7 +314,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type == "full_attention":
elif self.layer_type in ATTENTION_LAYER_TYPES:
attn_out = self.self_attn(
hidden_states=block_input,
positions=positions,
@@ -402,8 +405,11 @@ class Qwen4ExpModel(nn.Module):
self._qsa_layer_ids = frozenset(
layer_idx
for layer_idx, layer_type in enumerate(config.layer_types)
if layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
if layer_type == QSA_LAYER_TYPE
or (
layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
)
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
+12 -1
View File
@@ -19,6 +19,12 @@ _QSA_CONFIG_FIELDS = (
"indexer_compress_ratio",
)
# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
# Older checkpoints label them `full_attention` and mark QSA by setting the
# indexer fields for the whole model.
QSA_LAYER_TYPE = "qwen_sparse_attention"
ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
model_type = "qwen4_exp"
@@ -134,6 +140,11 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
def _validate_qsa_config(self) -> None:
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
if all(value is None for value in configured.values()):
if QSA_LAYER_TYPE in (self.layer_types or ()):
raise ValueError(
f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
f"{list(_QSA_CONFIG_FIELDS)}"
)
return
missing = [name for name, value in configured.items() if value is None]
@@ -165,7 +176,7 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
@property
def layers_block_type(self) -> list[str]:
return [
"attention" if layer_type == "full_attention" else layer_type
"attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
for layer_type in self.layer_types
]
+12 -6
View File
@@ -75,7 +75,7 @@ from vllm.transformers_utils.configs.qwen4_exp import (
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..config import Qwen4ExpConfig
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ple_layer import Qwen4ExpPLELayer
@@ -215,8 +215,11 @@ class Qwen4ExpDecoderLayer(nn.Module):
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
)
elif layer_type == "full_attention":
use_qsa = getattr(config, "indexer_n_heads", None) is not None
elif layer_type in ATTENTION_LAYER_TYPES:
use_qsa = (
layer_type == QSA_LAYER_TYPE
or getattr(config, "indexer_n_heads", None) is not None
)
if not use_qsa:
self.self_attn = Qwen3NextAttention(
config,
@@ -314,7 +317,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type == "full_attention":
elif self.layer_type in ATTENTION_LAYER_TYPES:
attn_out = self.self_attn(
hidden_states=block_input,
positions=positions,
@@ -394,8 +397,11 @@ class Qwen4ExpModel(nn.Module):
self._qsa_layer_ids = frozenset(
layer_idx
for layer_idx, layer_type in enumerate(config.layer_types)
if layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
if layer_type == QSA_LAYER_TYPE
or (
layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
)
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)