[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 02:25:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 866f02db8a
commit e3cae8d2ac
36 changed files with 366 additions and 2385 deletions
+10 -37
View File
@@ -7,6 +7,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
@@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import (
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextAttention,
Qwen3NextMLP,
Qwen3NextSparseMoeBlock,
)
from vllm.model_executor.models.qwen3_vl import (
@@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
@@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
) -> None:
super().__init__()
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
@@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
prefix=f"{prefix}.linear_attn",
gqa_interleaved_layout=False,
)
elif layer_type in ATTENTION_LAYER_TYPES:
use_qsa = (
layer_type == QSA_LAYER_TYPE
or getattr(config, "indexer_n_heads", None) is not None
)
elif layer_type == "qwen_sparse_attention":
use_qsa = getattr(config, "indexer_n_heads", None) is not None
if not use_qsa:
self.self_attn = Qwen3NextAttention(
config,
model_config=model_config,
cache_config=cache_config,
model_config=vllm_config.model_config,
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
@@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
else:
raise ValueError(f"Invalid layer_type {layer_type}")
mlp_only_layers = getattr(config, "mlp_only_layers", [])
num_experts = getattr(config, "num_experts", 0) or 0
absolute_layer_id = self.layer_idx + 1
is_moe_layer = self.layer_idx not in mlp_only_layers and (
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
if is_moe_layer:
self.mlp = Qwen4ExpSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
else:
self.mlp = Qwen3NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
@@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
if self.layer_type == "linear_attention":
attn_out = self.linear_attn(hidden_states=block_input)
elif self.layer_type in ATTENTION_LAYER_TYPES:
elif self.layer_type == "qwen_sparse_attention":
attn_out = self.self_attn(
hidden_states=block_input,
positions=positions,
@@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module):
self._qsa_layer_ids = frozenset(
layer_idx
for layer_idx, layer_type in enumerate(config.layer_types)
if layer_type == QSA_LAYER_TYPE
or (
layer_type == "full_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
if layer_type == "qwen_sparse_attention"
and getattr(config, "indexer_n_heads", None) is not None
)
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)