[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -8,6 +8,7 @@ from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
|
||||
@@ -7,6 +7,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import (
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextAttention,
|
||||
Qwen3NextMLP,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
@@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
@@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
@@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type in ATTENTION_LAYER_TYPES:
|
||||
use_qsa = (
|
||||
layer_type == QSA_LAYER_TYPE
|
||||
or getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
elif layer_type == "qwen_sparse_attention":
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
model_config=vllm_config.model_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
@@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
num_experts = getattr(config, "num_experts", 0) or 0
|
||||
absolute_layer_id = self.layer_idx + 1
|
||||
is_moe_layer = self.layer_idx not in mlp_only_layers and (
|
||||
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
if is_moe_layer:
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen3NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
@@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in ATTENTION_LAYER_TYPES:
|
||||
elif self.layer_type == "qwen_sparse_attention":
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == QSA_LAYER_TYPE
|
||||
or (
|
||||
layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if layer_type == "qwen_sparse_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from collections.abc import Iterable
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
@@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="full_attention",
|
||||
layer_type="qwen_sparse_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
|
||||
@@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
|
||||
Reference in New Issue
Block a user