[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -8,6 +8,7 @@ from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
|
||||
@@ -7,6 +7,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import (
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextAttention,
|
||||
Qwen3NextMLP,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
@@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
@@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
@@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type in ATTENTION_LAYER_TYPES:
|
||||
use_qsa = (
|
||||
layer_type == QSA_LAYER_TYPE
|
||||
or getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
elif layer_type == "qwen_sparse_attention":
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
model_config=vllm_config.model_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
@@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
num_experts = getattr(config, "num_experts", 0) or 0
|
||||
absolute_layer_id = self.layer_idx + 1
|
||||
is_moe_layer = self.layer_idx not in mlp_only_layers and (
|
||||
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
if is_moe_layer:
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen3NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
@@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in ATTENTION_LAYER_TYPES:
|
||||
elif self.layer_type == "qwen_sparse_attention":
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == QSA_LAYER_TYPE
|
||||
or (
|
||||
layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if layer_type == "qwen_sparse_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from collections.abc import Iterable
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
@@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="full_attention",
|
||||
layer_type="qwen_sparse_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
|
||||
@@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp model configuration."""
|
||||
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
|
||||
Qwen3VLVisionConfig,
|
||||
)
|
||||
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
_QSA_CONFIG_FIELDS = (
|
||||
"indexer_n_heads",
|
||||
"indexer_kv_heads",
|
||||
"indexer_head_dim",
|
||||
"indexer_budget",
|
||||
"indexer_compress_ratio",
|
||||
)
|
||||
|
||||
# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
|
||||
# Older checkpoints label them `full_attention` and mark QSA by setting the
|
||||
# indexer fields for the whole model.
|
||||
QSA_LAYER_TYPE = "qwen_sparse_attention"
|
||||
ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
|
||||
|
||||
|
||||
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
|
||||
model_type = "qwen4_exp"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
|
||||
class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
model_type = "qwen4_exp_text"
|
||||
base_config_key = "text_config"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_count: int = 4,
|
||||
hc_lowrank: int = 320,
|
||||
ple_layer_ids: list[int] | None = None,
|
||||
ple_embed_dim: int | None = None,
|
||||
ple_conv_kernel_size: int = 4,
|
||||
ngram_size: int = 3,
|
||||
heads_per_ngram: int = 8,
|
||||
ngram_vocab_size_base: int = 20_000_000,
|
||||
make_ngram_vocab_size_divisible_by: int = 128,
|
||||
output_gate_type: str = "sigmoid",
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
layer_types: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if hc_count <= 1:
|
||||
raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
|
||||
|
||||
if rope_parameters is not None:
|
||||
if kwargs.get("rope_scaling") is None:
|
||||
kwargs["rope_scaling"] = rope_parameters
|
||||
if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
|
||||
kwargs["rope_theta"] = rope_parameters["rope_theta"]
|
||||
if (
|
||||
kwargs.get("partial_rotary_factor") is None
|
||||
and "partial_rotary_factor" in rope_parameters
|
||||
):
|
||||
kwargs["partial_rotary_factor"] = rope_parameters[
|
||||
"partial_rotary_factor"
|
||||
]
|
||||
|
||||
rope_scaling = kwargs.get("rope_scaling")
|
||||
rope_theta = kwargs.get("rope_theta", 10_000.0)
|
||||
super().__init__(layer_types=layer_types, **kwargs)
|
||||
|
||||
normalized_rope_parameters = self.rope_parameters
|
||||
self.rope_scaling = (
|
||||
rope_scaling or rope_parameters or normalized_rope_parameters
|
||||
)
|
||||
self.rope_parameters = rope_parameters or normalized_rope_parameters
|
||||
self.rope_theta = rope_theta
|
||||
|
||||
self.hc_count = hc_count
|
||||
self.hc_lowrank = hc_lowrank
|
||||
self.ple_layer_ids = ple_layer_ids or []
|
||||
self.ple_embed_dim = (
|
||||
self.hidden_size if ple_embed_dim is None else ple_embed_dim
|
||||
)
|
||||
self.ple_conv_kernel_size = ple_conv_kernel_size
|
||||
self.ngram_size = ngram_size
|
||||
self.heads_per_ngram = heads_per_ngram
|
||||
self.ngram_vocab_size_base = ngram_vocab_size_base
|
||||
self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
|
||||
self.output_gate_type = output_gate_type
|
||||
|
||||
self._validate_ple_config()
|
||||
self._validate_ple_layer_ids()
|
||||
self._validate_qsa_config()
|
||||
|
||||
def _validate_ple_config(self) -> None:
|
||||
if self.hc_lowrank <= 0:
|
||||
raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
|
||||
if self.ngram_size < 2:
|
||||
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
|
||||
if self.heads_per_ngram <= 0:
|
||||
raise ValueError(
|
||||
f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
|
||||
)
|
||||
if self.ple_embed_dim <= 0:
|
||||
raise ValueError(
|
||||
f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
|
||||
)
|
||||
ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
|
||||
if self.ple_embed_dim % ngram_heads:
|
||||
raise ValueError(
|
||||
"ple_embed_dim must be divisible by total ngram heads: "
|
||||
f"{self.ple_embed_dim} % {ngram_heads} != 0"
|
||||
)
|
||||
if self.ple_conv_kernel_size <= 0:
|
||||
raise ValueError(
|
||||
"ple_conv_kernel_size must be positive, got "
|
||||
f"{self.ple_conv_kernel_size}"
|
||||
)
|
||||
if self.ngram_vocab_size_base <= 0:
|
||||
raise ValueError("ngram_vocab_size_base must be positive")
|
||||
if self.make_ngram_vocab_size_divisible_by <= 0:
|
||||
raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
|
||||
|
||||
def _validate_ple_layer_ids(self) -> None:
|
||||
invalid = [
|
||||
layer_id
|
||||
for layer_id in self.ple_layer_ids
|
||||
if not 1 <= int(layer_id) <= self.num_hidden_layers
|
||||
]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
"ple_layer_ids are 1-based and must refer to an existing layer; "
|
||||
f"got {invalid} for {self.num_hidden_layers} layers"
|
||||
)
|
||||
|
||||
def _validate_qsa_config(self) -> None:
|
||||
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
|
||||
if all(value is None for value in configured.values()):
|
||||
if QSA_LAYER_TYPE in (self.layer_types or ()):
|
||||
raise ValueError(
|
||||
f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
|
||||
f"{list(_QSA_CONFIG_FIELDS)}"
|
||||
)
|
||||
return
|
||||
|
||||
missing = [name for name, value in configured.items() if value is None]
|
||||
if missing:
|
||||
raise ValueError(f"QSA config is missing required fields: {missing}")
|
||||
|
||||
values = {name: int(cast(int, value)) for name, value in configured.items()}
|
||||
if any(value <= 0 for value in values.values()):
|
||||
raise ValueError(f"QSA config values must be positive: {values}")
|
||||
if values["indexer_kv_heads"] != 1:
|
||||
raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
|
||||
if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
|
||||
raise ValueError(
|
||||
"indexer_budget must be divisible by indexer_compress_ratio"
|
||||
)
|
||||
block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
|
||||
if block_topk not in (512, 2048):
|
||||
raise ValueError(
|
||||
"QSA requires indexer_budget / indexer_compress_ratio "
|
||||
f"to be 512 or 2048, got {block_topk}"
|
||||
)
|
||||
rotary_dim = int(self.head_dim * self.partial_rotary_factor)
|
||||
if rotary_dim > values["indexer_head_dim"]:
|
||||
raise ValueError(
|
||||
"QSA indexer_head_dim must cover the attention rotary "
|
||||
f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self) -> list[str]:
|
||||
return [
|
||||
"attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
|
||||
for layer_type in self.layer_types
|
||||
]
|
||||
|
||||
@property
|
||||
def short_conv_layer_ids(self) -> list[int]:
|
||||
if not self.ple_layer_ids:
|
||||
return []
|
||||
return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
|
||||
|
||||
@property
|
||||
def short_conv_state_shape(self) -> tuple[int, int] | None:
|
||||
if not self.short_conv_layer_ids:
|
||||
return None
|
||||
ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
|
||||
ple_channels = self.hidden_size * self.hc_count
|
||||
return ple_channels, ple_state_len
|
||||
|
||||
@property
|
||||
def ngram_context_len(self) -> int:
|
||||
if not self.ple_layer_ids:
|
||||
return 0
|
||||
return max(int(self.ngram_size) - 1, 0)
|
||||
|
||||
|
||||
class Qwen4ExpConfig(PreTrainedConfig):
|
||||
model_type = "qwen4_exp"
|
||||
sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
|
||||
"vision_config": Qwen4ExpVisionConfig,
|
||||
"text_config": Qwen4ExpTextConfig,
|
||||
}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
|
||||
vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
|
||||
image_token_id: int = 248056,
|
||||
video_token_id: int = 248057,
|
||||
vision_start_token_id: int = 248053,
|
||||
vision_end_token_id: int = 248054,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if text_config is not None:
|
||||
kwargs.pop("split_ngram_parts", None)
|
||||
|
||||
text_kwargs = (
|
||||
dict(kwargs)
|
||||
if text_config is None
|
||||
and "hidden_size" in kwargs
|
||||
and "num_hidden_layers" in kwargs
|
||||
else {}
|
||||
)
|
||||
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = self.sub_configs["vision_config"]()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = self.sub_configs["text_config"](**text_kwargs)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
self.rope_parameters = rope_parameters or getattr(
|
||||
self.text_config, "rope_parameters", {}
|
||||
)
|
||||
super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpConfig",
|
||||
"Qwen4ExpTextConfig",
|
||||
"Qwen4ExpVisionConfig",
|
||||
]
|
||||
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
|
||||
@@ -7,6 +7,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
@@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import (
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextAttention,
|
||||
Qwen3NextMLP,
|
||||
Qwen3NextSparseMoeBlock,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
@@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||
@@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
@@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type in ATTENTION_LAYER_TYPES:
|
||||
use_qsa = (
|
||||
layer_type == QSA_LAYER_TYPE
|
||||
or getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
elif layer_type == "qwen_sparse_attention":
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
model_config=vllm_config.model_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
@@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
num_experts = getattr(config, "num_experts", 0) or 0
|
||||
absolute_layer_id = self.layer_idx + 1
|
||||
is_moe_layer = self.layer_idx not in mlp_only_layers and (
|
||||
num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
if is_moe_layer:
|
||||
self.mlp = Qwen4ExpSparseMoeBlock(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
else:
|
||||
self.mlp = Qwen3NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
@@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type in ATTENTION_LAYER_TYPES:
|
||||
elif self.layer_type == "qwen_sparse_attention":
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == QSA_LAYER_TYPE
|
||||
or (
|
||||
layer_type == "full_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if layer_type == "qwen_sparse_attention"
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from collections.abc import Iterable
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
@@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="full_attention",
|
||||
layer_type="qwen_sparse_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
@@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user