[Chore] Remove vendored code made redundant by Transformers 5.16.1 (#60623)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-09 09:29:03 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent bc713cf8cc
commit 2c99ee9333
76 changed files with 719 additions and 3268 deletions
+2 -45
View File
@@ -79,8 +79,6 @@ class LazyConfigDict(dict):
_CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
afmoe="AfmoeConfig",
axk1="AXK1Config",
bagel="BagelConfig",
bailing_moe_v3_vl="BailingMoeV3VLConfig",
chatglm="ChatGLMConfig",
@@ -89,24 +87,20 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
colqwen3="ColQwen3Config",
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
cosmos3_omni="Cosmos3Config",
cosmos3_edge="Cosmos3EdgeConfig",
diffusion_gemma="DiffusionGemmaConfig",
deepseek_vl_v2="DeepseekVLV2Config",
deepseek_v32="DeepseekV3Config",
deepseek_v4="DeepseekV4Config",
dots3_note="Dots3NoteConfig",
k3_dspark="K3DSparkConfig",
funaudiochat="FunAudioChatConfig",
granite4_vision="Granite4VisionConfig",
hyperclovax="HyperCLOVAXConfig",
hy_v3="HYV3Config",
hy_v4="HYV4Config",
hyperclovax="HyperCLOVAXConfig", # Upstream class, hub remote code is broken
isaac="IsaacConfig",
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
kimi_linear="KimiLinearConfig",
kimi_vl="KimiVLConfig",
kimi_k25="KimiK25Config",
kimi_k25="Kimi_K25Config", # Upstream class, hub remote code uses old schema
muse_glimmer="MuseGlimmerConfig",
muse_glimmer_text="MuseGlimmerTextConfig",
muse_glimmer_vision="MuseGlimmerVisionConfig",
@@ -123,8 +117,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
moss_transcribe_diarize="MossTranscribeDiarizeConfig",
eagle="EAGLEConfig",
speculators="SpeculatorsConfig",
nemotron="NemotronConfig",
olmo_hybrid="OlmoHybridConfig",
openvla="OpenVLAConfig",
ovis="OvisConfig",
ultravox="UltravoxConfig",
@@ -133,12 +125,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
step3p5="Step3p5Config",
qianfan_ocr="QianfanOCRConfig",
qwen3_asr="Qwen3ASRConfig",
qwen3_next="Qwen3NextConfig",
qwen3_5="Qwen3_5Config",
qwen3_5_text="Qwen3_5TextConfig",
qwen3_5_moe="Qwen3_5MoeConfig",
qwen3_5_moe_text="Qwen3_5MoeTextConfig",
lfm2_moe="Lfm2MoeConfig",
**{"unlimited-ocr": "UnlimitedOCRConfig"},
**{"deepseek_v41": "DeepseekV41Config"},
inkling_mm_model="InklingMMConfig",
@@ -162,14 +148,6 @@ _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}
# `gemma4_text` ships them in the checkpoint itself.
_PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna", "gemma4_text"}
# Model types whose checkpoints declare `layer_types` entries that upstream
# transformers has not added to `ALLOWED_LAYER_TYPES` yet, so its strict config
# validation rejects them (e.g. GLM-5.2 `glm_moe_dsa` use
# `deepseek_sparse_attention`). Extend the allowed set for these model types.
_PATCH_HF_ALLOWED_LAYER_TYPES: dict[str, tuple[str, ...]] = {
"glm_moe_dsa": ("deepseek_sparse_attention",),
}
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
"llm_config": "text_config",
}
@@ -314,24 +292,6 @@ def _patch_hf_transformers_nested_rope_validation() -> None:
)
def _patch_hf_transformers_allowed_layer_types(
extra_layer_types: tuple[str, ...],
) -> None:
"""Extend transformers' ``ALLOWED_LAYER_TYPES`` so its strict config
validation accepts layer types (e.g. ``deepseek_sparse_attention``) that a
checkpoint declares but upstream transformers has not registered yet.
"""
import transformers.configuration_utils as hf_configuration_utils
missing = tuple(
layer_type
for layer_type in extra_layer_types
if layer_type not in hf_configuration_utils.ALLOWED_LAYER_TYPES
)
if missing:
hf_configuration_utils.ALLOWED_LAYER_TYPES += missing
class HFConfigParser(ConfigParserBase):
def parse(
self,
@@ -379,9 +339,6 @@ class HFConfigParser(ConfigParserBase):
if model_type in _PATCH_HF_NESTED_ROPE_VALIDATION:
_patch_hf_transformers_nested_rope_validation()
if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type):
_patch_hf_transformers_allowed_layer_types(extra_layer_types)
if model_type == "vlm":
# HyperCLOVAX remote code registers this alias in a bare
# `try/except` that fails silently since transformers 5.18
-216
View File
@@ -1,216 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig
class AXK1Config(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a [`AXK1Model`].
It is used to instantiate an A.X model according to the specified arguments,
defining the model architecture. Instantiating a configuration with the defaults
will yield a similar configuration to that of the A.X K1.
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control
the model outputs. Read the documentation from [`PreTrainedConfig`] for more
information.
Args:
vocab_size (`int`, *optional*, defaults to 163840):
Vocabulary size of the A.X K1 model. Defines the number of different
tokens that can be represented by the `inputs_ids` passed when calling
[`AXK1Model`]
hidden_size (`int`, *optional*, defaults to 7168):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 18432):
Dimension of the MLP representations.
moe_intermediate_size (`int`, *optional*, defaults to 2048):
Dimension of the MoE representations.
num_hidden_layers (`int`, *optional*, defaults to 61):
Number of hidden layers in the Transformer decoder.
num_nextn_predict_layers (`int`, *optional*, defaults to 1):
Number of nextn predict layers in the AXK1 Model.
num_attention_heads (`int`, *optional*, defaults to 64):
Number of attention heads for each attention layer in the Transformer
decoder.
n_shared_experts (`int`, *optional*, defaults to 1):
Number of shared experts, None means dense model.
n_routed_experts (`int`, *optional*, defaults to 192):
Number of routed experts, None means dense model.
routed_scaling_factor (`float`, *optional*, defaults to 2.5):
Scaling factor or routed experts.
topk_method (`str`, *optional*, defaults to `noaux_tc`):
Topk method used in routed gate.
n_group (`int`, *optional*, defaults to 8):
Number of groups for routed experts.
topk_group (`int`, *optional*, defaults to 4):
Number of selected groups for each token(for each token, ensuring the
selected experts is only within `topk_group` groups).
num_experts_per_tok (`int`, *optional*, defaults to 8):
Number of selected experts, None means dense model.
moe_layer_freq (`int`, *optional*, defaults to 1):
The frequency of the MoE layer: one expert layer for every
`moe_layer_freq - 1` dense layers.
first_k_dense_replace (`int`, *optional*, defaults to 1):
Number of dense layers in shallow layers
(embed->dense->dense->...->dense->moe->moe...->lm_head).
\--k dense layers--/
norm_topk_prob (`bool`, *optional*, defaults to True):
Whether to normalize the weights of the routed experts.
scoring_func (`str`, *optional*, defaults to 'sigmoid'):
Method of computing expert weights.
aux_loss_alpha (`float`, *optional*, defaults to 0.0001):
Auxiliary loss weight coefficient.
seq_aux = (`bool`, *optional*, defaults to True):
Whether to compute the auxiliary loss for each individual sample.
num_key_value_heads (`int`, *optional*):
This is the number of key_value heads that should be used to implement
Grouped Query Attention. If `num_key_value_heads=num_attention_heads`,
the model will use Multi Head Attention (MHA), if `num_key_value_heads=1
the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and
value head should be constructed by meanpooling all the original heads
within that group. For more details checkout
[this paper](https://arxiv.org/pdf/2305.13245.pdf).
If it is not specified, will default to `num_attention_heads`.
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
The non-linear activation function (function or string) in the decoder.
max_position_embeddings (`int`, *optional*, defaults to 131072):
The maximum sequence length that this model might ever be used with.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for
initializing all weight matrices.
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
The epsilon used by the rms normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions
(not used by all models). Only relevant if `config.is_decoder=True`.
pad_token_id (`int`, *optional*):
Padding token id.
bos_token_id (`int`, *optional*, defaults to 163691):
Beginning of stream token id.
eos_token_id (`int`, *optional*, defaults to 163691):
End of stream token id.
pretraining_tp (`int`, *optional*, defaults to 1):
Experimental feature. Tensor parallelism rank used during pretraining.
Please refer to
[this document](https://huggingface.co/docs/transformers/parallelism)
to understand more about it. This value is necessary to ensure exact
reproducibility of the pretraining results. Please refer to
[this issue](https://github.com/pytorch/pytorch/issues/76232).
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether to tie weight embeddings
rope_theta (`float`, *optional*, defaults to 10000.0):
The base period of the RoPE embeddings.
rope_scaling (`Dict`, *optional*):
Dictionary containing the scaling configuration for the RoPE embeddings.
Currently supports two scaling strategies: linear and dynamic.
Their scaling factor must be a float greater than 1. The expected format
is `{"type": strategy name, "factor": scaling factor}`. When using this
flag, don't update `max_position_embeddings` to the expected new maximum.
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
Whether to use a bias in the query, key, value and output projection
layers during self-attention.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
"""
model_type = "axk1"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size: int = 163840,
hidden_size: int = 7168,
intermediate_size: int = 18432,
moe_intermediate_size: int = 2048,
num_hidden_layers: int = 61,
num_nextn_predict_layers: int | None = 1,
num_attention_heads: int = 64,
num_key_value_heads: int = 64,
n_shared_experts: int | None = 1,
n_routed_experts: int | None = 192,
ep_size: int | None = 8, ## Ignored - Expert parallel size
routed_scaling_factor: float | None = 2.5,
kv_lora_rank: int | None = 512,
q_lora_rank: int | None = 1536,
qk_rope_head_dim: int | None = 64,
v_head_dim: int | None = 128,
qk_nope_head_dim: int | None = 128,
topk_method: str | None = "noaux_tc",
n_group: int | None = 8,
topk_group: int | None = 4,
num_experts_per_tok: int | None = 8,
moe_layer_freq: int | None = 1,
first_k_dense_replace: int = 1,
norm_topk_prob: bool = True,
scoring_func: str | None = "sigmoid",
aux_loss_alpha: float | None = 0.0001,
seq_aux: float | None = True,
hidden_act: str | None = "silu",
max_position_embeddings: int | None = 131072,
initializer_range: float | None = 0.02,
rms_norm_eps: float = 1e-6,
use_cache: bool | None = True,
pad_token_id: int | None = None,
bos_token_id: int | None = 163691,
eos_token_id: int | None = 163691,
pretraining_tp: int | None = 1,
tie_word_embeddings: bool | None = False,
rope_theta: float | None = 10000.0,
rope_scaling: dict[str, Any] | None = None,
rope_parameters: dict[str, Any] | None = None,
attention_bias: bool | None = False,
attention_dropout: float | None = 0.0,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.moe_intermediate_size = moe_intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_nextn_predict_layers = num_nextn_predict_layers
self.num_attention_heads = num_attention_heads
self.n_shared_experts = n_shared_experts
self.n_routed_experts = n_routed_experts
self.ep_size = ep_size
self.routed_scaling_factor = routed_scaling_factor
self.kv_lora_rank = kv_lora_rank
self.q_lora_rank = q_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.qk_nope_head_dim = qk_nope_head_dim
self.topk_method = topk_method
self.n_group = n_group
self.topk_group = topk_group
self.num_experts_per_tok = num_experts_per_tok
self.moe_layer_freq = moe_layer_freq
self.first_k_dense_replace = first_k_dense_replace
self.norm_topk_prob = norm_topk_prob
self.scoring_func = scoring_func
self.aux_loss_alpha = aux_loss_alpha
self.seq_aux = seq_aux
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.pretraining_tp = pretraining_tp
self.use_cache = use_cache
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
self.rope_parameters = rope_parameters
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
+4 -36
View File
@@ -14,8 +14,6 @@ from __future__ import annotations
import importlib
_CLASS_TO_MODULE: dict[str, str] = {
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
"AXK1Config": "vllm.transformers_utils.configs.AXK1",
"BagelConfig": "vllm.transformers_utils.configs.bagel",
"BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
"BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
@@ -26,13 +24,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
"DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
"DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
@@ -42,16 +37,12 @@ _CLASS_TO_MODULE: dict[str, str] = {
"EAGLEConfig": "vllm.transformers_utils.configs.eagle",
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
# `FalconConfig` class from the official HuggingFace transformers library.
"RWConfig": "vllm.transformers_utils.configs.falcon",
"Lfm2MoeConfig": "vllm.transformers_utils.configs.lfm2_moe",
"MedusaConfig": "vllm.transformers_utils.configs.medusa",
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
"MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3",
@@ -67,16 +58,12 @@ _CLASS_TO_MODULE: dict[str, str] = {
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
"KimiK25Config": "vllm.transformers_utils.configs.kimi_k25",
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
"NemotronHConfig": "vllm.transformers_utils.configs.nemotron_h",
"OlmoHybridConfig": "vllm.transformers_utils.configs.olmo_hybrid",
"OpenVLAConfig": "vllm.transformers_utils.configs.openvla",
"OvisConfig": "vllm.transformers_utils.configs.ovis",
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
@@ -91,22 +78,17 @@ _CLASS_TO_MODULE: dict[str, str] = {
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
"Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
"InklingModelConfig": "vllm.models.inkling.configs",
"InklingAudioConfig": "vllm.models.inkling.configs",
"InklingVisionConfig": "vllm.models.inkling.configs",
"InklingMMConfig": "vllm.models.inkling.configs",
# Special case: DeepseekV3Config is from HuggingFace Transformers
# Upstream Transformers classes registered in _CONFIG_REGISTRY
"DeepseekV3Config": "transformers",
"HyperCLOVAXConfig": "transformers",
"Kimi_K25Config": "transformers",
}
__all__ = [
"AfmoeConfig",
"AXK1Config",
"BagelConfig",
"BailingMoeV3TextConfig",
"BailingMoeV3VisionConfig",
@@ -117,13 +99,10 @@ __all__ = [
"ColQwen3Config",
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
"Cosmos3Config",
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
"DiffusionGemmaConfig",
"DiffusionGemmaTextConfig",
"DeepseekVLV2Config",
"DeepseekV3Config",
"DeepseekV4Config",
@@ -133,13 +112,10 @@ __all__ = [
"EAGLEConfig",
"FunAudioChatConfig",
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
"HYV3Config",
"HYV4Config",
"HyperCLOVAXConfig",
"IsaacConfig",
"RWConfig",
"Lfm2MoeConfig",
"MedusaConfig",
"MiDashengLMConfig",
"MiniMaxM3Config",
@@ -153,16 +129,13 @@ __all__ = [
"MoonViTConfig",
"KimiLinearConfig",
"KimiVLConfig",
"KimiK25Config",
"Kimi_K25Config",
"MuseGlimmerConfig",
"MuseGlimmerTextConfig",
"MuseGlimmerVisionConfig",
"MuseGlimmerAssistantConfig",
"KimiK3Config",
"KimiK3VisionConfig",
"NemotronConfig",
"NemotronHConfig",
"OlmoHybridConfig",
"OpenVLAConfig",
"OvisConfig",
"PixelShuffleSiglip2VisionConfig",
@@ -177,11 +150,6 @@ __all__ = [
"QianfanOCRConfig",
"QianfanOCRVisionConfig",
"Qwen3ASRConfig",
"Qwen3NextConfig",
"Qwen3_5Config",
"Qwen3_5TextConfig",
"Qwen3_5MoeConfig",
"Qwen3_5MoeTextConfig",
"InklingModelConfig",
"InklingAudioConfig",
"InklingVisionConfig",
-87
View File
@@ -1,87 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PreTrainedConfig
class AfmoeConfig(PreTrainedConfig):
model_type = "afmoe"
def __init__(
self,
vocab_size: int = 200_192,
hidden_size: int = 2048,
intermediate_size: int = 6144,
moe_intermediate_size: int = 1408,
num_hidden_layers: int = 32,
num_dense_layers: int = 1,
num_attention_heads: int = 16,
num_key_value_heads: int | None = None,
head_dim: int = 128,
hidden_act: str = "silu",
max_position_embeddings: int = 131072,
initializer_range: float = 0.02,
rms_norm_eps: float = 1e-5,
use_cache: bool = True,
tie_word_embeddings: bool = False,
rope_parameters: dict | None = None,
rope_scaling: dict | None = None,
num_experts: int = 64,
num_experts_per_tok: int = 6,
num_shared_experts: int = 2,
num_expert_groups: int = 1,
num_limited_groups: int = 1,
score_func: str = "sigmoid",
route_norm: bool = True,
route_scale: float = 1.0,
global_attn_every_n_layers: int = 4,
sliding_window: int = 2048,
layer_types: list[str] | None = None,
attention_dropout: float = 0.0,
mup_enabled: bool = False,
n_group: int = 1,
topk_group: int = 1,
**kwargs,
):
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_dense_layers = num_dense_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads or num_attention_heads
self.head_dim = head_dim
self.hidden_act = hidden_act
self.max_position_embeddings = max_position_embeddings
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
rope_theta = kwargs.pop("rope_theta", 10000.0)
if rope_parameters is None:
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
self.rope_parameters = rope_parameters
self.rope_scaling = rope_scaling
self.moe_intermediate_size = moe_intermediate_size
self.num_experts = num_experts
self.num_experts_per_tok = num_experts_per_tok
self.num_shared_experts = num_shared_experts
self.num_expert_groups = num_expert_groups
self.num_limited_groups = num_limited_groups
self.score_func = score_func
self.route_norm = route_norm
self.route_scale = route_scale
self.global_attn_every_n_layers = global_attn_every_n_layers
self.sliding_window = sliding_window
self.layer_types = layer_types
self.attention_dropout = attention_dropout
self.mup_enabled = mup_enabled
self.n_group = n_group
self.topk_group = topk_group
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
__all__ = ["AfmoeConfig"]
@@ -1,7 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLConfig
class Cosmos3Config(Qwen3VLConfig):
model_type = "cosmos3_omni"
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import NemotronHConfig
from transformers.configuration_utils import PreTrainedConfig
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
from .nemotron_h import NemotronHConfig
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
@@ -31,11 +30,14 @@ class Cosmos3EdgeTextConfig(NemotronHConfig):
num_hidden_layers: int = 28,
hidden_act: str = "relu2",
rms_norm_eps: float = 1e-5,
layers_block_type: list[str] | None = None,
**kwargs,
) -> None:
# Each decoder block is an attention layer followed by an MLP layer
if layers_block_type is None:
layers_block_type = ["full_attention", "mlp"] * num_hidden_layers
super().__init__(
num_hidden_layers=2 * num_hidden_layers,
hybrid_override_pattern="*-" * num_hidden_layers,
layers_block_type=layers_block_type,
mlp_hidden_act=hidden_act,
layer_norm_epsilon=rms_norm_eps,
**kwargs,
@@ -46,7 +48,6 @@ class Cosmos3EdgeTextConfig(NemotronHConfig):
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
config_dict.pop("hybrid_override_pattern", None)
return config_dict
@@ -90,11 +90,6 @@ class MlpProjectorConfig(PreTrainedConfig):
@strict
class DeepseekVLV2TextConfig(DeepseekV2Config):
# DeepSeek-VL2 checkpoints (e.g. deepseek-vl2-small) omit ``vocab_size`` and
# rely on the original DeepSeek-VL2 default of 102400; the generic
# Transformers ``DeepseekV2Config`` incorrectly defaults it to 32000.
# Upstream fix: https://github.com/huggingface/transformers/pull/48159
vocab_size: int = 102400
kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank
@@ -1,44 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PreTrainedConfig
from transformers.models.gemma4.configuration_gemma4 import Gemma4VisionConfig
def _init_text_config(self: PreTrainedConfig, **kwargs: Any) -> None:
PreTrainedConfig.__init__(self, **kwargs)
# DiffusionGemma always uses MoE and K=V sharing for full_attention
# layers. The HF reference removed these config fields entirely.
if getattr(self, "num_experts", None):
self.enable_moe_block = True
self.attention_k_eq_v = True
class DiffusionGemmaTextConfig(PreTrainedConfig):
model_type = "diffusion_gemma_text"
def __init__(self, **kwargs: Any):
_init_text_config(self, **kwargs)
class DiffusionGemmaConfig(PreTrainedConfig):
model_type = "diffusion_gemma"
def __init__(
self,
text_config: dict[str, Any] | None = None,
canvas_length: int = 256,
self_conditioning_size: int | None = None,
**kwargs: Any,
):
self.text_config = DiffusionGemmaTextConfig(**(text_config or {}))
self.canvas_length = canvas_length
self.self_conditioning_size = self_conditioning_size
vision_config = kwargs.pop("vision_config", None)
if isinstance(vision_config, dict):
self.vision_config = Gemma4VisionConfig(**vision_config)
else:
self.vision_config = vision_config
self.audio_config = None
PreTrainedConfig.__init__(self, **kwargs)
-34
View File
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-layer config resolution for Gemma4 and its variants."""
from copy import copy
from transformers import PreTrainedConfig
def gemma4_layer_config(
text_config: PreTrainedConfig, layer_idx: int
) -> PreTrainedConfig:
"""The Gemma4 text config as it applies to one layer.
Gemma4 uses a larger head dimension on its full attention layers than on its
sliding ones, and with `attention_k_eq_v` it uses more KV heads there too.
Transformers >= 5.15.0 says so in the config itself; before that the values
are flat attributes that `layer_types` picks between, so resolve them here.
The result is homogeneous either way, so callers read `head_dim` and
`num_key_value_heads` off it without caring which version they are on.
"""
if getattr(text_config, "is_heterogeneous", False):
return text_config.per_layer_config[layer_idx]
layer = copy(text_config)
if text_config.layer_types[layer_idx] == "full_attention":
global_head_dim = getattr(text_config, "global_head_dim", None)
layer.head_dim = global_head_dim or text_config.head_dim
if getattr(text_config, "attention_k_eq_v", False):
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
layer.num_key_value_heads = (
global_kv_heads or text_config.num_key_value_heads
)
return layer
@@ -1,94 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import transformers
class Granite4VisionConfig(transformers.PreTrainedConfig):
"""Configuration for Granite 4 Vision model.
This config is needed because the granite4_vision model type is not yet
in the transformers version pinned by vLLM. Once transformers adds native
support, this file can be removed and the _CONFIG_REGISTRY entry dropped.
"""
model_type = "granite4_vision"
is_composition = False
def __init__(
self,
vision_config: dict[str, Any] | None = None,
text_config: dict[str, Any] | None = None,
image_token_index: int = 100352,
image_seq_length: int = 576,
image_grid_pinpoints: list[list[int]] | None = None,
vision_feature_select_strategy: str = "full",
vision_feature_layer: int | list[int] = -2,
projector_hidden_act: str = "gelu",
projector_dropout: float = 0.1,
downsample_rate: str | None = None,
use_image_newline_parameter: bool = True,
deepstack_layer_map: list[list[int]] | None = None,
use_spatial_sampling: bool = False,
spatial_stride: int = 2,
spatial_vision_layer: int = -1,
spatial_target_layers: list[int] | None = None,
# Hub aliases — base model config uses different field names
vision_layer_to_llm_layer: list[list[int]] | None = None,
use_checkerboard_sampling: bool | None = None,
checkerboard_stride: int | None = None,
checkerboard_vision_layer: int | None = None,
checkerboard_llm_layers: list[int] | None = None,
**kwargs: Any,
):
self.image_token_index = image_token_index
self.image_seq_length = image_seq_length
self.image_grid_pinpoints = image_grid_pinpoints or []
self.vision_feature_select_strategy = vision_feature_select_strategy
self.vision_feature_layer = vision_feature_layer
self.projector_hidden_act = projector_hidden_act
self.projector_dropout = projector_dropout
self.downsample_rate = downsample_rate
self.use_image_newline_parameter = use_image_newline_parameter
self.deepstack_layer_map = deepstack_layer_map or vision_layer_to_llm_layer
self.use_spatial_sampling = (
use_spatial_sampling
if use_checkerboard_sampling is None
else use_checkerboard_sampling
)
self.spatial_stride = (
spatial_stride if checkerboard_stride is None else checkerboard_stride
)
self.spatial_vision_layer = (
spatial_vision_layer
if checkerboard_vision_layer is None
else checkerboard_vision_layer
)
self.spatial_target_layers = (
spatial_target_layers or checkerboard_llm_layers or [0, 10, 20, 30]
)
if vision_config is None:
vision_config = {}
if text_config is None:
text_config = {}
vision_model_type = vision_config.get("model_type", "siglip_vision_model")
if vision_model_type in transformers.CONFIG_MAPPING:
self.vision_config = transformers.CONFIG_MAPPING[vision_model_type](
**vision_config
)
else:
self.vision_config = transformers.PreTrainedConfig(**vision_config)
text_model_type = text_config.get("model_type", "granite")
if text_model_type in transformers.CONFIG_MAPPING:
self.text_config = transformers.CONFIG_MAPPING[text_model_type](
**text_config
)
else:
self.text_config = transformers.PreTrainedConfig(**text_config)
super().__init__(**kwargs)
-185
View File
@@ -1,185 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
class HYV3Config(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a [`HYV3Model`].
It is used to instantiate a HYV3 model (HY V3 MoE language model) according to
the specified arguments.
Configuration objects inherit from [`PreTrainedConfig`] and can be used to
control the model outputs. Read the documentation from [`PreTrainedConfig`]
for more information.
Args:
vocab_size (`int`, *optional*, defaults to 120832):
Vocabulary size of the model.
hidden_size (`int`, *optional*, defaults to 4096):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 13312):
Dimension of the dense FFN intermediate representations.
num_hidden_layers (`int`, *optional*, defaults to 80):
Number of hidden layers in the Transformer decoder.
num_attention_heads (`int`, *optional*, defaults to 64):
Number of attention heads for each attention layer.
num_key_value_heads (`int`, *optional*, defaults to 8):
Number of key-value heads for grouped-query attention.
head_dim (`int`, *optional*, defaults to 128):
Dimension per attention head.
hidden_act (`str`, *optional*, defaults to `"silu"`):
Activation function used in FFN layers.
max_position_embeddings (`int`, *optional*, defaults to 131072):
Maximum sequence length supported by the model.
initializer_range (`float`, *optional*, defaults to 0.006):
Standard deviation of the truncated normal initializer for weight
initialization.
rms_norm_eps (`float`, *optional*, defaults to 1e-5):
Epsilon for RMS normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether to use KV cache for decoding.
pad_token_id (`int`, *optional*):
Padding token id.
bos_token_id (`int`, *optional*):
Beginning-of-sequence token id.
eos_token_id (`int` or `List[int]`, *optional*):
End-of-sequence token id(s).
rope_parameters (`dict`, *optional*):
The parameters of the RoPE embeddings.
qk_norm (`bool`, *optional*, defaults to `True`):
Whether to apply RMSNorm to query and key states before attention.
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether to tie input and output embedding weights.
enable_attention_fp32_softmax (`bool`, *optional*, defaults to `False`):
Whether to upcast attention softmax to float32. Note: the eager attention
path always computes softmax in float32 regardless of this setting; this
flag is reserved for future use with custom attention backends.
enable_lm_head_fp32 (`bool`, *optional*, defaults to `True`):
Whether to upcast the LM head computation to float32.
num_experts (`int`, *optional*, defaults to 192):
Total number of MoE experts.
num_experts_per_tok (`int`, *optional*, defaults to 8):
Number of experts selected per token (top-k routing).
num_shared_experts (`int`, *optional*, defaults to 1):
Number of always-active shared experts combined into a single MLP.
expert_hidden_dim (`int`, *optional*, defaults to 1536):
Intermediate dimension of each individual MoE expert.
moe_router_enable_expert_bias (`bool`, *optional*, defaults to `True`):
Whether to use per-expert load-balancing bias in the router.
moe_router_use_sigmoid (`bool`, *optional*, defaults to `True`):
Whether to use sigmoid (instead of softmax) for router scoring.
route_norm (`bool`, *optional*, defaults to `True`):
Whether to normalize routing scores when using sigmoid routing.
router_scaling_factor (`float`, *optional*):
Optional multiplicative scaling factor applied to routing scores.
use_grouped_mm (`bool`, *optional*, defaults to `False`):
Whether to use grouped GEMM for expert computation (not yet implemented).
enable_moe_fp32_combine (`bool`, *optional*, defaults to `False`):
Whether to accumulate expert outputs in float32.
first_k_dense_replace (`int`, *optional*, defaults to 1):
Number of initial decoder layers that use a dense FFN instead of MoE.
output_router_logits (`bool`, *optional*, defaults to `False`):
Whether to output router logits from each MoE layer. Useful for computing
auxiliary load-balancing loss during training. Disabled by default to avoid
the memory overhead of storing per-layer router tensors during inference.
Example:
```python
>>> from transformers import HYV3Config, HYV3Model
>>> config = HYV3Config()
>>> model = HYV3Model(config)
```
"""
model_type = "hy_v3"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size=120832,
hidden_size=4096,
intermediate_size=13312,
num_hidden_layers=80,
num_attention_heads=64,
num_key_value_heads=8,
head_dim=128,
hidden_act="silu",
max_position_embeddings=131072,
initializer_range=0.006,
rms_norm_eps=1e-5,
use_cache=True,
pad_token_id=None,
bos_token_id=None,
eos_token_id=None,
rope_parameters: dict[str, Any] | None = None,
qk_norm=True,
tie_word_embeddings=False,
enable_attention_fp32_softmax=False,
enable_lm_head_fp32=True,
# MoE specific
num_experts=192,
num_experts_per_tok=8,
num_shared_experts=1,
expert_hidden_dim=1536,
moe_router_enable_expert_bias=True,
moe_router_use_sigmoid=True,
route_norm=True,
router_scaling_factor=None,
use_grouped_mm=False,
enable_moe_fp32_combine=False,
# Dense/MoE layer control
first_k_dense_replace=1,
output_router_logits=False,
**kwargs,
):
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.hidden_act = hidden_act
self.max_position_embeddings = max_position_embeddings
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
rope_theta = kwargs.pop("rope_theta", 11158840.0)
if rope_parameters is None:
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
self.rope_parameters = rope_parameters
self.qk_norm = qk_norm
self.tie_word_embeddings = tie_word_embeddings
self.enable_lm_head_fp32 = enable_lm_head_fp32
self.enable_attention_fp32_softmax = enable_attention_fp32_softmax
# MoE specific
self.num_experts = num_experts
self.num_experts_per_tok = num_experts_per_tok
self.num_shared_experts = num_shared_experts
self.expert_hidden_dim = expert_hidden_dim
self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
self.moe_router_use_sigmoid = moe_router_use_sigmoid
self.route_norm = route_norm
self.use_grouped_mm = use_grouped_mm
self.router_scaling_factor = router_scaling_factor
self.enable_moe_fp32_combine = enable_moe_fp32_combine
# Dense/MoE layer control
self.first_k_dense_replace = first_k_dense_replace
self.output_router_logits = output_router_logits
if eos_token_id is not None and isinstance(eos_token_id, int):
eos_token_id = [eos_token_id]
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@@ -1,277 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright 2025 NAVER Cloud HyperCLOVA team
#
# Copyright 2025 NAVER Cloud HyperCLOVA team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""HyperCLOVA X model configuration."""
from transformers.configuration_utils import PreTrainedConfig
class HyperCLOVAXConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a
[`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model
according to the specified arguments, defining the model architecture.
Configuration objects inherit from [`PreTrainedConfig`] and can be used
to control the model outputs. Read the documentation from
[`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32000):
Vocabulary size of the HyperCLOVAX model. Defines the number of
different tokens that can be represented by the `input_ids`
passed when calling [`HyperCLOVAXModel`]
hidden_size (`int`, *optional*, defaults to 4096):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 11008):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 32):
Number of hidden layers in the Transformer decoder.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the
Transformer decoder.
num_key_value_heads (`int`, *optional*):
This is the number of key_value heads that should be used to
implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use
Multi Head Attention (MHA), if `num_key_value_heads=1` the model
will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each
group key and value head should be constructed by meanpooling all
the original heads within that group. For more details checkout
[this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
specified, will default to `num_attention_heads`.
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
The non-linear activation function (function or string) in the
decoder.
max_position_embeddings (`int`, *optional*, defaults to 2048):
The maximum sequence length that this model might ever be used
with.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for
initializing all weight matrices.
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
The epsilon used by the rms normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values
attentions (not used by all models). Only relevant if
`config.is_decoder=True`.
pad_token_id (`int`, *optional*):
Padding token id.
bos_token_id (`int`, *optional*, defaults to 1):
Beginning of stream token id.
eos_token_id (`int`, *optional*, defaults to 2):
End of stream token id.
pretraining_tp (`int`, *optional*, defaults to 1):
Experimental feature. Tensor parallelism rank used during
pretraining. Please refer to [this document](https://huggingface.
co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism)
to understand more about it. This value is necessary to ensure
exact reproducibility of the pretraining results. Please refer to
[this issue](https://github.com/pytorch/pytorch/issues/76232).
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether to tie weight embeddings
rope_theta (`float`, *optional*, defaults to 10000.0):
The base period of the RoPE embeddings.
rope_scaling (`Dict`, *optional*):
Dictionary containing the scaling configuration for the RoPE
embeddings. NOTE: if you apply new rope type and you expect the
model to work on longer `max_position_embeddings`, we recommend
you to update this value accordingly.
Expected contents:
`rope_type` (`str`):
The sub-variant of RoPE to use. Can be one of ['default',
'linear', 'dynamic', 'yarn', 'longrope', 'llama3'], with
'default' being the original RoPE implementation.
`factor` (`float`, *optional*):
Used with all rope types except 'default'. The scaling
factor to apply to the RoPE embeddings. In most scaling
types, a `factor` of x will enable the model to handle
sequences of length x * original maximum pre-trained
length.
`original_max_position_embeddings` (`int`, *optional*):
Used with 'dynamic', 'longrope' and 'llama3'. The
original max position embeddings used during pretraining.
`attention_factor` (`float`, *optional*):
Used with 'yarn' and 'longrope'. The scaling factor to be
applied on the attention computation. If unspecified, it
defaults to value recommended by the implementation, using
the `factor` field to infer the suggested value.
`beta_fast` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for
extrapolation (only) in the linear ramp function. If
unspecified, it defaults to 32.
`beta_slow` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for
interpolation (only) in the linear ramp function. If
unspecified, it defaults to 1.
`short_factor` (`List[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be
applied to short contexts (<
`original_max_position_embeddings`). Must be a list of
numbers with the same length as the hidden size divided
by the number of attention heads divided by 2
`long_factor` (`List[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be
applied to long contexts (<
`original_max_position_embeddings`). Must be a list of
numbers with the same length as the hidden size divided
by the number of attention heads divided by 2
`low_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to low
frequency components of the RoPE
`high_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to high
frequency components of the RoPE
attention_bias (`bool`, *optional*, defaults to `False`):
Whether to use a bias in the query, key, value and output
projection layers during self-attention.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
mlp_bias (`bool`, *optional*, defaults to `False`):
Whether to use a bias in up_proj, down_proj and gate_proj layers
in the MLP layers.
head_dim (`int`, *optional*):
The attention head dimension. If None, it will default to
hidden_size // num_heads
embedding_multiplier (`float`, *optional*, defaults to `None`):
Multiplier applied to the embedding weights. If `None`, it is
equivalent to `1.0`.
logits_scaling (`float`, *optional*, defaults to `None`):
Scaling factor for logits. If `None`, it is equivalent to `1.0`.
attention_multiplier (`float`, *optional*, defaults to `None`):
Multiplier applied to the attention weights. If `None`, it is
equivalent to `self.head_dim ** -0.5`.
residual_multiplier (`float`, *optional*, defaults to `None`):
Scaling factor for residual connections. If `None`, it is
equivalent to `1.0`.
use_post_norm (`bool`, *optional*, defaults to `True`):
Determines whether to apply Peri-Layer Normalization. Set to
False to disable this feature.
rope_parameters (`dict`, *optional*):
Dictionary containing the RoPE parameters used by vLLM's
`get_rope`. When provided, takes precedence over `rope_theta`
and `rope_scaling`. If `None`, it is derived from `rope_theta`
and `rope_scaling` automatically.
"""
model_type = "hyperclovax"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size=32000,
hidden_size=4096,
intermediate_size=11008,
num_hidden_layers=32,
num_attention_heads=32,
num_key_value_heads=None,
hidden_act="silu",
max_position_embeddings=2048,
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
pad_token_id=None,
bos_token_id=1,
eos_token_id=2,
pretraining_tp=1,
tie_word_embeddings=False,
rope_theta=10000.0,
rope_scaling=None,
attention_bias=False,
attention_dropout=0.0,
mlp_bias=False,
head_dim=None,
embedding_multiplier=None, # mup
logits_scaling=None, # mup
attention_multiplier=None, # mup
residual_multiplier=None, # mup
use_post_norm=True, # post-norm(peri-LN)
rope_parameters=None,
auto_map=None,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.pretraining_tp = pretraining_tp
self.use_cache = use_cache
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.mlp_bias = mlp_bias
self.head_dim = (
head_dim
if head_dim is not None
else self.hidden_size // self.num_attention_heads
)
# Derive rope_parameters for vLLM's get_rope() from rope_theta /
# rope_scaling, unless the caller already provided rope_parameters.
if rope_parameters is None:
if rope_scaling is not None:
# Shallow-copy to avoid mutating the caller's dict.
rope_parameters = dict(rope_scaling)
# BC: 'type' field -> 'rope_type', remove stale key.
if "type" in rope_parameters:
rope_parameters.setdefault("rope_type", rope_parameters.pop("type"))
else:
rope_parameters = {"rope_type": "default"}
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
self.rope_parameters = rope_parameters
# BC: keep self.rope_scaling consistent for HF serialization.
if self.rope_scaling is not None and "type" in self.rope_scaling:
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
# mup
self.embedding_multiplier = (
embedding_multiplier if embedding_multiplier is not None else 1.0
)
self.logits_scaling = logits_scaling if logits_scaling is not None else 1.0
self.attention_multiplier = (
attention_multiplier
if attention_multiplier is not None
else self.head_dim**-0.5
)
self.residual_multiplier = (
residual_multiplier if residual_multiplier is not None else 1.0
)
# post-norm (Peri-LN)
self.use_post_norm = use_post_norm
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
auto_map=auto_map,
**kwargs,
)
-131
View File
@@ -1,131 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi-K2.5 Model Configuration.
This configuration supports video-chunk as an internal modality type.
A video-chunk is the smallest independently processable unit of video.
"""
from transformers import DeepseekV3Config
from transformers.configuration_utils import PreTrainedConfig
class KimiK25VisionConfig(PreTrainedConfig):
model_type = "kimi_k25_vision"
def __init__(
self,
# Vision Tower
patch_size: int = 14,
init_pos_emb_height: int = 64,
init_pos_emb_width: int = 64,
init_pos_emb_time: int = 4,
pos_emb_type: str = "divided_fixed",
num_attention_heads: int = 16,
num_hidden_layers: int = 27,
hidden_size: int = 1152,
intermediate_size: int = 4304,
merge_kernel_size: tuple[int, int] = (2, 2),
video_attn_type: str = "spatial_temporal",
merge_type: str = "sd2_tpool",
# MM Projector
mm_projector_type: str = "patchmerger",
mm_hidden_size: int | None = None,
projector_hidden_act: str = "gelu",
projector_ln_eps: float = 1e-5,
**kwargs,
):
super().__init__(**kwargs)
# Vision Tower
self.patch_size = patch_size
self.init_pos_emb_height = init_pos_emb_height
self.init_pos_emb_width = init_pos_emb_width
self.init_pos_emb_time = init_pos_emb_time
self.pos_emb_type = pos_emb_type
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.merge_kernel_size = merge_kernel_size
self.video_attn_type = video_attn_type
self.merge_type = merge_type
# MM Projector
self.mm_projector_type = mm_projector_type
if mm_hidden_size is not None:
self.mm_hidden_size = mm_hidden_size
else:
self.mm_hidden_size = hidden_size
self.projector_hidden_act = projector_hidden_act
self.projector_ln_eps = projector_ln_eps
class KimiK25Config(PreTrainedConfig):
"""Kimi-K2.5 model configuration.
Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks.
A video-chunk consists of multiple consecutive frames
that are processed together with temporal pooling.
Args:
vision_config: Configuration for the vision tower and projector.
text_config: Configuration for the text model (DeepseekV3).
ignore_index: The ignore index for the loss function.
media_placeholder_token_id: The token ID for media placeholders.
pad_token_id: The token ID for padding.
"""
model_type = "kimi_k25"
def __init__(
self,
vision_config: dict | KimiK25VisionConfig | None = None,
text_config: dict | DeepseekV3Config | None = None,
ignore_index: int = -100,
media_placeholder_token_id: int = 163605,
pad_token_id: int = 0,
use_unified_vision_chunk: bool = False,
video_placeholder: str = "<|kimi_k25_video_placeholder|>",
**kwargs,
):
# Vision config
if vision_config is None:
self.vision_config = KimiK25VisionConfig()
elif isinstance(vision_config, dict):
self.vision_config = KimiK25VisionConfig(**vision_config)
else:
self.vision_config = vision_config
# Text config
if text_config is None:
self.text_config = DeepseekV3Config()
elif isinstance(text_config, dict):
self.text_config = DeepseekV3Config(**text_config)
else:
self.text_config = text_config
# Set mm_hidden_size to text hidden size if not explicitly set
if self.vision_config.mm_hidden_size == self.vision_config.hidden_size:
self.vision_config.mm_hidden_size = self.text_config.hidden_size
# Other config
self.ignore_index = ignore_index
self.media_placeholder_token_id = media_placeholder_token_id
self.use_unified_vision_chunk = use_unified_vision_chunk
self.video_placeholder = video_placeholder
# Propagate quantization config from text model
if getattr(self.text_config, "quantization_config", None) is not None:
self.quantization_config = self.text_config.quantization_config
super().__init__(pad_token_id=pad_token_id, **kwargs)
@property
def hidden_size(self) -> int:
"""Get hidden size from text config for compatibility."""
return self.text_config.hidden_size
@property
def vocab_size(self) -> int:
"""Get vocab size from text config for compatibility."""
return self.text_config.vocab_size
-164
View File
@@ -1,164 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers.configuration_utils import PreTrainedConfig
class Lfm2MoeConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a [`Lfm2MoeModel`]. It is used to instantiate a LFM2 Moe
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the LFM2-8B-A1B model.
e.g. [LiquidAI/LFM2-8B-A1B](https://huggingface.co/LiquidAI/LFM2-8B-A1B)
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 65536):
Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
`inputs_ids` passed when calling [`Lfm2Model`]
hidden_size (`int`, *optional*, defaults to 2048):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 7168):
Dimension of the MLP representations.
moe_intermediate_size (`int`, *optional*, defaults to 1792):
Intermediate size of the routed expert.
num_hidden_layers (`int`, *optional*, defaults to 32):
Number of hidden layers in the Transformer decoder.
pad_token_id (`int`, *optional*, defaults to 0):
Padding token id.
bos_token_id (`int`, *optional*, defaults to 1):
Beginning of stream token id.
eos_token_id (`int`, *optional*, defaults to 2):
End of stream token id.
tie_word_embeddings (`bool`, *optional*, defaults to `True`):
Whether to tie weight embeddings
rope_parameters (`dict`, *optional*):
The parameters of the RoPE embeddings.
max_position_embeddings (`int`, *optional*, defaults to 128000):
The maximum sequence length that this model might ever be used with.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
norm_eps (`float`, *optional*, defaults to 1e-05):
The epsilon used by the rms normalization layers.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the Transformer decoder.
num_key_value_heads (`int`, *optional*, defaults to 8):
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details, check out [this
paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
`num_attention_heads`.
conv_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in the conv layers.
conv_L_cache (`int`, *optional*, defaults to 3):
L_cache dim in the conv layers.
num_dense_layers (`int`, *optional*, defaults to 2):
Number of dense Lfm2MoeMLP layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
num_experts_per_tok (`int`, *optional*, defaults to 4):
Number of selected experts.
num_experts (`int`, *optional*, defaults to 32):
Number of routed experts.
use_expert_bias (`bool`, *optional*, defaults to `True`):
Whether to use the expert bias on the routing weights.
routed_scaling_factor (`float`, *optional*, defaults to 1.0):
Scaling factor for routed experts in MoE models.
norm_topk_prob (`bool`, *optional*, defaults to `True`):
Whether to normalize the topk probabilities.
layer_types (`Optional`, *optional*):
Type of each layers.
```python
>>> from transformers import Lfm2MoeModel, Lfm2MoeConfig
>>> # Initializing a LFM2 Moe model
>>> configuration = Lfm2MoeConfig()
>>> # Initializing a model from the LFM2-8B-A1B style configuration
>>> model = Lfm2MoeModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
""" # noqa: E501
model_type = "lfm2_moe"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size: int = 65536,
hidden_size: int = 2048,
intermediate_size: int = 7168,
moe_intermediate_size: int = 1792,
num_hidden_layers: int = 32,
pad_token_id: int = 0,
bos_token_id: int = 1,
eos_token_id: int = 2,
tie_word_embeddings: bool = True,
rope_parameters: dict[str, Any] | None = None,
max_position_embeddings: int = 128_000,
use_cache: bool = True,
norm_eps: float = 0.00001,
num_attention_heads: int = 32,
num_key_value_heads: int = 8,
conv_bias: bool = False,
conv_L_cache: int = 3,
num_dense_layers: int = 2,
num_experts_per_tok: int = 4,
num_experts: int = 32,
use_expert_bias: bool = True,
routed_scaling_factor: float = 1.0,
norm_topk_prob: bool = True,
layer_types: list[str] | None = None,
**kwargs,
):
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
rope_theta = kwargs.pop("rope_theta", 1000000.0)
if rope_parameters is None:
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
self.rope_parameters = rope_parameters
self.max_position_embeddings = max_position_embeddings
self.use_cache = use_cache
self.norm_eps = norm_eps
# attn operator config
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
# custom operator config
self.conv_bias = conv_bias
self.conv_L_cache = conv_L_cache
# moe config
self.num_dense_layers = num_dense_layers
self.moe_intermediate_size = moe_intermediate_size
self.num_experts_per_tok = num_experts_per_tok
self.num_experts = num_experts
self.use_expert_bias = use_expert_bias
self.routed_scaling_factor = routed_scaling_factor
self.norm_topk_prob = norm_topk_prob
self.layer_types = layer_types
tie_word_embeddings = kwargs.get(
"tie_embedding", tie_word_embeddings
) # to fit original config keys
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
__all__ = ["Lfm2MoeConfig"]
-219
View File
@@ -1,219 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 HuggingFace Inc. team. All rights reserved.
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Nemotron model configuration."""
from transformers import PreTrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class NemotronConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a
[`NemotronModel`]. It is used to instantiate a Nemotron model
according to the specified arguments, defining the model architecture.
Instantiating a configuration with the defaults will yield a similar
configuration to that of the Nemotron-8B.
Configuration objects inherit from [`PreTrainedConfig`] and can be
used to control the model outputs. Read the documentation from
[`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 256000):
Vocabulary size of the Nemotron model. Defines the number of
different tokens that can be represented by the
`inputs_ids` passed when calling [`NemotronModel`]
hidden_size (`int`, *optional*, defaults to 6144):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 24576):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 32):
Number of hidden layers in the Transformer decoder.
num_attention_heads (`int`, *optional*, defaults to 48):
Number of attention heads for each attention layer in the
Transformer decoder.
head_dim (`int`, *optional*):
Projection weights dimension in multi-head attention. Set to
hidden_size // num_attention_heads if None
num_key_value_heads (`int`, *optional*):
This is the number of key_value heads that should be used to
implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use
Multi Head Attention (MHA), if
`num_key_value_heads=1 the model will use Multi Query Attention
(MQA) otherwise GQA is used. When converting a multi-head
checkpoint to a GQA checkpoint, each group key and value
head should be constructed by meanpooling all the original
heads within that group. For more details checkout
[this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
is not specified, will default to `num_attention_heads`.
hidden_act (`str` or `function`, *optional*, defaults to `"relu2"`):
The non-linear activation function (function or string) in the
decoder.
max_position_embeddings (`int`, *optional*, defaults to 4096):
The maximum sequence length that this model might ever be used
with.
initializer_range (`float`, *optional*, defaults to 0.0134):
The standard deviation of the truncated_normal_initializer for
initializing all weight matrices.
norm_eps (`float`, *optional*, defaults to 1e-05):
The epsilon used by the normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values
attentions (not used by all models). Only relevant if
`config.is_decoder=True`.
pad_token_id (`int`, *optional*):
Padding token id.
bos_token_id (`int`, *optional*, defaults to 2):
Beginning of stream token id.
eos_token_id (`int`, *optional*, defaults to 3):
End of stream token id.
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether to tie weight embeddings
rope_parameters (`dict`, *optional*):
The parameters of the RoPE embeddings. Expected contents:
`rope_theta` (`float`): The base period of the RoPE embeddings.
`rope_type` (`str`):
The sub-variant of RoPE to use. Can be one of ['default', 'linear',
'dynamic', 'yarn', 'longrope', 'llama3'], with 'default' being the
original RoPE implementation.
`partial_rotary_factor` (`float`, *optional*, defaults to 0.5):
Percentage of the query and keys which will have rotary embedding.
attention_bias (`bool`, *optional*, defaults to `False`):
Whether to use a bias in the query, key, value and output
projection layers during self-attention.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
mlp_bias (`bool`, *optional*, defaults to `False`):
Whether to use a bias in up_proj and down_proj layers in the MLP
layers.
```python
>>> from transformers import NemotronModel, NemotronConfig
>>> # Initializing a Nemotron nemotron-15b style configuration
>>> configuration = NemotronConfig()
>>> # Initializing a model from the nemotron-15b style configuration
>>> model = NemotronModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
"""
model_type = "nemotron"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size=256000,
hidden_size=6144,
intermediate_size=24576,
num_hidden_layers=32,
num_attention_heads=48,
head_dim=None,
num_key_value_heads=None,
hidden_act="relu2",
max_position_embeddings=4096,
initializer_range=0.0134,
norm_eps=1e-5,
use_cache=True,
pad_token_id=None,
bos_token_id=2,
eos_token_id=3,
tie_word_embeddings=False,
rope_parameters=None,
attention_bias=False,
attention_dropout=0.0,
mlp_bias=False,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
head_dim = head_dim or kwargs.get("kv_channels")
self.head_dim = (
head_dim if head_dim is not None else (hidden_size // num_attention_heads)
)
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.norm_eps = norm_eps
self.use_cache = use_cache
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
rope_scaling = kwargs.pop("rope_scaling", None)
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
rope_theta = kwargs.pop("rope_theta", 10000.0)
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
# for backward compatibility
partial_rotary_factor = (
kwargs.get("rope_percent")
or kwargs.get("rope_percentage")
or kwargs.get("partial_rotary_factor")
or 0.5
)
if "partial_rotary_factor" not in rope_parameters:
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
self.rope_parameters = rope_parameters
self._rope_parameters_validation()
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.mlp_bias = mlp_bias
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
def _rope_parameters_validation(self):
"""Validate the `rope_parameters` configuration."""
if self.rope_parameters is None:
return
rope_type: str | None = self.rope_parameters.get("rope_type", None)
factor: float | None = self.rope_parameters.get("factor", None)
if rope_type not in {"default", "linear", "dynamic"}:
raise ValueError(
"`rope_type` must be one of ['default', 'linear', 'dynamic'], "
f"got {rope_type}"
)
if rope_type != "default":
if factor is None:
raise ValueError(
"If `rope_type` is not 'default', `rope_parameters` "
"must include a `factor` field. Got `None`."
)
if not isinstance(factor, float) or factor <= 1.0:
raise ValueError(
"`rope_parameters`'s factor field must be a float > 1, got "
f"{factor}"
)
@@ -1,288 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 HuggingFace Inc. team. All rights reserved.
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""NemotronH model configuration."""
import regex as re
from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class NemotronHConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a
[`NemotronHModel`]. It is used to instantiate a NemotronH model according
to the specified arguments, defining the model architecture. Instantiating
a configuration with the defaults will yield a similar configuration to
that of the NemotronH-v0.1 model.
Args:
vocab_size (`int`, *optional*, defaults to 131072):
Vocabulary size of the NemotronH model. Defines the number of
different tokens that can be represented by the `inputs_ids`
passed when calling [`NemotronHModel`]
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether the model's input and output word embeddings should be
tied. Note that this is only relevant if the model has an output
word embedding layer.
hidden_size (`int`, *optional*, defaults to 4096):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 21504):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 52):
Number of hidden layers in the Transformer encoder.
hybrid_override_pattern (`str`, *optional*, defaults to
`"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
The pattern of the hybrid model. The pattern is a string of
characters where each character represents
M: Mamba2, *: Attention, -: MLP
mtp_hybrid_override_pattern (`str`, *optional*, defaults to `"*E"`):
The pattern of the MTP layers.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the
Transformer encoder.
attention_head_dim (`int`, *optional*, defaults to 128):
Dimension of each attention head.
num_key_value_heads (`int`, *optional*, defaults to 8):
This is the number of key_value heads that should be used to
implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use
Multi Head Attention (MHA), if `num_key_value_heads=1` the model
will use Multi Query Attention (MQA) otherwise GQA is used.
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
The non-linear activation function in the MLP layers.
attention_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in attention layers.
mlp_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in MLP layers.
use_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in the model.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for
initializing all weight matrices.
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
The epsilon used by the layer normalization layers.
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
Whether or not residuals should be in `float32`. If set to `False`
residuals will keep the same `dtype` as the rest of the model.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values
attentions (not used by all models). Only relevant if
`config.is_decoder=True`.
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
Number of prompt logits to calculate during generation. If `None`,
all logits will be calculated. If an integer value, only last
`num_logits_to_keep` logits will be calculated.
pad_token_id (`int`, *optional*, defaults to 0):
The id of the padding token.
bos_token_id (`int`, *optional*, defaults to 1):
The id of the "beginning-of-sequence" token.
eos_token_id (`int`, *optional*, defaults to 2):
The id of the "end-of-sequence" token.
sliding_window (`int`, *optional*, defaults to None):
Sliding window attention window size.
max_position_embeddings (`int`, *optional*, defaults to 4096):
The maximum sequence length that this model might ever be used
with.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
hidden_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the hidden states.
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
Flag indicating whether or not to use the fast mamba kernels.
These are available only if `mamba-ssm` and `causal-conv1d`
are installed, and the mamba modules are running on a CUDA device.
ssm_state_size (`int`, *optional*, defaults to 128):
The dimension of the mamba state space latents.
mamba_num_heads (`int`, *optional*, defaults to 128):
Number of heads in Mamba layers.
mamba_n_groups (`int`, *optional*, defaults to 8):
Number of groups in Mamba layers.
mamba_head_dim (`int`, *optional*, defaults to 64):
Dimension of each Mamba head.
mamba_d_conv (`int`, *optional*, defaults to 4):
The size of the mamba convolution kernel.
mamba_expand (`int`, *optional*, defaults to 2):
Expanding factor used to determine the mamba intermediate size.
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
The non-linear activation function in the Mamba layers.
mamba_dt_min (`float`, *optional*, defaults to 0.001):
Minimum value for the time step in Mamba.
mamba_dt_max (`float`, *optional*, defaults to 0.1):
Maximum value for the time step in Mamba.
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
Limits for the time step in Mamba.
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
Floor value for time step initialization in Mamba.
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
Whether to use bias in the convolution layer of the mamba mixer
block.
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in the input and output projections of the
mamba mixer block.
mamba_chunk_size (`int`, *optional*, defaults to 256):
Size of chunks for Mamba processing.
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
Whether to rescale the pre-normalization residual connections.
"""
model_type = "nemotron_h"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size=131072,
tie_word_embeddings=False,
hidden_size=4096,
intermediate_size=21504,
num_hidden_layers=52,
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
mtp_hybrid_override_pattern="*E",
num_attention_heads=32,
head_dim=128,
num_key_value_heads=8, # nemo: num_query_groups
mlp_hidden_act="relu2",
attention_bias=False,
mlp_bias=False,
use_bias=False,
initializer_range=0.02, # nemo: init_method_std
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
residual_in_fp32=False, # Megatron Core default value
use_cache=True,
num_logits_to_keep=1,
pad_token_id=0,
bos_token_id=1,
eos_token_id=2,
sliding_window=None,
max_position_embeddings=4096,
attention_dropout=0.0,
hidden_dropout=0.0, # * ADDED
use_mamba_kernels=True,
ssm_state_size=128, # mamba_state_size
mamba_num_heads=128,
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
mamba_head_dim=64,
mamba_d_conv=4,
mamba_expand=2,
mamba_hidden_act="silu",
mamba_dt_min=0.001,
mamba_dt_max=0.1,
mamba_dt_limit=(0.0, float("inf")),
mamba_dt_init_floor=1e-4,
mamba_conv_bias=True,
mamba_proj_bias=False,
mamba_chunk_size=256,
rescale_prenorm_residual=True,
n_routed_experts=8,
n_shared_experts=1,
moe_intermediate_size=7688,
moe_shared_expert_intermediate_size=7688,
moe_latent_size=None,
num_experts_per_tok=2,
routed_scaling_factor=1.0,
n_group=1,
topk_group=1,
norm_topk_prob=True,
**kwargs,
):
self.vocab_size = vocab_size
self.tie_word_embeddings = tie_word_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.hybrid_override_pattern = hybrid_override_pattern
self.mtp_hybrid_override_pattern = mtp_hybrid_override_pattern
self.num_attention_heads = num_attention_heads
self.head_dim = head_dim
self.sliding_window = sliding_window
self.max_position_embeddings = max_position_embeddings
self.attention_dropout = attention_dropout
self.hidden_dropout = hidden_dropout
# Validate hybrid_override_pattern
# M: Mamba2, *: Attention, -: MLP
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, (
"hybrid_override_pattern must have same length as num_hidden_layers"
)
assert re.match(r"^[*-ME]+$", self.hybrid_override_pattern), (
"hybrid_override_pattern must only contain characters 'M', '*', '-', or 'E'"
)
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.mlp_hidden_act = mlp_hidden_act
self.attention_bias = attention_bias
self.mlp_bias = mlp_bias
self.use_bias = use_bias
self.initializer_range = initializer_range
self.layer_norm_epsilon = layer_norm_epsilon
self.residual_in_fp32 = residual_in_fp32
self.use_cache = use_cache
self.num_logits_to_keep = num_logits_to_keep
self.use_mamba_kernels = use_mamba_kernels
self.n_groups = mamba_n_groups
self.mamba_head_dim = mamba_head_dim
self.ssm_state_size = ssm_state_size
self.mamba_num_heads = mamba_num_heads
self.conv_kernel = mamba_d_conv
self.expand = mamba_expand
self.mamba_hidden_act = mamba_hidden_act
self.time_step_min = mamba_dt_min
self.time_step_max = mamba_dt_max
self.time_step_limit = mamba_dt_limit
self.time_step_floor = mamba_dt_init_floor
self.use_conv_bias = mamba_conv_bias
self.mamba_proj_bias = mamba_proj_bias
self.chunk_size = mamba_chunk_size
self.rescale_prenorm_residual = rescale_prenorm_residual
self.n_routed_experts = n_routed_experts
self.n_shared_experts = n_shared_experts
self.moe_intermediate_size = moe_intermediate_size
self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size # noqa: E501
self.moe_latent_size = moe_latent_size
self.num_experts_per_tok = num_experts_per_tok
self.routed_scaling_factor = routed_scaling_factor
self.n_group = n_group
self.topk_group = topk_group
self.norm_topk_prob = norm_topk_prob
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@property
def layers_block_type(self):
return [
"mamba"
if self.hybrid_override_pattern[i] == "M"
else "attention"
if self.hybrid_override_pattern[i] == "*"
else "mlp"
if self.hybrid_override_pattern[i] == "-"
else "moe"
for i in range(self.num_hidden_layers)
]
@@ -1,285 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PreTrainedConfig
class OlmoHybridConfig(PreTrainedConfig):
r"""Configuration class for [`OlmoHybridModel`]. It is used to
instantiate an OLMo Hybrid model according to the specified
arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar
configuration to that of the
[allenai/Olmo-Hybrid-7B](https://huggingface.co/allenai/Olmo-Hybrid-7B)
model.
Configuration objects inherit from [`PreTrainedConfig`] and
can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 100352):
Vocabulary size of the OlmoHybrid model. Defines
the number of different tokens that can be
represented by the `inputs_ids` passed when
calling [`OlmoHybridModel`].
hidden_size (`int`, *optional*, defaults to 3840):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*,
defaults to 11008):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*,
defaults to 32):
Number of hidden layers in the Transformer
decoder.
num_attention_heads (`int`, *optional*,
defaults to 30):
Number of attention heads for each attention
layer in the Transformer decoder.
num_key_value_heads (`int`, *optional*):
This is the number of key_value heads that
should be used to implement Grouped Query
Attention. If
`num_key_value_heads=num_attention_heads`,
the model will use Multi Head Attention (MHA),
if `num_key_value_heads=1` the model will use
Multi Query Attention (MQA) otherwise GQA is
used. When converting a multi-head checkpoint
to a GQA checkpoint, each group key and value
head should be constructed by meanpooling all
the original heads within that group. For more
details, check out
[this paper](https://huggingface.co/papers/2305.13245).
If it is not specified, will default to
`num_attention_heads`.
hidden_act (`str` or `function`, *optional*,
defaults to `"silu"`):
The non-linear activation function (function
or string) in the decoder.
max_position_embeddings (`int`, *optional*,
defaults to 65536):
The maximum sequence length that this model
might ever be used with.
initializer_range (`float`, *optional*,
defaults to 0.02):
The standard deviation of the
truncated_normal_initializer for initializing
all weight matrices.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last
key/values attentions (not used by all models).
Only relevant if `config.is_decoder=True`.
pad_token_id (`int`, *optional*,
defaults to 100277):
Padding token id.
bos_token_id (`int`, *optional*):
Beginning of stream token id.
eos_token_id (`int`, *optional*,
defaults to 100257):
End of stream token id.
tie_word_embeddings (`bool`, *optional*,
defaults to `False`):
Whether to tie weight embeddings.
rope_parameters (`RopeParameters`, *optional*):
Dictionary containing the configuration
parameters for the RoPE embeddings. Can be
`None` to disable RoPE.
attention_bias (`bool`, *optional*,
defaults to `False`):
Whether to use a bias in the query, key, value
and output projection layers during
self-attention.
attention_dropout (`float`, *optional*,
defaults to 0.0):
The dropout ratio for the attention
probabilities.
rms_norm_eps (`float`, *optional*,
defaults to 1e-06):
The epsilon used by the rms normalization
layers.
layer_types (`list`, *optional*):
Attention pattern for each layer. Can contain
`"full_attention"` or `"linear_attention"`.
Defaults to linear attention for most layers
with full attention for every 4th layer.
linear_num_key_heads (`int`, *optional*):
Number of key heads for the linear attention
layers. Defaults to `num_attention_heads`.
linear_num_value_heads (`int`, *optional*):
Number of value heads for the linear attention
layers. Defaults to `num_attention_heads`.
linear_key_head_dim (`int`, *optional*):
Dimension of each key head in linear attention
layers. Defaults to
`0.75 * hidden_size / linear_num_key_heads`.
linear_value_head_dim (`int`, *optional*):
Dimension of each value head in linear
attention layers. Defaults to
`2 * linear_key_head_dim`.
linear_a_log_min (`float`, *optional*,
defaults to 0.0):
Minimum value for uniform initialization of
A_log in GatedDeltaNet layers.
linear_a_log_max (`float`, *optional*,
defaults to 16.0):
Maximum value for uniform initialization of
A_log in GatedDeltaNet layers.
linear_dt_min (`float`, *optional*,
defaults to 0.001):
Minimum value for dt initialization in
GatedDeltaNet layers.
linear_dt_max (`float`, *optional*,
defaults to 0.1):
Maximum value for dt initialization in
GatedDeltaNet layers.
linear_dt_init_floor (`float`, *optional*,
defaults to 0.0001):
Floor value for clamping dt during
initialization in GatedDeltaNet layers.
linear_conv_kernel_dim (`int`, *optional*,
defaults to 4):
Kernel size for the short convolution applied
to queries, keys, and values in linear
attention layers.
linear_allow_neg_eigval (`bool`, *optional*,
defaults to `True`):
Whether to allow negative eigenvalues in the
GatedDeltaNet recurrence. When `True`, the
beta parameter is scaled by 2.0 to allow
values in range [0, 2] instead of [0, 1].
```python
>>> from transformers import (
... OlmoHybridModel,
... OlmoHybridConfig,
... )
>>> configuration = OlmoHybridConfig()
>>> model = OlmoHybridModel(configuration)
>>> configuration = model.config
```
"""
model_type = "olmo_hybrid"
keys_to_ignore_at_inference = ["past_key_values"]
base_model_tp_plan = {
"layers.*.self_attn.q_proj": "colwise_gather_output",
"layers.*.self_attn.k_proj": "colwise_gather_output",
"layers.*.self_attn.v_proj": "colwise_gather_output",
"layers.*.self_attn.o_proj": "rowwise_split_input",
"layers.*.mlp.gate_proj": "colwise",
"layers.*.mlp.up_proj": "colwise",
"layers.*.mlp.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
def __init__(
self,
vocab_size: int | None = 100352,
hidden_size: int | None = 3840,
intermediate_size: int | None = 11008,
num_hidden_layers: int | None = 32,
num_attention_heads: int | None = 30,
num_key_value_heads: int | None = None,
hidden_act: str | None = "silu",
max_position_embeddings: int | None = 65536,
initializer_range: float | None = 0.02,
use_cache: bool | None = True,
pad_token_id: int | None = 100277,
bos_token_id: int | None = None,
eos_token_id: int | None = 100257,
tie_word_embeddings: bool | None = False,
rope_parameters=None,
attention_bias: bool | None = False,
attention_dropout: float | None = 0.0,
rms_norm_eps: float | None = 1e-06,
layer_types: list[str] | None = None,
linear_num_key_heads: int | None = None,
linear_num_value_heads: int | None = None,
linear_key_head_dim: int | None = None,
linear_value_head_dim: int | None = None,
linear_a_log_min: float = 0.0,
linear_a_log_max: float = 16.0,
linear_dt_min: float = 0.001,
linear_dt_max: float = 0.1,
linear_dt_init_floor: float = 1e-4,
linear_conv_kernel_dim: int = 4,
linear_allow_neg_eigval: bool = True,
**kwargs,
):
super().__init__(**kwargs)
assert num_hidden_layers is not None
assert hidden_size is not None
assert num_attention_heads is not None
if layer_types is None:
# Default: linear attention for most layers, full attention every 4th layer
layer_types = ["linear_attention"] * int(num_hidden_layers)
for i in range(int(num_hidden_layers)):
if i % 4 == 3:
layer_types[i] = "full_attention"
# Ensure at least one full attention layer for small num_hidden_layers
if "full_attention" not in layer_types:
layer_types[-1] = "full_attention"
self.layer_types = layer_types
self.validate_layer_type()
if "linear_attention" not in layer_types:
raise ValueError(
"OLMoHybrid expects at least one 'linear_attention' layer."
)
if all(t == "linear_attention" for t in layer_types):
raise ValueError("OLMoHybrid expects at least one attention layer.")
self.layer_types = layer_types
if linear_num_key_heads is None:
linear_num_key_heads = num_attention_heads
if linear_num_value_heads is None:
linear_num_value_heads = num_attention_heads
if linear_key_head_dim is None:
linear_key_head_dim = int(0.75 * hidden_size / linear_num_key_heads)
if linear_value_head_dim is None:
linear_value_head_dim = 2 * linear_key_head_dim
self.linear_num_key_heads = linear_num_key_heads
self.linear_num_value_heads = linear_num_value_heads
self.linear_key_head_dim = linear_key_head_dim
self.linear_value_head_dim = linear_value_head_dim
self.linear_a_log_min = linear_a_log_min
self.linear_a_log_max = linear_a_log_max
self.linear_dt_min = linear_dt_min
self.linear_dt_max = linear_dt_max
self.linear_dt_init_floor = linear_dt_init_floor
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_allow_neg_eigval = linear_allow_neg_eigval
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.rope_parameters = rope_parameters
self.tie_word_embeddings = tie_word_embeddings
self.pad_token_id = pad_token_id
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
+6 -66
View File
@@ -2,69 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py
# and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
# Ovis Config with AimV2 config registration removed for Transformers compatibility
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
from typing import Any
from transformers import AutoConfig, PreTrainedConfig
class AIMv2Config(PreTrainedConfig):
"""This is the configuration class to store the configuration of an [`AIMv2Model`].
Instantiating a configuration with the defaults will yield a similar configuration
to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224).
Args:
hidden_size: Dimension of the hidden representations.
intermediate_size: Dimension of the SwiGLU representations.
num_hidden_layers: Number of hidden layers in the Transformer.
num_attention_heads: Number of attention heads for each attention layer
in the Transformer.
num_channels: Number of input channels.
image_size: Image size.
patch_size: Patch size.
rms_norm_eps: Epsilon value used for the RMS normalization layer.
attention_dropout: Dropout ratio for attention probabilities.
projection_dropout: Dropout ratio for the projection layer after the attention.
qkv_bias: Whether to add a bias to the queries, keys and values.
use_bias: Whether to add a bias in the feed-forward and projection layers.
kwargs: Keyword arguments for the [`PreTrainedConfig`].
"""
model_type: str = "aimv2"
def __init__(
self,
hidden_size: int = 1024,
intermediate_size: int = 2816,
num_hidden_layers: int = 24,
num_attention_heads: int = 8,
num_channels: int = 3,
image_size: int = 224,
patch_size: int = 14,
rms_norm_eps: float = 1e-5,
attention_dropout: float = 0.0,
projection_dropout: float = 0.0,
qkv_bias: bool = False,
use_bias: bool = False,
**kwargs: Any,
):
super().__init__(**kwargs)
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_channels = num_channels
self.patch_size = patch_size
self.image_size = image_size
self.attention_dropout = attention_dropout
self.rms_norm_eps = rms_norm_eps
self.projection_dropout = projection_dropout
self.qkv_bias = qkv_bias
self.use_bias = use_bias
from transformers import Aimv2VisionConfig, AutoConfig, PreTrainedConfig
# ----------------------------------------------------------------------
@@ -96,14 +37,13 @@ class BaseVisualTokenizerConfig(PreTrainedConfig):
f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type"
)
if not isinstance(backbone_config, PreTrainedConfig):
model_type = backbone_config["model_type"]
if model_type != "aimv2":
backbone_config.pop("model_type")
model_type = backbone_config.pop("model_type")
if model_type == "aimv2":
backbone_config = Aimv2VisionConfig(**backbone_config)
else:
backbone_config = AutoConfig.for_model(
model_type, **backbone_config
)
else:
backbone_config = AIMv2Config(**backbone_config)
self.backbone_config = backbone_config
self.hidden_stride = hidden_stride
-193
View File
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Qwen3.5 model configuration."""
from transformers.configuration_utils import PreTrainedConfig
class Qwen3_5TextConfig(PreTrainedConfig):
model_type = "qwen3_5_text"
keys_to_ignore_at_inference = ["past_key_values"]
base_model_tp_plan = {
"layers.*.self_attn.q_proj": "colwise",
"layers.*.self_attn.k_proj": "colwise",
"layers.*.self_attn.v_proj": "colwise",
"layers.*.self_attn.o_proj": "rowwise",
"layers.*.mlp.gate_proj": "colwise",
"layers.*.mlp.up_proj": "colwise",
"layers.*.mlp.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
base_config_key = "text_config"
def __init__(
self,
vocab_size=248320,
hidden_size=4096,
intermediate_size=12288,
num_hidden_layers=32,
num_attention_heads=16,
num_key_value_heads=4,
hidden_act="silu",
max_position_embeddings=32768,
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
tie_word_embeddings=False,
rope_parameters=None,
attention_bias=False,
attention_dropout=0.0,
head_dim=256,
linear_conv_kernel_dim=4,
linear_key_head_dim=128,
linear_value_head_dim=128,
linear_num_key_heads=16,
linear_num_value_heads=32,
layer_types=None,
pad_token_id=None,
bos_token_id=None,
eos_token_id=None,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.head_dim = head_dim
self.rope_parameters = rope_parameters
kwargs.setdefault("partial_rotary_factor", 0.25)
self.layer_types = layer_types
if self.layer_types is None:
interval_pattern = kwargs.get("full_attention_interval", 4)
self.layer_types = [
"linear_attention"
if bool((i + 1) % interval_pattern)
else "full_attention"
for i in range(self.num_hidden_layers)
]
kwargs["ignore_keys_at_rope_validation"] = {
"mrope_section",
"mrope_interleaved",
}
self.validate_layer_type()
# linear attention part
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_key_head_dim = linear_key_head_dim
self.linear_value_head_dim = linear_value_head_dim
self.linear_num_key_heads = linear_num_key_heads
self.linear_num_value_heads = linear_num_value_heads
super().__init__(**kwargs)
# Set these AFTER super().__init__() because transformers v4's
# PreTrainedConfig.__init__ has these as explicit params with different
# defaults (e.g. tie_word_embeddings=True) that would overwrite our values.
self.pad_token_id = pad_token_id
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
self.tie_word_embeddings = tie_word_embeddings
class Qwen3_5VisionConfig(PreTrainedConfig):
model_type = "qwen3_5"
base_config_key = "vision_config"
def __init__(
self,
depth=27,
hidden_size=1152,
hidden_act="gelu_pytorch_tanh",
intermediate_size=4304,
num_heads=16,
in_channels=3,
patch_size=16,
spatial_merge_size=2,
temporal_patch_size=2,
out_hidden_size=3584,
num_position_embeddings=2304,
initializer_range=0.02,
**kwargs,
):
super().__init__(**kwargs)
self.depth = depth
self.hidden_size = hidden_size
self.hidden_act = hidden_act
self.intermediate_size = intermediate_size
self.num_heads = num_heads
self.in_channels = in_channels
self.patch_size = patch_size
self.spatial_merge_size = spatial_merge_size
self.temporal_patch_size = temporal_patch_size
self.out_hidden_size = out_hidden_size
self.num_position_embeddings = num_position_embeddings
self.initializer_range = initializer_range
class Qwen3_5Config(PreTrainedConfig):
model_type = "qwen3_5"
sub_configs = {
"vision_config": Qwen3_5VisionConfig,
"text_config": Qwen3_5TextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config=None,
vision_config=None,
image_token_id=248056,
video_token_id=248057,
vision_start_token_id=248053,
vision_end_token_id=248054,
tie_word_embeddings=False,
**kwargs,
):
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
self.text_config = self.sub_configs["text_config"]()
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
super().__init__(**kwargs)
# Set after super().__init__() to avoid v4 PreTrainedConfig overwrite
self.tie_word_embeddings = tie_word_embeddings
__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig"]
@@ -1,205 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Qwen3.5-MoE model configuration."""
from transformers.configuration_utils import PreTrainedConfig
class Qwen3_5MoeTextConfig(PreTrainedConfig):
model_type = "qwen3_5_moe_text"
keys_to_ignore_at_inference = ["past_key_values"]
base_model_tp_plan = {
"layers.*.self_attn.q_proj": "colwise",
"layers.*.self_attn.k_proj": "colwise",
"layers.*.self_attn.v_proj": "colwise",
"layers.*.self_attn.o_proj": "rowwise",
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
"layers.*.mlp.experts.down_proj": "rowwise",
"layers.*.mlp.shared_expert.gate_proj": "colwise",
"layers.*.mlp.shared_expert.up_proj": "colwise",
"layers.*.mlp.shared_expert.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
base_config_key = "text_config"
def __init__(
self,
vocab_size=248320,
hidden_size=2048,
num_hidden_layers=40,
num_attention_heads=16,
num_key_value_heads=2,
hidden_act="silu",
max_position_embeddings=32768,
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
tie_word_embeddings=False,
rope_parameters=None,
attention_bias=False,
attention_dropout=0.0,
head_dim=256,
linear_conv_kernel_dim=4,
linear_key_head_dim=128,
linear_value_head_dim=128,
linear_num_key_heads=16,
linear_num_value_heads=32,
moe_intermediate_size=512,
shared_expert_intermediate_size=512,
num_experts_per_tok=8,
num_experts=256,
output_router_logits=False,
router_aux_loss_coef=0.001,
layer_types=None,
pad_token_id=None,
bos_token_id=None,
eos_token_id=None,
**kwargs,
):
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.head_dim = head_dim
self.rope_parameters = rope_parameters
kwargs.setdefault("partial_rotary_factor", 0.25)
self.layer_types = layer_types
if self.layer_types is None:
interval_pattern = kwargs.get("full_attention_interval", 4)
self.layer_types = [
"linear_attention"
if bool((i + 1) % interval_pattern)
else "full_attention"
for i in range(self.num_hidden_layers)
]
kwargs["ignore_keys_at_rope_validation"] = {
"mrope_section",
"mrope_interleaved",
}
self.validate_layer_type()
# linear attention part
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_key_head_dim = linear_key_head_dim
self.linear_value_head_dim = linear_value_head_dim
self.linear_num_key_heads = linear_num_key_heads
self.linear_num_value_heads = linear_num_value_heads
self.moe_intermediate_size = moe_intermediate_size
self.shared_expert_intermediate_size = shared_expert_intermediate_size
self.num_experts_per_tok = num_experts_per_tok
self.num_experts = num_experts
self.output_router_logits = output_router_logits
self.router_aux_loss_coef = router_aux_loss_coef
super().__init__(**kwargs)
# Set these AFTER super().__init__() because transformers v4's
# PreTrainedConfig.__init__ has these as explicit params with different
# defaults (e.g. tie_word_embeddings=True) that would overwrite our values.
self.pad_token_id = pad_token_id
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
self.tie_word_embeddings = tie_word_embeddings
class Qwen3_5MoeVisionConfig(PreTrainedConfig):
model_type = "qwen3_5_moe"
base_config_key = "vision_config"
def __init__(
self,
depth=27,
hidden_size=1152,
hidden_act="gelu_pytorch_tanh",
intermediate_size=4304,
num_heads=16,
in_channels=3,
patch_size=16,
spatial_merge_size=2,
temporal_patch_size=2,
out_hidden_size=3584,
num_position_embeddings=2304,
initializer_range=0.02,
**kwargs,
):
super().__init__(**kwargs)
self.depth = depth
self.hidden_size = hidden_size
self.hidden_act = hidden_act
self.intermediate_size = intermediate_size
self.num_heads = num_heads
self.in_channels = in_channels
self.patch_size = patch_size
self.spatial_merge_size = spatial_merge_size
self.temporal_patch_size = temporal_patch_size
self.out_hidden_size = out_hidden_size
self.num_position_embeddings = num_position_embeddings
self.initializer_range = initializer_range
class Qwen3_5MoeConfig(PreTrainedConfig):
model_type = "qwen3_5_moe"
sub_configs = {
"vision_config": Qwen3_5MoeVisionConfig,
"text_config": Qwen3_5MoeTextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config=None,
vision_config=None,
image_token_id=248056,
video_token_id=248057,
vision_start_token_id=248053,
vision_end_token_id=248054,
tie_word_embeddings=False,
**kwargs,
):
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
self.text_config = self.sub_configs["text_config"]()
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
super().__init__(**kwargs)
# Set after super().__init__() to avoid v4 PreTrainedConfig overwrite
self.tie_word_embeddings = tie_word_embeddings
__all__ = ["Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig"]
@@ -1,277 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Qwen3-Next model configuration."""
from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class Qwen3NextConfig(PreTrainedConfig):
r"""This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a
Qwen3-Next model according to the specified arguments, defining the model architecture.
Instantiating a configuration with the defaults will yield a similar configuration to that of
Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct).
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PreTrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 151936):
Vocabulary size of the model. Defines the number of different tokens that can be represented by the
`inputs_ids`.
hidden_size (`int`, *optional*, defaults to 2048):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 5632):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 48):
Number of hidden layers in the Transformer encoder.
num_attention_heads (`int`, *optional*, defaults to 16):
Number of attention heads for each attention layer in the Transformer encoder.
num_key_value_heads (`int`, *optional*, defaults to 2):
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details checkout [this
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
hidden_act (`str`, *optional*, defaults to `"silu"`):
The non-linear activation function in the decoder.
max_position_embeddings (`int`, *optional*, defaults to 32768):
The maximum sequence length that this model might ever be used with.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
The epsilon used by the rms normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether the model's input and output word embeddings should be tied.
rope_parameters (`dict`, *optional*):
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
accordingly.
Expected contents:
`rope_theta` (`float`): The base period of the RoPE embeddings.
`rope_type` (`str`):
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
'llama3'], with 'default' being the original RoPE implementation.
`factor` (`float`, *optional*):
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
original maximum pre-trained length.
`original_max_position_embeddings` (`int`, *optional*):
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
pretraining.
`attention_factor` (`float`, *optional*):
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
computation. If unspecified, it defaults to value recommended by the implementation, using the
`factor` field to infer the suggested value.
`beta_fast` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
ramp function. If unspecified, it defaults to 32.
`beta_slow` (`float`, *optional*):
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
ramp function. If unspecified, it defaults to 1.
`short_factor` (`List[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
size divided by the number of attention heads divided by 2
`long_factor` (`List[float]`, *optional*):
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
size divided by the number of attention heads divided by 2
`low_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
`high_freq_factor` (`float`, *optional*):
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
`partial_rotary_factor` (`float`, *optional*, defaults to 0.25):
Percentage of the query and keys which will have rotary embedding.
attention_bias (`bool`, *optional*, defaults to `False`):
Whether to use a bias in the query, key, value and output projection layers during self-attention.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
head_dim (`int`, *optional*, defaults to 256):
Projection weights dimension in multi-head attention.
linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
Kernel size of the convolution used in linear attention layers.
linear_key_head_dim (`int`, *optional*, defaults to 128):
Dimension of each key head in linear attention.
linear_value_head_dim (`int`, *optional*, defaults to 128):
Dimension of each value head in linear attention.
linear_num_key_heads (`int`, *optional*, defaults to 16):
Number of key heads used in linear attention layers.
linear_num_value_heads (`int`, *optional*, defaults to 32):
Number of value heads used in linear attention layers.
decoder_sparse_step (`int`, *optional*, defaults to 1):
The frequency of the MoE layer.
moe_intermediate_size (`int`, *optional*, defaults to 512):
Intermediate size of the routed expert.
shared_expert_intermediate_size (`int`, *optional*, defaults to 512):
Intermediate size of the shared expert.
num_experts_per_tok (`int`, *optional*, defaults to 10):
Number of selected experts.
num_experts (`int`, *optional*, defaults to 512):
Number of routed experts.
norm_topk_prob (`bool`, *optional*, defaults to `True`):
Whether to normalize the topk probabilities.
output_router_logits (`bool`, *optional*, defaults to `False`):
Whether or not the router logits should be returned by the model. Enabling this will also
allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
The aux loss factor for the total loss.
mlp_only_layers (`list[int]`, *optional*, defaults to `[]`):
Indicate which layers use Qwen3NextMLP rather than Qwen3NextSparseMoeBlock
The list contains layer index, from 0 to num_layers-1 if we have num_layers layers
If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity.
layer_types (`list[str]`, *optional*):
Types of each layer (attention or linear).
```python
>>> from transformers import Qwen3NextModel, Qwen3NextConfig
>>> # Initializing a Qwen3Next style configuration
>>> configuration = Qwen3NextConfig()
>>> # Initializing a model from the Qwen3-Next-80B-A3B style configuration
>>> model = Qwen3NextModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
""" # noqa: E501
model_type = "qwen3_next"
keys_to_ignore_at_inference = ["past_key_values"]
base_model_tp_plan = {
"layers.*.self_attn.q_proj": "colwise",
"layers.*.self_attn.k_proj": "colwise",
"layers.*.self_attn.v_proj": "colwise",
"layers.*.self_attn.o_proj": "rowwise",
"layers.*.mlp.experts.*.gate_proj": "colwise",
"layers.*.mlp.experts.*.up_proj": "colwise",
"layers.*.mlp.experts.*.down_proj": "rowwise",
"layers.*.mlp.shared_experts.gate_proj": "colwise",
"layers.*.mlp.shared_experts.up_proj": "colwise",
"layers.*.mlp.shared_experts.down_proj": "rowwise",
"layers.*.mlp.gate_proj": "colwise",
"layers.*.mlp.up_proj": "colwise",
"layers.*.mlp.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
def __init__(
self,
vocab_size=151936,
hidden_size=2048,
intermediate_size=5632,
num_hidden_layers=48,
num_attention_heads=16,
num_key_value_heads=2,
hidden_act="silu",
max_position_embeddings=32768,
initializer_range=0.02,
rms_norm_eps=1e-6,
use_cache=True,
tie_word_embeddings=False,
rope_parameters=None,
attention_bias=False,
attention_dropout=0.0,
head_dim=256,
linear_conv_kernel_dim=4,
linear_key_head_dim=128,
linear_value_head_dim=128,
linear_num_key_heads=16,
linear_num_value_heads=32,
decoder_sparse_step=1,
moe_intermediate_size=512,
shared_expert_intermediate_size=512,
num_experts_per_tok=10,
num_experts=512,
norm_topk_prob=True,
output_router_logits=False,
router_aux_loss_coef=0.001,
mlp_only_layers=None,
layer_types=None,
**kwargs,
):
if mlp_only_layers is None:
mlp_only_layers = []
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = hidden_act
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
rope_scaling = kwargs.pop("rope_scaling", None)
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
rope_theta = kwargs.pop("rope_theta", 10000.0)
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
partial_rotary_factor = kwargs.pop("partial_rotary_factor", 0.25)
if "partial_rotary_factor" not in rope_parameters:
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
self.rope_parameters = rope_parameters
self.partial_rotary_factor = partial_rotary_factor
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.head_dim = head_dim
self.layer_types = layer_types
if self.layer_types is None:
self.layer_types = [
"linear_attention" if bool((i + 1) % 4) else "full_attention"
for i in range(self.num_hidden_layers)
]
self.validate_layer_type()
# linear attention part
self.linear_conv_kernel_dim = linear_conv_kernel_dim
self.linear_key_head_dim = linear_key_head_dim
self.linear_value_head_dim = linear_value_head_dim
self.linear_num_key_heads = linear_num_key_heads
self.linear_num_value_heads = linear_num_value_heads
# MoE arguments
self.decoder_sparse_step = decoder_sparse_step
self.moe_intermediate_size = moe_intermediate_size
self.shared_expert_intermediate_size = shared_expert_intermediate_size
self.num_experts_per_tok = num_experts_per_tok
self.num_experts = num_experts
self.norm_topk_prob = norm_topk_prob
self.output_router_logits = output_router_logits
self.router_aux_loss_coef = router_aux_loss_coef
self.mlp_only_layers = mlp_only_layers
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
__all__ = ["Qwen3NextConfig"]
@@ -17,7 +17,6 @@ from vllm.transformers_utils.config import (
ConfigFormat,
get_safetensors_params_metadata,
)
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@@ -41,7 +40,7 @@ class ModelArchConfigConvertorBase:
This is the only place that decides whether a checkpoint is heterogeneous
and how many layers it has, so a convertor can declare variation that
Transformers does not express (see `Gemma4ModelArchConfigConvertor`).
Transformers does not express.
Each pair is converted independently and the results are diffed, so a
convertor that overrides this must not also collapse a varying field in
@@ -745,27 +744,6 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
== "vision"
)
def get_per_layer_hf_configs(
self,
) -> list[tuple[PreTrainedConfig, PreTrainedConfig]] | None:
# Gemma4 uses a larger head dimension, and sometimes more KV heads, on its
# full attention layers than on its sliding ones. Transformers >= 5.15.0
# says so in the config; before that the values are flat attributes picked
# apart by `layer_types`, so build the per-layer configs here instead. Both
# then reach the base convertor as ordinary homogeneous configs.
if getattr(self.hf_text_config, "is_heterogeneous", False):
return super().get_per_layer_hf_configs()
text_config = self.hf_text_config
if not getattr(text_config, "layer_types", None):
return None
# `num_hidden_layers` is what the stack is built from, and it is what
# Transformers sizes `per_layer_config` by. `layer_types` can be longer,
# e.g. when a test truncates the layer count to build a small model.
return [
(self.hf_config, gemma4_layer_config(text_config, layer_idx))
for layer_idx in range(self.get_num_hidden_layers())
]
class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase):
def _language_config(self) -> PreTrainedConfig:
@@ -17,10 +17,8 @@ __all__ = [
"FireRedASR2Processor",
"FunASRProcessor",
"GLM4VProcessor",
"Granite4VisionProcessor",
"H2OVLProcessor",
"Moondream3Processor",
"InternVLProcessor",
"IsaacProcessor",
"KimiAudioProcessor",
"KimiK25Processor",
@@ -55,9 +53,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
"IsaacProcessor": "vllm.transformers_utils.processors.isaac",
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
"KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from transformers import LlavaNextProcessor
from transformers.image_processing_utils import select_best_resolution
class Granite4VisionProcessor(LlavaNextProcessor):
"""Processor for Granite 4 Vision.
Extends LlavaNextProcessor to account for the Window Q-Former
downsampling when computing the number of image features.
This processor is needed because the granite4_vision processor type
is not yet in the transformers version pinned by vLLM.
"""
model_type = "granite4_vision"
def __init__(
self,
image_processor=None,
tokenizer=None,
patch_size=None,
vision_feature_select_strategy=None,
chat_template=None,
image_token="<image>",
num_additional_image_tokens=0,
downsample_rate=None,
**kwargs,
):
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
patch_size=patch_size,
vision_feature_select_strategy=vision_feature_select_strategy,
chat_template=chat_template,
image_token=image_token,
num_additional_image_tokens=num_additional_image_tokens,
)
self.downsample_rate = downsample_rate
def _get_number_of_features(
self,
orig_height: int,
orig_width: int,
height: int,
width: int,
) -> int:
image_grid_pinpoints = self.image_processor.image_grid_pinpoints
height_best_resolution, width_best_resolution = select_best_resolution(
[orig_height, orig_width], image_grid_pinpoints
)
scale_height = height_best_resolution // height
scale_width = width_best_resolution // width
patches_height = height // self.patch_size
patches_width = width // self.patch_size
if self.downsample_rate is not None:
ds_rate = Fraction(self.downsample_rate)
patches_height = int(patches_height * ds_rate)
patches_width = int(patches_width * ds_rate)
unpadded_features, newline_features = self._get_unpadded_features(
orig_height,
orig_width,
patches_height,
patches_width,
scale_height,
scale_width,
)
base_features = (
patches_height * patches_width + self.num_additional_image_tokens
)
return unpadded_features + newline_features + base_features