[Chore] Remove vendored code made redundant by Transformers 5.16.1 (#60623)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bc713cf8cc
commit
2c99ee9333
@@ -79,8 +79,6 @@ class LazyConfigDict(dict):
|
||||
|
||||
|
||||
_CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
afmoe="AfmoeConfig",
|
||||
axk1="AXK1Config",
|
||||
bagel="BagelConfig",
|
||||
bailing_moe_v3_vl="BailingMoeV3VLConfig",
|
||||
chatglm="ChatGLMConfig",
|
||||
@@ -89,24 +87,20 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
colqwen3="ColQwen3Config",
|
||||
ops_colqwen3="OpsColQwen3Config",
|
||||
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
|
||||
cosmos3_omni="Cosmos3Config",
|
||||
cosmos3_edge="Cosmos3EdgeConfig",
|
||||
diffusion_gemma="DiffusionGemmaConfig",
|
||||
deepseek_vl_v2="DeepseekVLV2Config",
|
||||
deepseek_v32="DeepseekV3Config",
|
||||
deepseek_v4="DeepseekV4Config",
|
||||
dots3_note="Dots3NoteConfig",
|
||||
k3_dspark="K3DSparkConfig",
|
||||
funaudiochat="FunAudioChatConfig",
|
||||
granite4_vision="Granite4VisionConfig",
|
||||
hyperclovax="HyperCLOVAXConfig",
|
||||
hy_v3="HYV3Config",
|
||||
hy_v4="HYV4Config",
|
||||
hyperclovax="HyperCLOVAXConfig", # Upstream class, hub remote code is broken
|
||||
isaac="IsaacConfig",
|
||||
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
|
||||
kimi_linear="KimiLinearConfig",
|
||||
kimi_vl="KimiVLConfig",
|
||||
kimi_k25="KimiK25Config",
|
||||
kimi_k25="Kimi_K25Config", # Upstream class, hub remote code uses old schema
|
||||
muse_glimmer="MuseGlimmerConfig",
|
||||
muse_glimmer_text="MuseGlimmerTextConfig",
|
||||
muse_glimmer_vision="MuseGlimmerVisionConfig",
|
||||
@@ -123,8 +117,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
moss_transcribe_diarize="MossTranscribeDiarizeConfig",
|
||||
eagle="EAGLEConfig",
|
||||
speculators="SpeculatorsConfig",
|
||||
nemotron="NemotronConfig",
|
||||
olmo_hybrid="OlmoHybridConfig",
|
||||
openvla="OpenVLAConfig",
|
||||
ovis="OvisConfig",
|
||||
ultravox="UltravoxConfig",
|
||||
@@ -133,12 +125,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
step3p5="Step3p5Config",
|
||||
qianfan_ocr="QianfanOCRConfig",
|
||||
qwen3_asr="Qwen3ASRConfig",
|
||||
qwen3_next="Qwen3NextConfig",
|
||||
qwen3_5="Qwen3_5Config",
|
||||
qwen3_5_text="Qwen3_5TextConfig",
|
||||
qwen3_5_moe="Qwen3_5MoeConfig",
|
||||
qwen3_5_moe_text="Qwen3_5MoeTextConfig",
|
||||
lfm2_moe="Lfm2MoeConfig",
|
||||
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
||||
**{"deepseek_v41": "DeepseekV41Config"},
|
||||
inkling_mm_model="InklingMMConfig",
|
||||
@@ -162,14 +148,6 @@ _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}
|
||||
# `gemma4_text` ships them in the checkpoint itself.
|
||||
_PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna", "gemma4_text"}
|
||||
|
||||
# Model types whose checkpoints declare `layer_types` entries that upstream
|
||||
# transformers has not added to `ALLOWED_LAYER_TYPES` yet, so its strict config
|
||||
# validation rejects them (e.g. GLM-5.2 `glm_moe_dsa` use
|
||||
# `deepseek_sparse_attention`). Extend the allowed set for these model types.
|
||||
_PATCH_HF_ALLOWED_LAYER_TYPES: dict[str, tuple[str, ...]] = {
|
||||
"glm_moe_dsa": ("deepseek_sparse_attention",),
|
||||
}
|
||||
|
||||
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
|
||||
"llm_config": "text_config",
|
||||
}
|
||||
@@ -314,24 +292,6 @@ def _patch_hf_transformers_nested_rope_validation() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _patch_hf_transformers_allowed_layer_types(
|
||||
extra_layer_types: tuple[str, ...],
|
||||
) -> None:
|
||||
"""Extend transformers' ``ALLOWED_LAYER_TYPES`` so its strict config
|
||||
validation accepts layer types (e.g. ``deepseek_sparse_attention``) that a
|
||||
checkpoint declares but upstream transformers has not registered yet.
|
||||
"""
|
||||
import transformers.configuration_utils as hf_configuration_utils
|
||||
|
||||
missing = tuple(
|
||||
layer_type
|
||||
for layer_type in extra_layer_types
|
||||
if layer_type not in hf_configuration_utils.ALLOWED_LAYER_TYPES
|
||||
)
|
||||
if missing:
|
||||
hf_configuration_utils.ALLOWED_LAYER_TYPES += missing
|
||||
|
||||
|
||||
class HFConfigParser(ConfigParserBase):
|
||||
def parse(
|
||||
self,
|
||||
@@ -379,9 +339,6 @@ class HFConfigParser(ConfigParserBase):
|
||||
if model_type in _PATCH_HF_NESTED_ROPE_VALIDATION:
|
||||
_patch_hf_transformers_nested_rope_validation()
|
||||
|
||||
if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type):
|
||||
_patch_hf_transformers_allowed_layer_types(extra_layer_types)
|
||||
|
||||
if model_type == "vlm":
|
||||
# HyperCLOVAX remote code registers this alias in a bare
|
||||
# `try/except` that fails silently since transformers 5.18
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
class AXK1Config(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a [`AXK1Model`].
|
||||
It is used to instantiate an A.X model according to the specified arguments,
|
||||
defining the model architecture. Instantiating a configuration with the defaults
|
||||
will yield a similar configuration to that of the A.X K1.
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control
|
||||
the model outputs. Read the documentation from [`PreTrainedConfig`] for more
|
||||
information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 163840):
|
||||
Vocabulary size of the A.X K1 model. Defines the number of different
|
||||
tokens that can be represented by the `inputs_ids` passed when calling
|
||||
[`AXK1Model`]
|
||||
hidden_size (`int`, *optional*, defaults to 7168):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 18432):
|
||||
Dimension of the MLP representations.
|
||||
moe_intermediate_size (`int`, *optional*, defaults to 2048):
|
||||
Dimension of the MoE representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 61):
|
||||
Number of hidden layers in the Transformer decoder.
|
||||
num_nextn_predict_layers (`int`, *optional*, defaults to 1):
|
||||
Number of nextn predict layers in the AXK1 Model.
|
||||
num_attention_heads (`int`, *optional*, defaults to 64):
|
||||
Number of attention heads for each attention layer in the Transformer
|
||||
decoder.
|
||||
n_shared_experts (`int`, *optional*, defaults to 1):
|
||||
Number of shared experts, None means dense model.
|
||||
n_routed_experts (`int`, *optional*, defaults to 192):
|
||||
Number of routed experts, None means dense model.
|
||||
routed_scaling_factor (`float`, *optional*, defaults to 2.5):
|
||||
Scaling factor or routed experts.
|
||||
topk_method (`str`, *optional*, defaults to `noaux_tc`):
|
||||
Topk method used in routed gate.
|
||||
n_group (`int`, *optional*, defaults to 8):
|
||||
Number of groups for routed experts.
|
||||
topk_group (`int`, *optional*, defaults to 4):
|
||||
Number of selected groups for each token(for each token, ensuring the
|
||||
selected experts is only within `topk_group` groups).
|
||||
num_experts_per_tok (`int`, *optional*, defaults to 8):
|
||||
Number of selected experts, None means dense model.
|
||||
moe_layer_freq (`int`, *optional*, defaults to 1):
|
||||
The frequency of the MoE layer: one expert layer for every
|
||||
`moe_layer_freq - 1` dense layers.
|
||||
first_k_dense_replace (`int`, *optional*, defaults to 1):
|
||||
Number of dense layers in shallow layers
|
||||
(embed->dense->dense->...->dense->moe->moe...->lm_head).
|
||||
\--k dense layers--/
|
||||
norm_topk_prob (`bool`, *optional*, defaults to True):
|
||||
Whether to normalize the weights of the routed experts.
|
||||
scoring_func (`str`, *optional*, defaults to 'sigmoid'):
|
||||
Method of computing expert weights.
|
||||
aux_loss_alpha (`float`, *optional*, defaults to 0.0001):
|
||||
Auxiliary loss weight coefficient.
|
||||
seq_aux = (`bool`, *optional*, defaults to True):
|
||||
Whether to compute the auxiliary loss for each individual sample.
|
||||
num_key_value_heads (`int`, *optional*):
|
||||
This is the number of key_value heads that should be used to implement
|
||||
Grouped Query Attention. If `num_key_value_heads=num_attention_heads`,
|
||||
the model will use Multi Head Attention (MHA), if `num_key_value_heads=1
|
||||
the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each group key and
|
||||
value head should be constructed by meanpooling all the original heads
|
||||
within that group. For more details checkout
|
||||
[this paper](https://arxiv.org/pdf/2305.13245.pdf).
|
||||
If it is not specified, will default to `num_attention_heads`.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
||||
The non-linear activation function (function or string) in the decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 131072):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for
|
||||
initializing all weight matrices.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the rms normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions
|
||||
(not used by all models). Only relevant if `config.is_decoder=True`.
|
||||
pad_token_id (`int`, *optional*):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*, defaults to 163691):
|
||||
Beginning of stream token id.
|
||||
eos_token_id (`int`, *optional*, defaults to 163691):
|
||||
End of stream token id.
|
||||
pretraining_tp (`int`, *optional*, defaults to 1):
|
||||
Experimental feature. Tensor parallelism rank used during pretraining.
|
||||
Please refer to
|
||||
[this document](https://huggingface.co/docs/transformers/parallelism)
|
||||
to understand more about it. This value is necessary to ensure exact
|
||||
reproducibility of the pretraining results. Please refer to
|
||||
[this issue](https://github.com/pytorch/pytorch/issues/76232).
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie weight embeddings
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_scaling (`Dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE embeddings.
|
||||
Currently supports two scaling strategies: linear and dynamic.
|
||||
Their scaling factor must be a float greater than 1. The expected format
|
||||
is `{"type": strategy name, "factor": scaling factor}`. When using this
|
||||
flag, don't update `max_position_embeddings` to the expected new maximum.
|
||||
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output projection
|
||||
layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
|
||||
"""
|
||||
|
||||
model_type = "axk1"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 163840,
|
||||
hidden_size: int = 7168,
|
||||
intermediate_size: int = 18432,
|
||||
moe_intermediate_size: int = 2048,
|
||||
num_hidden_layers: int = 61,
|
||||
num_nextn_predict_layers: int | None = 1,
|
||||
num_attention_heads: int = 64,
|
||||
num_key_value_heads: int = 64,
|
||||
n_shared_experts: int | None = 1,
|
||||
n_routed_experts: int | None = 192,
|
||||
ep_size: int | None = 8, ## Ignored - Expert parallel size
|
||||
routed_scaling_factor: float | None = 2.5,
|
||||
kv_lora_rank: int | None = 512,
|
||||
q_lora_rank: int | None = 1536,
|
||||
qk_rope_head_dim: int | None = 64,
|
||||
v_head_dim: int | None = 128,
|
||||
qk_nope_head_dim: int | None = 128,
|
||||
topk_method: str | None = "noaux_tc",
|
||||
n_group: int | None = 8,
|
||||
topk_group: int | None = 4,
|
||||
num_experts_per_tok: int | None = 8,
|
||||
moe_layer_freq: int | None = 1,
|
||||
first_k_dense_replace: int = 1,
|
||||
norm_topk_prob: bool = True,
|
||||
scoring_func: str | None = "sigmoid",
|
||||
aux_loss_alpha: float | None = 0.0001,
|
||||
seq_aux: float | None = True,
|
||||
hidden_act: str | None = "silu",
|
||||
max_position_embeddings: int | None = 131072,
|
||||
initializer_range: float | None = 0.02,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
use_cache: bool | None = True,
|
||||
pad_token_id: int | None = None,
|
||||
bos_token_id: int | None = 163691,
|
||||
eos_token_id: int | None = 163691,
|
||||
pretraining_tp: int | None = 1,
|
||||
tie_word_embeddings: bool | None = False,
|
||||
rope_theta: float | None = 10000.0,
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
attention_bias: bool | None = False,
|
||||
attention_dropout: float | None = 0.0,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.n_shared_experts = n_shared_experts
|
||||
self.n_routed_experts = n_routed_experts
|
||||
self.ep_size = ep_size
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.topk_method = topk_method
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.moe_layer_freq = moe_layer_freq
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
self.scoring_func = scoring_func
|
||||
self.aux_loss_alpha = aux_loss_alpha
|
||||
self.seq_aux = seq_aux
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.pretraining_tp = pretraining_tp
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
self.rope_parameters = rope_parameters
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -14,8 +14,6 @@ from __future__ import annotations
|
||||
import importlib
|
||||
|
||||
_CLASS_TO_MODULE: dict[str, str] = {
|
||||
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
|
||||
"AXK1Config": "vllm.transformers_utils.configs.AXK1",
|
||||
"BagelConfig": "vllm.transformers_utils.configs.bagel",
|
||||
"BailingMoeV3TextConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
|
||||
"BailingMoeV3VisionConfig": "vllm.transformers_utils.configs.bailing_moe_v3_vl",
|
||||
@@ -26,13 +24,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
|
||||
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
|
||||
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
|
||||
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
|
||||
"DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
|
||||
"DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41",
|
||||
@@ -42,16 +37,12 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"EAGLEConfig": "vllm.transformers_utils.configs.eagle",
|
||||
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
|
||||
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
|
||||
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
|
||||
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
|
||||
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
|
||||
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
|
||||
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
|
||||
# `FalconConfig` class from the official HuggingFace transformers library.
|
||||
"RWConfig": "vllm.transformers_utils.configs.falcon",
|
||||
"Lfm2MoeConfig": "vllm.transformers_utils.configs.lfm2_moe",
|
||||
"MedusaConfig": "vllm.transformers_utils.configs.medusa",
|
||||
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
|
||||
"MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3",
|
||||
@@ -67,16 +58,12 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
|
||||
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
|
||||
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
|
||||
"KimiK25Config": "vllm.transformers_utils.configs.kimi_k25",
|
||||
"MuseGlimmerConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerTextConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerVisionConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"MuseGlimmerAssistantConfig": "vllm.transformers_utils.configs.muse_glimmer",
|
||||
"KimiK3Config": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"KimiK3VisionConfig": "vllm.transformers_utils.configs.kimi_k3",
|
||||
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
|
||||
"NemotronHConfig": "vllm.transformers_utils.configs.nemotron_h",
|
||||
"OlmoHybridConfig": "vllm.transformers_utils.configs.olmo_hybrid",
|
||||
"OpenVLAConfig": "vllm.transformers_utils.configs.openvla",
|
||||
"OvisConfig": "vllm.transformers_utils.configs.ovis",
|
||||
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
|
||||
@@ -91,22 +78,17 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
|
||||
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
|
||||
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
|
||||
"Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
|
||||
"Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
||||
"Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
|
||||
"InklingModelConfig": "vllm.models.inkling.configs",
|
||||
"InklingAudioConfig": "vllm.models.inkling.configs",
|
||||
"InklingVisionConfig": "vllm.models.inkling.configs",
|
||||
"InklingMMConfig": "vllm.models.inkling.configs",
|
||||
# Special case: DeepseekV3Config is from HuggingFace Transformers
|
||||
# Upstream Transformers classes registered in _CONFIG_REGISTRY
|
||||
"DeepseekV3Config": "transformers",
|
||||
"HyperCLOVAXConfig": "transformers",
|
||||
"Kimi_K25Config": "transformers",
|
||||
}
|
||||
|
||||
__all__ = [
|
||||
"AfmoeConfig",
|
||||
"AXK1Config",
|
||||
"BagelConfig",
|
||||
"BailingMoeV3TextConfig",
|
||||
"BailingMoeV3VisionConfig",
|
||||
@@ -117,13 +99,10 @@ __all__ = [
|
||||
"ColQwen3Config",
|
||||
"OpsColQwen3Config",
|
||||
"Qwen3VLNemotronEmbedConfig",
|
||||
"Cosmos3Config",
|
||||
"Cosmos3EdgeConfig",
|
||||
"Cosmos3EdgeProjectorConfig",
|
||||
"Cosmos3EdgeTextConfig",
|
||||
"Cosmos3EdgeVisionConfig",
|
||||
"DiffusionGemmaConfig",
|
||||
"DiffusionGemmaTextConfig",
|
||||
"DeepseekVLV2Config",
|
||||
"DeepseekV3Config",
|
||||
"DeepseekV4Config",
|
||||
@@ -133,13 +112,10 @@ __all__ = [
|
||||
"EAGLEConfig",
|
||||
"FunAudioChatConfig",
|
||||
"FunAudioChatAudioEncoderConfig",
|
||||
"Granite4VisionConfig",
|
||||
"HYV3Config",
|
||||
"HYV4Config",
|
||||
"HyperCLOVAXConfig",
|
||||
"IsaacConfig",
|
||||
"RWConfig",
|
||||
"Lfm2MoeConfig",
|
||||
"MedusaConfig",
|
||||
"MiDashengLMConfig",
|
||||
"MiniMaxM3Config",
|
||||
@@ -153,16 +129,13 @@ __all__ = [
|
||||
"MoonViTConfig",
|
||||
"KimiLinearConfig",
|
||||
"KimiVLConfig",
|
||||
"KimiK25Config",
|
||||
"Kimi_K25Config",
|
||||
"MuseGlimmerConfig",
|
||||
"MuseGlimmerTextConfig",
|
||||
"MuseGlimmerVisionConfig",
|
||||
"MuseGlimmerAssistantConfig",
|
||||
"KimiK3Config",
|
||||
"KimiK3VisionConfig",
|
||||
"NemotronConfig",
|
||||
"NemotronHConfig",
|
||||
"OlmoHybridConfig",
|
||||
"OpenVLAConfig",
|
||||
"OvisConfig",
|
||||
"PixelShuffleSiglip2VisionConfig",
|
||||
@@ -177,11 +150,6 @@ __all__ = [
|
||||
"QianfanOCRConfig",
|
||||
"QianfanOCRVisionConfig",
|
||||
"Qwen3ASRConfig",
|
||||
"Qwen3NextConfig",
|
||||
"Qwen3_5Config",
|
||||
"Qwen3_5TextConfig",
|
||||
"Qwen3_5MoeConfig",
|
||||
"Qwen3_5MoeTextConfig",
|
||||
"InklingModelConfig",
|
||||
"InklingAudioConfig",
|
||||
"InklingVisionConfig",
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class AfmoeConfig(PreTrainedConfig):
|
||||
model_type = "afmoe"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 200_192,
|
||||
hidden_size: int = 2048,
|
||||
intermediate_size: int = 6144,
|
||||
moe_intermediate_size: int = 1408,
|
||||
num_hidden_layers: int = 32,
|
||||
num_dense_layers: int = 1,
|
||||
num_attention_heads: int = 16,
|
||||
num_key_value_heads: int | None = None,
|
||||
head_dim: int = 128,
|
||||
hidden_act: str = "silu",
|
||||
max_position_embeddings: int = 131072,
|
||||
initializer_range: float = 0.02,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
use_cache: bool = True,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_parameters: dict | None = None,
|
||||
rope_scaling: dict | None = None,
|
||||
num_experts: int = 64,
|
||||
num_experts_per_tok: int = 6,
|
||||
num_shared_experts: int = 2,
|
||||
num_expert_groups: int = 1,
|
||||
num_limited_groups: int = 1,
|
||||
score_func: str = "sigmoid",
|
||||
route_norm: bool = True,
|
||||
route_scale: float = 1.0,
|
||||
global_attn_every_n_layers: int = 4,
|
||||
sliding_window: int = 2048,
|
||||
layer_types: list[str] | None = None,
|
||||
attention_dropout: float = 0.0,
|
||||
mup_enabled: bool = False,
|
||||
n_group: int = 1,
|
||||
topk_group: int = 1,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_dense_layers = num_dense_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads or num_attention_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_act = hidden_act
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.rope_scaling = rope_scaling
|
||||
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.num_experts = num_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_expert_groups = num_expert_groups
|
||||
self.num_limited_groups = num_limited_groups
|
||||
self.score_func = score_func
|
||||
self.route_norm = route_norm
|
||||
self.route_scale = route_scale
|
||||
|
||||
self.global_attn_every_n_layers = global_attn_every_n_layers
|
||||
self.sliding_window = sliding_window
|
||||
self.layer_types = layer_types
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
self.mup_enabled = mup_enabled
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
|
||||
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
|
||||
|
||||
__all__ = ["AfmoeConfig"]
|
||||
@@ -1,7 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLConfig
|
||||
|
||||
|
||||
class Cosmos3Config(Qwen3VLConfig):
|
||||
model_type = "cosmos3_omni"
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import NemotronHConfig
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
|
||||
|
||||
from .nemotron_h import NemotronHConfig
|
||||
|
||||
|
||||
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
|
||||
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
|
||||
@@ -31,11 +30,14 @@ class Cosmos3EdgeTextConfig(NemotronHConfig):
|
||||
num_hidden_layers: int = 28,
|
||||
hidden_act: str = "relu2",
|
||||
rms_norm_eps: float = 1e-5,
|
||||
layers_block_type: list[str] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Each decoder block is an attention layer followed by an MLP layer
|
||||
if layers_block_type is None:
|
||||
layers_block_type = ["full_attention", "mlp"] * num_hidden_layers
|
||||
super().__init__(
|
||||
num_hidden_layers=2 * num_hidden_layers,
|
||||
hybrid_override_pattern="*-" * num_hidden_layers,
|
||||
layers_block_type=layers_block_type,
|
||||
mlp_hidden_act=hidden_act,
|
||||
layer_norm_epsilon=rms_norm_eps,
|
||||
**kwargs,
|
||||
@@ -46,7 +48,6 @@ class Cosmos3EdgeTextConfig(NemotronHConfig):
|
||||
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
|
||||
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
|
||||
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
|
||||
config_dict.pop("hybrid_override_pattern", None)
|
||||
return config_dict
|
||||
|
||||
|
||||
|
||||
@@ -90,11 +90,6 @@ class MlpProjectorConfig(PreTrainedConfig):
|
||||
|
||||
@strict
|
||||
class DeepseekVLV2TextConfig(DeepseekV2Config):
|
||||
# DeepSeek-VL2 checkpoints (e.g. deepseek-vl2-small) omit ``vocab_size`` and
|
||||
# rely on the original DeepSeek-VL2 default of 102400; the generic
|
||||
# Transformers ``DeepseekV2Config`` incorrectly defaults it to 32000.
|
||||
# Upstream fix: https://github.com/huggingface/transformers/pull/48159
|
||||
vocab_size: int = 102400
|
||||
kv_lora_rank: int | None = DeepseekV2Config.kv_lora_rank
|
||||
|
||||
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.models.gemma4.configuration_gemma4 import Gemma4VisionConfig
|
||||
|
||||
|
||||
def _init_text_config(self: PreTrainedConfig, **kwargs: Any) -> None:
|
||||
PreTrainedConfig.__init__(self, **kwargs)
|
||||
# DiffusionGemma always uses MoE and K=V sharing for full_attention
|
||||
# layers. The HF reference removed these config fields entirely.
|
||||
if getattr(self, "num_experts", None):
|
||||
self.enable_moe_block = True
|
||||
self.attention_k_eq_v = True
|
||||
|
||||
|
||||
class DiffusionGemmaTextConfig(PreTrainedConfig):
|
||||
model_type = "diffusion_gemma_text"
|
||||
|
||||
def __init__(self, **kwargs: Any):
|
||||
_init_text_config(self, **kwargs)
|
||||
|
||||
|
||||
class DiffusionGemmaConfig(PreTrainedConfig):
|
||||
model_type = "diffusion_gemma"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict[str, Any] | None = None,
|
||||
canvas_length: int = 256,
|
||||
self_conditioning_size: int | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
self.text_config = DiffusionGemmaTextConfig(**(text_config or {}))
|
||||
self.canvas_length = canvas_length
|
||||
self.self_conditioning_size = self_conditioning_size
|
||||
vision_config = kwargs.pop("vision_config", None)
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = Gemma4VisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
self.audio_config = None
|
||||
PreTrainedConfig.__init__(self, **kwargs)
|
||||
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-layer config resolution for Gemma4 and its variants."""
|
||||
|
||||
from copy import copy
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
|
||||
def gemma4_layer_config(
|
||||
text_config: PreTrainedConfig, layer_idx: int
|
||||
) -> PreTrainedConfig:
|
||||
"""The Gemma4 text config as it applies to one layer.
|
||||
|
||||
Gemma4 uses a larger head dimension on its full attention layers than on its
|
||||
sliding ones, and with `attention_k_eq_v` it uses more KV heads there too.
|
||||
Transformers >= 5.15.0 says so in the config itself; before that the values
|
||||
are flat attributes that `layer_types` picks between, so resolve them here.
|
||||
The result is homogeneous either way, so callers read `head_dim` and
|
||||
`num_key_value_heads` off it without caring which version they are on.
|
||||
"""
|
||||
if getattr(text_config, "is_heterogeneous", False):
|
||||
return text_config.per_layer_config[layer_idx]
|
||||
|
||||
layer = copy(text_config)
|
||||
if text_config.layer_types[layer_idx] == "full_attention":
|
||||
global_head_dim = getattr(text_config, "global_head_dim", None)
|
||||
layer.head_dim = global_head_dim or text_config.head_dim
|
||||
if getattr(text_config, "attention_k_eq_v", False):
|
||||
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
|
||||
layer.num_key_value_heads = (
|
||||
global_kv_heads or text_config.num_key_value_heads
|
||||
)
|
||||
return layer
|
||||
@@ -1,94 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import transformers
|
||||
|
||||
|
||||
class Granite4VisionConfig(transformers.PreTrainedConfig):
|
||||
"""Configuration for Granite 4 Vision model.
|
||||
|
||||
This config is needed because the granite4_vision model type is not yet
|
||||
in the transformers version pinned by vLLM. Once transformers adds native
|
||||
support, this file can be removed and the _CONFIG_REGISTRY entry dropped.
|
||||
"""
|
||||
|
||||
model_type = "granite4_vision"
|
||||
is_composition = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict[str, Any] | None = None,
|
||||
text_config: dict[str, Any] | None = None,
|
||||
image_token_index: int = 100352,
|
||||
image_seq_length: int = 576,
|
||||
image_grid_pinpoints: list[list[int]] | None = None,
|
||||
vision_feature_select_strategy: str = "full",
|
||||
vision_feature_layer: int | list[int] = -2,
|
||||
projector_hidden_act: str = "gelu",
|
||||
projector_dropout: float = 0.1,
|
||||
downsample_rate: str | None = None,
|
||||
use_image_newline_parameter: bool = True,
|
||||
deepstack_layer_map: list[list[int]] | None = None,
|
||||
use_spatial_sampling: bool = False,
|
||||
spatial_stride: int = 2,
|
||||
spatial_vision_layer: int = -1,
|
||||
spatial_target_layers: list[int] | None = None,
|
||||
# Hub aliases — base model config uses different field names
|
||||
vision_layer_to_llm_layer: list[list[int]] | None = None,
|
||||
use_checkerboard_sampling: bool | None = None,
|
||||
checkerboard_stride: int | None = None,
|
||||
checkerboard_vision_layer: int | None = None,
|
||||
checkerboard_llm_layers: list[int] | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
self.image_token_index = image_token_index
|
||||
self.image_seq_length = image_seq_length
|
||||
self.image_grid_pinpoints = image_grid_pinpoints or []
|
||||
self.vision_feature_select_strategy = vision_feature_select_strategy
|
||||
self.vision_feature_layer = vision_feature_layer
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_dropout = projector_dropout
|
||||
self.downsample_rate = downsample_rate
|
||||
self.use_image_newline_parameter = use_image_newline_parameter
|
||||
self.deepstack_layer_map = deepstack_layer_map or vision_layer_to_llm_layer
|
||||
self.use_spatial_sampling = (
|
||||
use_spatial_sampling
|
||||
if use_checkerboard_sampling is None
|
||||
else use_checkerboard_sampling
|
||||
)
|
||||
self.spatial_stride = (
|
||||
spatial_stride if checkerboard_stride is None else checkerboard_stride
|
||||
)
|
||||
self.spatial_vision_layer = (
|
||||
spatial_vision_layer
|
||||
if checkerboard_vision_layer is None
|
||||
else checkerboard_vision_layer
|
||||
)
|
||||
self.spatial_target_layers = (
|
||||
spatial_target_layers or checkerboard_llm_layers or [0, 10, 20, 30]
|
||||
)
|
||||
|
||||
if vision_config is None:
|
||||
vision_config = {}
|
||||
if text_config is None:
|
||||
text_config = {}
|
||||
|
||||
vision_model_type = vision_config.get("model_type", "siglip_vision_model")
|
||||
if vision_model_type in transformers.CONFIG_MAPPING:
|
||||
self.vision_config = transformers.CONFIG_MAPPING[vision_model_type](
|
||||
**vision_config
|
||||
)
|
||||
else:
|
||||
self.vision_config = transformers.PreTrainedConfig(**vision_config)
|
||||
|
||||
text_model_type = text_config.get("model_type", "granite")
|
||||
if text_model_type in transformers.CONFIG_MAPPING:
|
||||
self.text_config = transformers.CONFIG_MAPPING[text_model_type](
|
||||
**text_config
|
||||
)
|
||||
else:
|
||||
self.text_config = transformers.PreTrainedConfig(**text_config)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
@@ -1,185 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class HYV3Config(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a [`HYV3Model`].
|
||||
It is used to instantiate a HYV3 model (HY V3 MoE language model) according to
|
||||
the specified arguments.
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to
|
||||
control the model outputs. Read the documentation from [`PreTrainedConfig`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 120832):
|
||||
Vocabulary size of the model.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 13312):
|
||||
Dimension of the dense FFN intermediate representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 80):
|
||||
Number of hidden layers in the Transformer decoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 64):
|
||||
Number of attention heads for each attention layer.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
Number of key-value heads for grouped-query attention.
|
||||
head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension per attention head.
|
||||
hidden_act (`str`, *optional*, defaults to `"silu"`):
|
||||
Activation function used in FFN layers.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 131072):
|
||||
Maximum sequence length supported by the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.006):
|
||||
Standard deviation of the truncated normal initializer for weight
|
||||
initialization.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-5):
|
||||
Epsilon for RMS normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use KV cache for decoding.
|
||||
pad_token_id (`int`, *optional*):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*):
|
||||
Beginning-of-sequence token id.
|
||||
eos_token_id (`int` or `List[int]`, *optional*):
|
||||
End-of-sequence token id(s).
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings.
|
||||
qk_norm (`bool`, *optional*, defaults to `True`):
|
||||
Whether to apply RMSNorm to query and key states before attention.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie input and output embedding weights.
|
||||
enable_attention_fp32_softmax (`bool`, *optional*, defaults to `False`):
|
||||
Whether to upcast attention softmax to float32. Note: the eager attention
|
||||
path always computes softmax in float32 regardless of this setting; this
|
||||
flag is reserved for future use with custom attention backends.
|
||||
enable_lm_head_fp32 (`bool`, *optional*, defaults to `True`):
|
||||
Whether to upcast the LM head computation to float32.
|
||||
num_experts (`int`, *optional*, defaults to 192):
|
||||
Total number of MoE experts.
|
||||
num_experts_per_tok (`int`, *optional*, defaults to 8):
|
||||
Number of experts selected per token (top-k routing).
|
||||
num_shared_experts (`int`, *optional*, defaults to 1):
|
||||
Number of always-active shared experts combined into a single MLP.
|
||||
expert_hidden_dim (`int`, *optional*, defaults to 1536):
|
||||
Intermediate dimension of each individual MoE expert.
|
||||
moe_router_enable_expert_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use per-expert load-balancing bias in the router.
|
||||
moe_router_use_sigmoid (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use sigmoid (instead of softmax) for router scoring.
|
||||
route_norm (`bool`, *optional*, defaults to `True`):
|
||||
Whether to normalize routing scores when using sigmoid routing.
|
||||
router_scaling_factor (`float`, *optional*):
|
||||
Optional multiplicative scaling factor applied to routing scores.
|
||||
use_grouped_mm (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use grouped GEMM for expert computation (not yet implemented).
|
||||
enable_moe_fp32_combine (`bool`, *optional*, defaults to `False`):
|
||||
Whether to accumulate expert outputs in float32.
|
||||
first_k_dense_replace (`int`, *optional*, defaults to 1):
|
||||
Number of initial decoder layers that use a dense FFN instead of MoE.
|
||||
output_router_logits (`bool`, *optional*, defaults to `False`):
|
||||
Whether to output router logits from each MoE layer. Useful for computing
|
||||
auxiliary load-balancing loss during training. Disabled by default to avoid
|
||||
the memory overhead of storing per-layer router tensors during inference.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from transformers import HYV3Config, HYV3Model
|
||||
|
||||
>>> config = HYV3Config()
|
||||
>>> model = HYV3Model(config)
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
model_type = "hy_v3"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=120832,
|
||||
hidden_size=4096,
|
||||
intermediate_size=13312,
|
||||
num_hidden_layers=80,
|
||||
num_attention_heads=64,
|
||||
num_key_value_heads=8,
|
||||
head_dim=128,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=131072,
|
||||
initializer_range=0.006,
|
||||
rms_norm_eps=1e-5,
|
||||
use_cache=True,
|
||||
pad_token_id=None,
|
||||
bos_token_id=None,
|
||||
eos_token_id=None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
qk_norm=True,
|
||||
tie_word_embeddings=False,
|
||||
enable_attention_fp32_softmax=False,
|
||||
enable_lm_head_fp32=True,
|
||||
# MoE specific
|
||||
num_experts=192,
|
||||
num_experts_per_tok=8,
|
||||
num_shared_experts=1,
|
||||
expert_hidden_dim=1536,
|
||||
moe_router_enable_expert_bias=True,
|
||||
moe_router_use_sigmoid=True,
|
||||
route_norm=True,
|
||||
router_scaling_factor=None,
|
||||
use_grouped_mm=False,
|
||||
enable_moe_fp32_combine=False,
|
||||
# Dense/MoE layer control
|
||||
first_k_dense_replace=1,
|
||||
output_router_logits=False,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_act = hidden_act
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
rope_theta = kwargs.pop("rope_theta", 11158840.0)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.qk_norm = qk_norm
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.enable_lm_head_fp32 = enable_lm_head_fp32
|
||||
self.enable_attention_fp32_softmax = enable_attention_fp32_softmax
|
||||
|
||||
# MoE specific
|
||||
self.num_experts = num_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.expert_hidden_dim = expert_hidden_dim
|
||||
self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
|
||||
self.moe_router_use_sigmoid = moe_router_use_sigmoid
|
||||
self.route_norm = route_norm
|
||||
self.use_grouped_mm = use_grouped_mm
|
||||
self.router_scaling_factor = router_scaling_factor
|
||||
self.enable_moe_fp32_combine = enable_moe_fp32_combine
|
||||
|
||||
# Dense/MoE layer control
|
||||
self.first_k_dense_replace = first_k_dense_replace
|
||||
self.output_router_logits = output_router_logits
|
||||
|
||||
if eos_token_id is not None and isinstance(eos_token_id, int):
|
||||
eos_token_id = [eos_token_id]
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,277 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright 2025 NAVER Cloud HyperCLOVA team
|
||||
#
|
||||
# Copyright 2025 NAVER Cloud HyperCLOVA team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""HyperCLOVA X model configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class HyperCLOVAXConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a
|
||||
[`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model
|
||||
according to the specified arguments, defining the model architecture.
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used
|
||||
to control the model outputs. Read the documentation from
|
||||
[`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 32000):
|
||||
Vocabulary size of the HyperCLOVAX model. Defines the number of
|
||||
different tokens that can be represented by the `input_ids`
|
||||
passed when calling [`HyperCLOVAXModel`]
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 11008):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 32):
|
||||
Number of hidden layers in the Transformer decoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the
|
||||
Transformer decoder.
|
||||
num_key_value_heads (`int`, *optional*):
|
||||
This is the number of key_value heads that should be used to
|
||||
implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use
|
||||
Multi Head Attention (MHA), if `num_key_value_heads=1` the model
|
||||
will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each
|
||||
group key and value head should be constructed by meanpooling all
|
||||
the original heads within that group. For more details checkout
|
||||
[this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
|
||||
specified, will default to `num_attention_heads`.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
||||
The non-linear activation function (function or string) in the
|
||||
decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 2048):
|
||||
The maximum sequence length that this model might ever be used
|
||||
with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for
|
||||
initializing all weight matrices.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the rms normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values
|
||||
attentions (not used by all models). Only relevant if
|
||||
`config.is_decoder=True`.
|
||||
pad_token_id (`int`, *optional*):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
Beginning of stream token id.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
End of stream token id.
|
||||
pretraining_tp (`int`, *optional*, defaults to 1):
|
||||
Experimental feature. Tensor parallelism rank used during
|
||||
pretraining. Please refer to [this document](https://huggingface.
|
||||
co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism)
|
||||
to understand more about it. This value is necessary to ensure
|
||||
exact reproducibility of the pretraining results. Please refer to
|
||||
[this issue](https://github.com/pytorch/pytorch/issues/76232).
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie weight embeddings
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_scaling (`Dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE
|
||||
embeddings. NOTE: if you apply new rope type and you expect the
|
||||
model to work on longer `max_position_embeddings`, we recommend
|
||||
you to update this value accordingly.
|
||||
Expected contents:
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default',
|
||||
'linear', 'dynamic', 'yarn', 'longrope', 'llama3'], with
|
||||
'default' being the original RoPE implementation.
|
||||
`factor` (`float`, *optional*):
|
||||
Used with all rope types except 'default'. The scaling
|
||||
factor to apply to the RoPE embeddings. In most scaling
|
||||
types, a `factor` of x will enable the model to handle
|
||||
sequences of length x * original maximum pre-trained
|
||||
length.
|
||||
`original_max_position_embeddings` (`int`, *optional*):
|
||||
Used with 'dynamic', 'longrope' and 'llama3'. The
|
||||
original max position embeddings used during pretraining.
|
||||
`attention_factor` (`float`, *optional*):
|
||||
Used with 'yarn' and 'longrope'. The scaling factor to be
|
||||
applied on the attention computation. If unspecified, it
|
||||
defaults to value recommended by the implementation, using
|
||||
the `factor` field to infer the suggested value.
|
||||
`beta_fast` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for
|
||||
extrapolation (only) in the linear ramp function. If
|
||||
unspecified, it defaults to 32.
|
||||
`beta_slow` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for
|
||||
interpolation (only) in the linear ramp function. If
|
||||
unspecified, it defaults to 1.
|
||||
`short_factor` (`List[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be
|
||||
applied to short contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of
|
||||
numbers with the same length as the hidden size divided
|
||||
by the number of attention heads divided by 2
|
||||
`long_factor` (`List[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be
|
||||
applied to long contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of
|
||||
numbers with the same length as the hidden size divided
|
||||
by the number of attention heads divided by 2
|
||||
`low_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to low
|
||||
frequency components of the RoPE
|
||||
`high_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to high
|
||||
frequency components of the RoPE
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output
|
||||
projection layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in up_proj, down_proj and gate_proj layers
|
||||
in the MLP layers.
|
||||
head_dim (`int`, *optional*):
|
||||
The attention head dimension. If None, it will default to
|
||||
hidden_size // num_heads
|
||||
embedding_multiplier (`float`, *optional*, defaults to `None`):
|
||||
Multiplier applied to the embedding weights. If `None`, it is
|
||||
equivalent to `1.0`.
|
||||
logits_scaling (`float`, *optional*, defaults to `None`):
|
||||
Scaling factor for logits. If `None`, it is equivalent to `1.0`.
|
||||
attention_multiplier (`float`, *optional*, defaults to `None`):
|
||||
Multiplier applied to the attention weights. If `None`, it is
|
||||
equivalent to `self.head_dim ** -0.5`.
|
||||
residual_multiplier (`float`, *optional*, defaults to `None`):
|
||||
Scaling factor for residual connections. If `None`, it is
|
||||
equivalent to `1.0`.
|
||||
use_post_norm (`bool`, *optional*, defaults to `True`):
|
||||
Determines whether to apply Peri-Layer Normalization. Set to
|
||||
False to disable this feature.
|
||||
rope_parameters (`dict`, *optional*):
|
||||
Dictionary containing the RoPE parameters used by vLLM's
|
||||
`get_rope`. When provided, takes precedence over `rope_theta`
|
||||
and `rope_scaling`. If `None`, it is derived from `rope_theta`
|
||||
and `rope_scaling` automatically.
|
||||
|
||||
"""
|
||||
|
||||
model_type = "hyperclovax"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=32000,
|
||||
hidden_size=4096,
|
||||
intermediate_size=11008,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=None,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=2048,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
pad_token_id=None,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
pretraining_tp=1,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=10000.0,
|
||||
rope_scaling=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
mlp_bias=False,
|
||||
head_dim=None,
|
||||
embedding_multiplier=None, # mup
|
||||
logits_scaling=None, # mup
|
||||
attention_multiplier=None, # mup
|
||||
residual_multiplier=None, # mup
|
||||
use_post_norm=True, # post-norm(peri-LN)
|
||||
rope_parameters=None,
|
||||
auto_map=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.pretraining_tp = pretraining_tp
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.mlp_bias = mlp_bias
|
||||
self.head_dim = (
|
||||
head_dim
|
||||
if head_dim is not None
|
||||
else self.hidden_size // self.num_attention_heads
|
||||
)
|
||||
# Derive rope_parameters for vLLM's get_rope() from rope_theta /
|
||||
# rope_scaling, unless the caller already provided rope_parameters.
|
||||
if rope_parameters is None:
|
||||
if rope_scaling is not None:
|
||||
# Shallow-copy to avoid mutating the caller's dict.
|
||||
rope_parameters = dict(rope_scaling)
|
||||
# BC: 'type' field -> 'rope_type', remove stale key.
|
||||
if "type" in rope_parameters:
|
||||
rope_parameters.setdefault("rope_type", rope_parameters.pop("type"))
|
||||
else:
|
||||
rope_parameters = {"rope_type": "default"}
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
# BC: keep self.rope_scaling consistent for HF serialization.
|
||||
if self.rope_scaling is not None and "type" in self.rope_scaling:
|
||||
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
|
||||
|
||||
# mup
|
||||
self.embedding_multiplier = (
|
||||
embedding_multiplier if embedding_multiplier is not None else 1.0
|
||||
)
|
||||
self.logits_scaling = logits_scaling if logits_scaling is not None else 1.0
|
||||
self.attention_multiplier = (
|
||||
attention_multiplier
|
||||
if attention_multiplier is not None
|
||||
else self.head_dim**-0.5
|
||||
)
|
||||
self.residual_multiplier = (
|
||||
residual_multiplier if residual_multiplier is not None else 1.0
|
||||
)
|
||||
|
||||
# post-norm (Peri-LN)
|
||||
self.use_post_norm = use_post_norm
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
auto_map=auto_map,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,131 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K2.5 Model Configuration.
|
||||
|
||||
This configuration supports video-chunk as an internal modality type.
|
||||
A video-chunk is the smallest independently processable unit of video.
|
||||
"""
|
||||
|
||||
from transformers import DeepseekV3Config
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class KimiK25VisionConfig(PreTrainedConfig):
|
||||
model_type = "kimi_k25_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
# Vision Tower
|
||||
patch_size: int = 14,
|
||||
init_pos_emb_height: int = 64,
|
||||
init_pos_emb_width: int = 64,
|
||||
init_pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
num_attention_heads: int = 16,
|
||||
num_hidden_layers: int = 27,
|
||||
hidden_size: int = 1152,
|
||||
intermediate_size: int = 4304,
|
||||
merge_kernel_size: tuple[int, int] = (2, 2),
|
||||
video_attn_type: str = "spatial_temporal",
|
||||
merge_type: str = "sd2_tpool",
|
||||
# MM Projector
|
||||
mm_projector_type: str = "patchmerger",
|
||||
mm_hidden_size: int | None = None,
|
||||
projector_hidden_act: str = "gelu",
|
||||
projector_ln_eps: float = 1e-5,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
# Vision Tower
|
||||
self.patch_size = patch_size
|
||||
self.init_pos_emb_height = init_pos_emb_height
|
||||
self.init_pos_emb_width = init_pos_emb_width
|
||||
self.init_pos_emb_time = init_pos_emb_time
|
||||
self.pos_emb_type = pos_emb_type
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.merge_kernel_size = merge_kernel_size
|
||||
self.video_attn_type = video_attn_type
|
||||
self.merge_type = merge_type
|
||||
# MM Projector
|
||||
self.mm_projector_type = mm_projector_type
|
||||
if mm_hidden_size is not None:
|
||||
self.mm_hidden_size = mm_hidden_size
|
||||
else:
|
||||
self.mm_hidden_size = hidden_size
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_ln_eps = projector_ln_eps
|
||||
|
||||
|
||||
class KimiK25Config(PreTrainedConfig):
|
||||
"""Kimi-K2.5 model configuration.
|
||||
|
||||
Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks.
|
||||
A video-chunk consists of multiple consecutive frames
|
||||
that are processed together with temporal pooling.
|
||||
|
||||
Args:
|
||||
vision_config: Configuration for the vision tower and projector.
|
||||
text_config: Configuration for the text model (DeepseekV3).
|
||||
ignore_index: The ignore index for the loss function.
|
||||
media_placeholder_token_id: The token ID for media placeholders.
|
||||
pad_token_id: The token ID for padding.
|
||||
|
||||
"""
|
||||
|
||||
model_type = "kimi_k25"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict | KimiK25VisionConfig | None = None,
|
||||
text_config: dict | DeepseekV3Config | None = None,
|
||||
ignore_index: int = -100,
|
||||
media_placeholder_token_id: int = 163605,
|
||||
pad_token_id: int = 0,
|
||||
use_unified_vision_chunk: bool = False,
|
||||
video_placeholder: str = "<|kimi_k25_video_placeholder|>",
|
||||
**kwargs,
|
||||
):
|
||||
# Vision config
|
||||
if vision_config is None:
|
||||
self.vision_config = KimiK25VisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = KimiK25VisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
# Text config
|
||||
if text_config is None:
|
||||
self.text_config = DeepseekV3Config()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = DeepseekV3Config(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
# Set mm_hidden_size to text hidden size if not explicitly set
|
||||
if self.vision_config.mm_hidden_size == self.vision_config.hidden_size:
|
||||
self.vision_config.mm_hidden_size = self.text_config.hidden_size
|
||||
|
||||
# Other config
|
||||
self.ignore_index = ignore_index
|
||||
self.media_placeholder_token_id = media_placeholder_token_id
|
||||
self.use_unified_vision_chunk = use_unified_vision_chunk
|
||||
self.video_placeholder = video_placeholder
|
||||
|
||||
# Propagate quantization config from text model
|
||||
if getattr(self.text_config, "quantization_config", None) is not None:
|
||||
self.quantization_config = self.text_config.quantization_config
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
||||
|
||||
@property
|
||||
def hidden_size(self) -> int:
|
||||
"""Get hidden size from text config for compatibility."""
|
||||
return self.text_config.hidden_size
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
"""Get vocab size from text config for compatibility."""
|
||||
return self.text_config.vocab_size
|
||||
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class Lfm2MoeConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a [`Lfm2MoeModel`]. It is used to instantiate a LFM2 Moe
|
||||
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
|
||||
defaults will yield a similar configuration to that of the LFM2-8B-A1B model.
|
||||
e.g. [LiquidAI/LFM2-8B-A1B](https://huggingface.co/LiquidAI/LFM2-8B-A1B)
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 65536):
|
||||
Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`Lfm2Model`]
|
||||
hidden_size (`int`, *optional*, defaults to 2048):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 7168):
|
||||
Dimension of the MLP representations.
|
||||
moe_intermediate_size (`int`, *optional*, defaults to 1792):
|
||||
Intermediate size of the routed expert.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 32):
|
||||
Number of hidden layers in the Transformer decoder.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
Beginning of stream token id.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
End of stream token id.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `True`):
|
||||
Whether to tie weight embeddings
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 128000):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
norm_eps (`float`, *optional*, defaults to 1e-05):
|
||||
The epsilon used by the rms normalization layers.
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer decoder.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
||||
by meanpooling all the original heads within that group. For more details, check out [this
|
||||
paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
|
||||
`num_attention_heads`.
|
||||
conv_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the conv layers.
|
||||
conv_L_cache (`int`, *optional*, defaults to 3):
|
||||
L_cache dim in the conv layers.
|
||||
num_dense_layers (`int`, *optional*, defaults to 2):
|
||||
Number of dense Lfm2MoeMLP layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
|
||||
num_experts_per_tok (`int`, *optional*, defaults to 4):
|
||||
Number of selected experts.
|
||||
num_experts (`int`, *optional*, defaults to 32):
|
||||
Number of routed experts.
|
||||
use_expert_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use the expert bias on the routing weights.
|
||||
routed_scaling_factor (`float`, *optional*, defaults to 1.0):
|
||||
Scaling factor for routed experts in MoE models.
|
||||
norm_topk_prob (`bool`, *optional*, defaults to `True`):
|
||||
Whether to normalize the topk probabilities.
|
||||
layer_types (`Optional`, *optional*):
|
||||
Type of each layers.
|
||||
|
||||
```python
|
||||
>>> from transformers import Lfm2MoeModel, Lfm2MoeConfig
|
||||
|
||||
>>> # Initializing a LFM2 Moe model
|
||||
>>> configuration = Lfm2MoeConfig()
|
||||
|
||||
>>> # Initializing a model from the LFM2-8B-A1B style configuration
|
||||
>>> model = Lfm2MoeModel(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
model_type = "lfm2_moe"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 65536,
|
||||
hidden_size: int = 2048,
|
||||
intermediate_size: int = 7168,
|
||||
moe_intermediate_size: int = 1792,
|
||||
num_hidden_layers: int = 32,
|
||||
pad_token_id: int = 0,
|
||||
bos_token_id: int = 1,
|
||||
eos_token_id: int = 2,
|
||||
tie_word_embeddings: bool = True,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
max_position_embeddings: int = 128_000,
|
||||
use_cache: bool = True,
|
||||
norm_eps: float = 0.00001,
|
||||
num_attention_heads: int = 32,
|
||||
num_key_value_heads: int = 8,
|
||||
conv_bias: bool = False,
|
||||
conv_L_cache: int = 3,
|
||||
num_dense_layers: int = 2,
|
||||
num_experts_per_tok: int = 4,
|
||||
num_experts: int = 32,
|
||||
use_expert_bias: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
norm_topk_prob: bool = True,
|
||||
layer_types: list[str] | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
rope_theta = kwargs.pop("rope_theta", 1000000.0)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.use_cache = use_cache
|
||||
self.norm_eps = norm_eps
|
||||
|
||||
# attn operator config
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
|
||||
# custom operator config
|
||||
self.conv_bias = conv_bias
|
||||
self.conv_L_cache = conv_L_cache
|
||||
|
||||
# moe config
|
||||
self.num_dense_layers = num_dense_layers
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_experts = num_experts
|
||||
self.use_expert_bias = use_expert_bias
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
self.layer_types = layer_types
|
||||
|
||||
tie_word_embeddings = kwargs.get(
|
||||
"tie_embedding", tie_word_embeddings
|
||||
) # to fit original config keys
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["Lfm2MoeConfig"]
|
||||
@@ -1,219 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Nemotron model configuration."""
|
||||
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a
|
||||
[`NemotronModel`]. It is used to instantiate a Nemotron model
|
||||
according to the specified arguments, defining the model architecture.
|
||||
Instantiating a configuration with the defaults will yield a similar
|
||||
configuration to that of the Nemotron-8B.
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be
|
||||
used to control the model outputs. Read the documentation from
|
||||
[`PreTrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 256000):
|
||||
Vocabulary size of the Nemotron model. Defines the number of
|
||||
different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronModel`]
|
||||
hidden_size (`int`, *optional*, defaults to 6144):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 24576):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 32):
|
||||
Number of hidden layers in the Transformer decoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 48):
|
||||
Number of attention heads for each attention layer in the
|
||||
Transformer decoder.
|
||||
head_dim (`int`, *optional*):
|
||||
Projection weights dimension in multi-head attention. Set to
|
||||
hidden_size // num_attention_heads if None
|
||||
num_key_value_heads (`int`, *optional*):
|
||||
This is the number of key_value heads that should be used to
|
||||
implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use
|
||||
Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1 the model will use Multi Query Attention
|
||||
(MQA) otherwise GQA is used. When converting a multi-head
|
||||
checkpoint to a GQA checkpoint, each group key and value
|
||||
head should be constructed by meanpooling all the original
|
||||
heads within that group. For more details checkout
|
||||
[this paper](https://arxiv.org/pdf/2305.13245.pdf). If it
|
||||
is not specified, will default to `num_attention_heads`.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"relu2"`):
|
||||
The non-linear activation function (function or string) in the
|
||||
decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used
|
||||
with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.0134):
|
||||
The standard deviation of the truncated_normal_initializer for
|
||||
initializing all weight matrices.
|
||||
norm_eps (`float`, *optional*, defaults to 1e-05):
|
||||
The epsilon used by the normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values
|
||||
attentions (not used by all models). Only relevant if
|
||||
`config.is_decoder=True`.
|
||||
pad_token_id (`int`, *optional*):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*, defaults to 2):
|
||||
Beginning of stream token id.
|
||||
eos_token_id (`int`, *optional*, defaults to 3):
|
||||
End of stream token id.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie weight embeddings
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings. Expected contents:
|
||||
`rope_theta` (`float`): The base period of the RoPE embeddings.
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear',
|
||||
'dynamic', 'yarn', 'longrope', 'llama3'], with 'default' being the
|
||||
original RoPE implementation.
|
||||
`partial_rotary_factor` (`float`, *optional*, defaults to 0.5):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output
|
||||
projection layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in up_proj and down_proj layers in the MLP
|
||||
layers.
|
||||
|
||||
```python
|
||||
>>> from transformers import NemotronModel, NemotronConfig
|
||||
>>> # Initializing a Nemotron nemotron-15b style configuration
|
||||
>>> configuration = NemotronConfig()
|
||||
>>> # Initializing a model from the nemotron-15b style configuration
|
||||
>>> model = NemotronModel(configuration)
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
model_type = "nemotron"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=256000,
|
||||
hidden_size=6144,
|
||||
intermediate_size=24576,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=48,
|
||||
head_dim=None,
|
||||
num_key_value_heads=None,
|
||||
hidden_act="relu2",
|
||||
max_position_embeddings=4096,
|
||||
initializer_range=0.0134,
|
||||
norm_eps=1e-5,
|
||||
use_cache=True,
|
||||
pad_token_id=None,
|
||||
bos_token_id=2,
|
||||
eos_token_id=3,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
mlp_bias=False,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
head_dim = head_dim or kwargs.get("kv_channels")
|
||||
self.head_dim = (
|
||||
head_dim if head_dim is not None else (hidden_size // num_attention_heads)
|
||||
)
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.norm_eps = norm_eps
|
||||
self.use_cache = use_cache
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
# for backward compatibility
|
||||
partial_rotary_factor = (
|
||||
kwargs.get("rope_percent")
|
||||
or kwargs.get("rope_percentage")
|
||||
or kwargs.get("partial_rotary_factor")
|
||||
or 0.5
|
||||
)
|
||||
if "partial_rotary_factor" not in rope_parameters:
|
||||
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
|
||||
self.rope_parameters = rope_parameters
|
||||
self._rope_parameters_validation()
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.mlp_bias = mlp_bias
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _rope_parameters_validation(self):
|
||||
"""Validate the `rope_parameters` configuration."""
|
||||
if self.rope_parameters is None:
|
||||
return
|
||||
|
||||
rope_type: str | None = self.rope_parameters.get("rope_type", None)
|
||||
factor: float | None = self.rope_parameters.get("factor", None)
|
||||
|
||||
if rope_type not in {"default", "linear", "dynamic"}:
|
||||
raise ValueError(
|
||||
"`rope_type` must be one of ['default', 'linear', 'dynamic'], "
|
||||
f"got {rope_type}"
|
||||
)
|
||||
if rope_type != "default":
|
||||
if factor is None:
|
||||
raise ValueError(
|
||||
"If `rope_type` is not 'default', `rope_parameters` "
|
||||
"must include a `factor` field. Got `None`."
|
||||
)
|
||||
if not isinstance(factor, float) or factor <= 1.0:
|
||||
raise ValueError(
|
||||
"`rope_parameters`'s factor field must be a float > 1, got "
|
||||
f"{factor}"
|
||||
)
|
||||
@@ -1,288 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration."""
|
||||
|
||||
import regex as re
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a
|
||||
[`NemotronHModel`]. It is used to instantiate a NemotronH model according
|
||||
to the specified arguments, defining the model architecture. Instantiating
|
||||
a configuration with the defaults will yield a similar configuration to
|
||||
that of the NemotronH-v0.1 model.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of
|
||||
different tokens that can be represented by the `inputs_ids`
|
||||
passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be
|
||||
tied. Note that this is only relevant if the model has an output
|
||||
word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to
|
||||
`"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of
|
||||
characters where each character represents
|
||||
M: Mamba2, *: Attention, -: MLP
|
||||
mtp_hybrid_override_pattern (`str`, *optional*, defaults to `"*E"`):
|
||||
The pattern of the MTP layers.
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the
|
||||
Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to
|
||||
implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use
|
||||
Multi Head Attention (MHA), if `num_key_value_heads=1` the model
|
||||
will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for
|
||||
initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False`
|
||||
residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values
|
||||
attentions (not used by all models). Only relevant if
|
||||
`config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`,
|
||||
all logits will be calculated. If an integer value, only last
|
||||
`num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used
|
||||
with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels.
|
||||
These are available only if `mamba-ssm` and `causal-conv1d`
|
||||
are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer
|
||||
block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the
|
||||
mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
mtp_hybrid_override_pattern="*E",
|
||||
num_attention_heads=32,
|
||||
head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
n_routed_experts=8,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=7688,
|
||||
moe_shared_expert_intermediate_size=7688,
|
||||
moe_latent_size=None,
|
||||
num_experts_per_tok=2,
|
||||
routed_scaling_factor=1.0,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.mtp_hybrid_override_pattern = mtp_hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.head_dim = head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, (
|
||||
"hybrid_override_pattern must have same length as num_hidden_layers"
|
||||
)
|
||||
assert re.match(r"^[*-ME]+$", self.hybrid_override_pattern), (
|
||||
"hybrid_override_pattern must only contain characters 'M', '*', '-', or 'E'"
|
||||
)
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
self.n_routed_experts = n_routed_experts
|
||||
self.n_shared_experts = n_shared_experts
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size # noqa: E501
|
||||
self.moe_latent_size = moe_latent_size
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.n_group = n_group
|
||||
self.topk_group = topk_group
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba"
|
||||
if self.hybrid_override_pattern[i] == "M"
|
||||
else "attention"
|
||||
if self.hybrid_override_pattern[i] == "*"
|
||||
else "mlp"
|
||||
if self.hybrid_override_pattern[i] == "-"
|
||||
else "moe"
|
||||
for i in range(self.num_hidden_layers)
|
||||
]
|
||||
@@ -1,285 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class OlmoHybridConfig(PreTrainedConfig):
|
||||
r"""Configuration class for [`OlmoHybridModel`]. It is used to
|
||||
instantiate an OLMo Hybrid model according to the specified
|
||||
arguments, defining the model architecture. Instantiating a
|
||||
configuration with the defaults will yield a similar
|
||||
configuration to that of the
|
||||
[allenai/Olmo-Hybrid-7B](https://huggingface.co/allenai/Olmo-Hybrid-7B)
|
||||
model.
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and
|
||||
can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 100352):
|
||||
Vocabulary size of the OlmoHybrid model. Defines
|
||||
the number of different tokens that can be
|
||||
represented by the `inputs_ids` passed when
|
||||
calling [`OlmoHybridModel`].
|
||||
hidden_size (`int`, *optional*, defaults to 3840):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*,
|
||||
defaults to 11008):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*,
|
||||
defaults to 32):
|
||||
Number of hidden layers in the Transformer
|
||||
decoder.
|
||||
num_attention_heads (`int`, *optional*,
|
||||
defaults to 30):
|
||||
Number of attention heads for each attention
|
||||
layer in the Transformer decoder.
|
||||
num_key_value_heads (`int`, *optional*):
|
||||
This is the number of key_value heads that
|
||||
should be used to implement Grouped Query
|
||||
Attention. If
|
||||
`num_key_value_heads=num_attention_heads`,
|
||||
the model will use Multi Head Attention (MHA),
|
||||
if `num_key_value_heads=1` the model will use
|
||||
Multi Query Attention (MQA) otherwise GQA is
|
||||
used. When converting a multi-head checkpoint
|
||||
to a GQA checkpoint, each group key and value
|
||||
head should be constructed by meanpooling all
|
||||
the original heads within that group. For more
|
||||
details, check out
|
||||
[this paper](https://huggingface.co/papers/2305.13245).
|
||||
If it is not specified, will default to
|
||||
`num_attention_heads`.
|
||||
hidden_act (`str` or `function`, *optional*,
|
||||
defaults to `"silu"`):
|
||||
The non-linear activation function (function
|
||||
or string) in the decoder.
|
||||
max_position_embeddings (`int`, *optional*,
|
||||
defaults to 65536):
|
||||
The maximum sequence length that this model
|
||||
might ever be used with.
|
||||
initializer_range (`float`, *optional*,
|
||||
defaults to 0.02):
|
||||
The standard deviation of the
|
||||
truncated_normal_initializer for initializing
|
||||
all weight matrices.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last
|
||||
key/values attentions (not used by all models).
|
||||
Only relevant if `config.is_decoder=True`.
|
||||
pad_token_id (`int`, *optional*,
|
||||
defaults to 100277):
|
||||
Padding token id.
|
||||
bos_token_id (`int`, *optional*):
|
||||
Beginning of stream token id.
|
||||
eos_token_id (`int`, *optional*,
|
||||
defaults to 100257):
|
||||
End of stream token id.
|
||||
tie_word_embeddings (`bool`, *optional*,
|
||||
defaults to `False`):
|
||||
Whether to tie weight embeddings.
|
||||
rope_parameters (`RopeParameters`, *optional*):
|
||||
Dictionary containing the configuration
|
||||
parameters for the RoPE embeddings. Can be
|
||||
`None` to disable RoPE.
|
||||
attention_bias (`bool`, *optional*,
|
||||
defaults to `False`):
|
||||
Whether to use a bias in the query, key, value
|
||||
and output projection layers during
|
||||
self-attention.
|
||||
attention_dropout (`float`, *optional*,
|
||||
defaults to 0.0):
|
||||
The dropout ratio for the attention
|
||||
probabilities.
|
||||
rms_norm_eps (`float`, *optional*,
|
||||
defaults to 1e-06):
|
||||
The epsilon used by the rms normalization
|
||||
layers.
|
||||
layer_types (`list`, *optional*):
|
||||
Attention pattern for each layer. Can contain
|
||||
`"full_attention"` or `"linear_attention"`.
|
||||
Defaults to linear attention for most layers
|
||||
with full attention for every 4th layer.
|
||||
linear_num_key_heads (`int`, *optional*):
|
||||
Number of key heads for the linear attention
|
||||
layers. Defaults to `num_attention_heads`.
|
||||
linear_num_value_heads (`int`, *optional*):
|
||||
Number of value heads for the linear attention
|
||||
layers. Defaults to `num_attention_heads`.
|
||||
linear_key_head_dim (`int`, *optional*):
|
||||
Dimension of each key head in linear attention
|
||||
layers. Defaults to
|
||||
`0.75 * hidden_size / linear_num_key_heads`.
|
||||
linear_value_head_dim (`int`, *optional*):
|
||||
Dimension of each value head in linear
|
||||
attention layers. Defaults to
|
||||
`2 * linear_key_head_dim`.
|
||||
linear_a_log_min (`float`, *optional*,
|
||||
defaults to 0.0):
|
||||
Minimum value for uniform initialization of
|
||||
A_log in GatedDeltaNet layers.
|
||||
linear_a_log_max (`float`, *optional*,
|
||||
defaults to 16.0):
|
||||
Maximum value for uniform initialization of
|
||||
A_log in GatedDeltaNet layers.
|
||||
linear_dt_min (`float`, *optional*,
|
||||
defaults to 0.001):
|
||||
Minimum value for dt initialization in
|
||||
GatedDeltaNet layers.
|
||||
linear_dt_max (`float`, *optional*,
|
||||
defaults to 0.1):
|
||||
Maximum value for dt initialization in
|
||||
GatedDeltaNet layers.
|
||||
linear_dt_init_floor (`float`, *optional*,
|
||||
defaults to 0.0001):
|
||||
Floor value for clamping dt during
|
||||
initialization in GatedDeltaNet layers.
|
||||
linear_conv_kernel_dim (`int`, *optional*,
|
||||
defaults to 4):
|
||||
Kernel size for the short convolution applied
|
||||
to queries, keys, and values in linear
|
||||
attention layers.
|
||||
linear_allow_neg_eigval (`bool`, *optional*,
|
||||
defaults to `True`):
|
||||
Whether to allow negative eigenvalues in the
|
||||
GatedDeltaNet recurrence. When `True`, the
|
||||
beta parameter is scaled by 2.0 to allow
|
||||
values in range [0, 2] instead of [0, 1].
|
||||
```python
|
||||
>>> from transformers import (
|
||||
... OlmoHybridModel,
|
||||
... OlmoHybridConfig,
|
||||
... )
|
||||
|
||||
>>> configuration = OlmoHybridConfig()
|
||||
>>> model = OlmoHybridModel(configuration)
|
||||
>>> configuration = model.config
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
model_type = "olmo_hybrid"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_proj": "colwise_gather_output",
|
||||
"layers.*.self_attn.k_proj": "colwise_gather_output",
|
||||
"layers.*.self_attn.v_proj": "colwise_gather_output",
|
||||
"layers.*.self_attn.o_proj": "rowwise_split_input",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int | None = 100352,
|
||||
hidden_size: int | None = 3840,
|
||||
intermediate_size: int | None = 11008,
|
||||
num_hidden_layers: int | None = 32,
|
||||
num_attention_heads: int | None = 30,
|
||||
num_key_value_heads: int | None = None,
|
||||
hidden_act: str | None = "silu",
|
||||
max_position_embeddings: int | None = 65536,
|
||||
initializer_range: float | None = 0.02,
|
||||
use_cache: bool | None = True,
|
||||
pad_token_id: int | None = 100277,
|
||||
bos_token_id: int | None = None,
|
||||
eos_token_id: int | None = 100257,
|
||||
tie_word_embeddings: bool | None = False,
|
||||
rope_parameters=None,
|
||||
attention_bias: bool | None = False,
|
||||
attention_dropout: float | None = 0.0,
|
||||
rms_norm_eps: float | None = 1e-06,
|
||||
layer_types: list[str] | None = None,
|
||||
linear_num_key_heads: int | None = None,
|
||||
linear_num_value_heads: int | None = None,
|
||||
linear_key_head_dim: int | None = None,
|
||||
linear_value_head_dim: int | None = None,
|
||||
linear_a_log_min: float = 0.0,
|
||||
linear_a_log_max: float = 16.0,
|
||||
linear_dt_min: float = 0.001,
|
||||
linear_dt_max: float = 0.1,
|
||||
linear_dt_init_floor: float = 1e-4,
|
||||
linear_conv_kernel_dim: int = 4,
|
||||
linear_allow_neg_eigval: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
assert num_hidden_layers is not None
|
||||
assert hidden_size is not None
|
||||
assert num_attention_heads is not None
|
||||
|
||||
if layer_types is None:
|
||||
# Default: linear attention for most layers, full attention every 4th layer
|
||||
layer_types = ["linear_attention"] * int(num_hidden_layers)
|
||||
for i in range(int(num_hidden_layers)):
|
||||
if i % 4 == 3:
|
||||
layer_types[i] = "full_attention"
|
||||
# Ensure at least one full attention layer for small num_hidden_layers
|
||||
if "full_attention" not in layer_types:
|
||||
layer_types[-1] = "full_attention"
|
||||
|
||||
self.layer_types = layer_types
|
||||
self.validate_layer_type()
|
||||
if "linear_attention" not in layer_types:
|
||||
raise ValueError(
|
||||
"OLMoHybrid expects at least one 'linear_attention' layer."
|
||||
)
|
||||
if all(t == "linear_attention" for t in layer_types):
|
||||
raise ValueError("OLMoHybrid expects at least one attention layer.")
|
||||
|
||||
self.layer_types = layer_types
|
||||
|
||||
if linear_num_key_heads is None:
|
||||
linear_num_key_heads = num_attention_heads
|
||||
if linear_num_value_heads is None:
|
||||
linear_num_value_heads = num_attention_heads
|
||||
if linear_key_head_dim is None:
|
||||
linear_key_head_dim = int(0.75 * hidden_size / linear_num_key_heads)
|
||||
if linear_value_head_dim is None:
|
||||
linear_value_head_dim = 2 * linear_key_head_dim
|
||||
|
||||
self.linear_num_key_heads = linear_num_key_heads
|
||||
self.linear_num_value_heads = linear_num_value_heads
|
||||
self.linear_key_head_dim = linear_key_head_dim
|
||||
self.linear_value_head_dim = linear_value_head_dim
|
||||
self.linear_a_log_min = linear_a_log_min
|
||||
self.linear_a_log_max = linear_a_log_max
|
||||
self.linear_dt_min = linear_dt_min
|
||||
self.linear_dt_max = linear_dt_max
|
||||
self.linear_dt_init_floor = linear_dt_init_floor
|
||||
self.linear_conv_kernel_dim = linear_conv_kernel_dim
|
||||
self.linear_allow_neg_eigval = linear_allow_neg_eigval
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.pad_token_id = pad_token_id
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
@@ -2,69 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa: E501
|
||||
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py
|
||||
# and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
|
||||
# Ovis Config with AimV2 config registration removed for Transformers compatibility
|
||||
# adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
|
||||
from typing import Any
|
||||
|
||||
from transformers import AutoConfig, PreTrainedConfig
|
||||
|
||||
|
||||
class AIMv2Config(PreTrainedConfig):
|
||||
"""This is the configuration class to store the configuration of an [`AIMv2Model`].
|
||||
Instantiating a configuration with the defaults will yield a similar configuration
|
||||
to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224).
|
||||
|
||||
Args:
|
||||
hidden_size: Dimension of the hidden representations.
|
||||
intermediate_size: Dimension of the SwiGLU representations.
|
||||
num_hidden_layers: Number of hidden layers in the Transformer.
|
||||
num_attention_heads: Number of attention heads for each attention layer
|
||||
in the Transformer.
|
||||
num_channels: Number of input channels.
|
||||
image_size: Image size.
|
||||
patch_size: Patch size.
|
||||
rms_norm_eps: Epsilon value used for the RMS normalization layer.
|
||||
attention_dropout: Dropout ratio for attention probabilities.
|
||||
projection_dropout: Dropout ratio for the projection layer after the attention.
|
||||
qkv_bias: Whether to add a bias to the queries, keys and values.
|
||||
use_bias: Whether to add a bias in the feed-forward and projection layers.
|
||||
kwargs: Keyword arguments for the [`PreTrainedConfig`].
|
||||
|
||||
"""
|
||||
|
||||
model_type: str = "aimv2"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 1024,
|
||||
intermediate_size: int = 2816,
|
||||
num_hidden_layers: int = 24,
|
||||
num_attention_heads: int = 8,
|
||||
num_channels: int = 3,
|
||||
image_size: int = 224,
|
||||
patch_size: int = 14,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
attention_dropout: float = 0.0,
|
||||
projection_dropout: float = 0.0,
|
||||
qkv_bias: bool = False,
|
||||
use_bias: bool = False,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_channels = num_channels
|
||||
self.patch_size = patch_size
|
||||
self.image_size = image_size
|
||||
self.attention_dropout = attention_dropout
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
|
||||
self.projection_dropout = projection_dropout
|
||||
self.qkv_bias = qkv_bias
|
||||
self.use_bias = use_bias
|
||||
from transformers import Aimv2VisionConfig, AutoConfig, PreTrainedConfig
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -96,14 +37,13 @@ class BaseVisualTokenizerConfig(PreTrainedConfig):
|
||||
f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type"
|
||||
)
|
||||
if not isinstance(backbone_config, PreTrainedConfig):
|
||||
model_type = backbone_config["model_type"]
|
||||
if model_type != "aimv2":
|
||||
backbone_config.pop("model_type")
|
||||
model_type = backbone_config.pop("model_type")
|
||||
if model_type == "aimv2":
|
||||
backbone_config = Aimv2VisionConfig(**backbone_config)
|
||||
else:
|
||||
backbone_config = AutoConfig.for_model(
|
||||
model_type, **backbone_config
|
||||
)
|
||||
else:
|
||||
backbone_config = AIMv2Config(**backbone_config)
|
||||
self.backbone_config = backbone_config
|
||||
self.hidden_stride = hidden_stride
|
||||
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Qwen3.5 model configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class Qwen3_5TextConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_5_text"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_proj": "colwise",
|
||||
"layers.*.self_attn.k_proj": "colwise",
|
||||
"layers.*.self_attn.v_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
base_config_key = "text_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=248320,
|
||||
hidden_size=4096,
|
||||
intermediate_size=12288,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=4,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=32768,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
head_dim=256,
|
||||
linear_conv_kernel_dim=4,
|
||||
linear_key_head_dim=128,
|
||||
linear_value_head_dim=128,
|
||||
linear_num_key_heads=16,
|
||||
linear_num_value_heads=32,
|
||||
layer_types=None,
|
||||
pad_token_id=None,
|
||||
bos_token_id=None,
|
||||
eos_token_id=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.head_dim = head_dim
|
||||
self.rope_parameters = rope_parameters
|
||||
kwargs.setdefault("partial_rotary_factor", 0.25)
|
||||
|
||||
self.layer_types = layer_types
|
||||
if self.layer_types is None:
|
||||
interval_pattern = kwargs.get("full_attention_interval", 4)
|
||||
self.layer_types = [
|
||||
"linear_attention"
|
||||
if bool((i + 1) % interval_pattern)
|
||||
else "full_attention"
|
||||
for i in range(self.num_hidden_layers)
|
||||
]
|
||||
kwargs["ignore_keys_at_rope_validation"] = {
|
||||
"mrope_section",
|
||||
"mrope_interleaved",
|
||||
}
|
||||
self.validate_layer_type()
|
||||
|
||||
# linear attention part
|
||||
self.linear_conv_kernel_dim = linear_conv_kernel_dim
|
||||
self.linear_key_head_dim = linear_key_head_dim
|
||||
self.linear_value_head_dim = linear_value_head_dim
|
||||
self.linear_num_key_heads = linear_num_key_heads
|
||||
self.linear_num_value_heads = linear_num_value_heads
|
||||
super().__init__(**kwargs)
|
||||
# Set these AFTER super().__init__() because transformers v4's
|
||||
# PreTrainedConfig.__init__ has these as explicit params with different
|
||||
# defaults (e.g. tie_word_embeddings=True) that would overwrite our values.
|
||||
self.pad_token_id = pad_token_id
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
|
||||
|
||||
class Qwen3_5VisionConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_5"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
depth=27,
|
||||
hidden_size=1152,
|
||||
hidden_act="gelu_pytorch_tanh",
|
||||
intermediate_size=4304,
|
||||
num_heads=16,
|
||||
in_channels=3,
|
||||
patch_size=16,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
out_hidden_size=3584,
|
||||
num_position_embeddings=2304,
|
||||
initializer_range=0.02,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.depth = depth
|
||||
self.hidden_size = hidden_size
|
||||
self.hidden_act = hidden_act
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_heads = num_heads
|
||||
self.in_channels = in_channels
|
||||
self.patch_size = patch_size
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.num_position_embeddings = num_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
|
||||
|
||||
class Qwen3_5Config(PreTrainedConfig):
|
||||
model_type = "qwen3_5"
|
||||
sub_configs = {
|
||||
"vision_config": Qwen3_5VisionConfig,
|
||||
"text_config": Qwen3_5TextConfig,
|
||||
}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
image_token_id=248056,
|
||||
video_token_id=248057,
|
||||
vision_start_token_id=248053,
|
||||
vision_end_token_id=248054,
|
||||
tie_word_embeddings=False,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = self.sub_configs["vision_config"]()
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = self.sub_configs["text_config"]()
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
super().__init__(**kwargs)
|
||||
# Set after super().__init__() to avoid v4 PreTrainedConfig overwrite
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
|
||||
|
||||
__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig"]
|
||||
@@ -1,205 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Qwen3.5-MoE model configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
|
||||
class Qwen3_5MoeTextConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_5_moe_text"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_proj": "colwise",
|
||||
"layers.*.self_attn.k_proj": "colwise",
|
||||
"layers.*.self_attn.v_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
|
||||
"layers.*.mlp.experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.shared_expert.gate_proj": "colwise",
|
||||
"layers.*.mlp.shared_expert.up_proj": "colwise",
|
||||
"layers.*.mlp.shared_expert.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
base_config_key = "text_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=248320,
|
||||
hidden_size=2048,
|
||||
num_hidden_layers=40,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=2,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=32768,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
head_dim=256,
|
||||
linear_conv_kernel_dim=4,
|
||||
linear_key_head_dim=128,
|
||||
linear_value_head_dim=128,
|
||||
linear_num_key_heads=16,
|
||||
linear_num_value_heads=32,
|
||||
moe_intermediate_size=512,
|
||||
shared_expert_intermediate_size=512,
|
||||
num_experts_per_tok=8,
|
||||
num_experts=256,
|
||||
output_router_logits=False,
|
||||
router_aux_loss_coef=0.001,
|
||||
layer_types=None,
|
||||
pad_token_id=None,
|
||||
bos_token_id=None,
|
||||
eos_token_id=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.head_dim = head_dim
|
||||
self.rope_parameters = rope_parameters
|
||||
kwargs.setdefault("partial_rotary_factor", 0.25)
|
||||
|
||||
self.layer_types = layer_types
|
||||
if self.layer_types is None:
|
||||
interval_pattern = kwargs.get("full_attention_interval", 4)
|
||||
self.layer_types = [
|
||||
"linear_attention"
|
||||
if bool((i + 1) % interval_pattern)
|
||||
else "full_attention"
|
||||
for i in range(self.num_hidden_layers)
|
||||
]
|
||||
kwargs["ignore_keys_at_rope_validation"] = {
|
||||
"mrope_section",
|
||||
"mrope_interleaved",
|
||||
}
|
||||
self.validate_layer_type()
|
||||
|
||||
# linear attention part
|
||||
self.linear_conv_kernel_dim = linear_conv_kernel_dim
|
||||
self.linear_key_head_dim = linear_key_head_dim
|
||||
self.linear_value_head_dim = linear_value_head_dim
|
||||
self.linear_num_key_heads = linear_num_key_heads
|
||||
self.linear_num_value_heads = linear_num_value_heads
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.shared_expert_intermediate_size = shared_expert_intermediate_size
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_experts = num_experts
|
||||
self.output_router_logits = output_router_logits
|
||||
self.router_aux_loss_coef = router_aux_loss_coef
|
||||
super().__init__(**kwargs)
|
||||
# Set these AFTER super().__init__() because transformers v4's
|
||||
# PreTrainedConfig.__init__ has these as explicit params with different
|
||||
# defaults (e.g. tie_word_embeddings=True) that would overwrite our values.
|
||||
self.pad_token_id = pad_token_id
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
|
||||
|
||||
class Qwen3_5MoeVisionConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_5_moe"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
depth=27,
|
||||
hidden_size=1152,
|
||||
hidden_act="gelu_pytorch_tanh",
|
||||
intermediate_size=4304,
|
||||
num_heads=16,
|
||||
in_channels=3,
|
||||
patch_size=16,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
out_hidden_size=3584,
|
||||
num_position_embeddings=2304,
|
||||
initializer_range=0.02,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.depth = depth
|
||||
self.hidden_size = hidden_size
|
||||
self.hidden_act = hidden_act
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_heads = num_heads
|
||||
self.in_channels = in_channels
|
||||
self.patch_size = patch_size
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.num_position_embeddings = num_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
|
||||
|
||||
class Qwen3_5MoeConfig(PreTrainedConfig):
|
||||
model_type = "qwen3_5_moe"
|
||||
sub_configs = {
|
||||
"vision_config": Qwen3_5MoeVisionConfig,
|
||||
"text_config": Qwen3_5MoeTextConfig,
|
||||
}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
image_token_id=248056,
|
||||
video_token_id=248057,
|
||||
vision_start_token_id=248053,
|
||||
vision_end_token_id=248054,
|
||||
tie_word_embeddings=False,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = self.sub_configs["vision_config"]()
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = self.sub_configs["text_config"]()
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
super().__init__(**kwargs)
|
||||
# Set after super().__init__() to avoid v4 PreTrainedConfig overwrite
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
|
||||
|
||||
__all__ = ["Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig"]
|
||||
@@ -1,277 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Qwen3-Next model configuration."""
|
||||
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class Qwen3NextConfig(PreTrainedConfig):
|
||||
r"""This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a
|
||||
Qwen3-Next model according to the specified arguments, defining the model architecture.
|
||||
Instantiating a configuration with the defaults will yield a similar configuration to that of
|
||||
Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct).
|
||||
|
||||
Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PreTrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 151936):
|
||||
Vocabulary size of the model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids`.
|
||||
hidden_size (`int`, *optional*, defaults to 2048):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 5632):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 48):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 16):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 2):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
||||
by meanpooling all the original heads within that group. For more details checkout [this
|
||||
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
|
||||
hidden_act (`str`, *optional*, defaults to `"silu"`):
|
||||
The non-linear activation function in the decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 32768):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the rms normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied.
|
||||
rope_parameters (`dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
||||
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
||||
accordingly.
|
||||
Expected contents:
|
||||
`rope_theta` (`float`): The base period of the RoPE embeddings.
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
||||
'llama3'], with 'default' being the original RoPE implementation.
|
||||
`factor` (`float`, *optional*):
|
||||
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
|
||||
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
|
||||
original maximum pre-trained length.
|
||||
`original_max_position_embeddings` (`int`, *optional*):
|
||||
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
|
||||
pretraining.
|
||||
`attention_factor` (`float`, *optional*):
|
||||
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
|
||||
computation. If unspecified, it defaults to value recommended by the implementation, using the
|
||||
`factor` field to infer the suggested value.
|
||||
`beta_fast` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
|
||||
ramp function. If unspecified, it defaults to 32.
|
||||
`beta_slow` (`float`, *optional*):
|
||||
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
|
||||
ramp function. If unspecified, it defaults to 1.
|
||||
`short_factor` (`List[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
||||
size divided by the number of attention heads divided by 2
|
||||
`long_factor` (`List[float]`, *optional*):
|
||||
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
|
||||
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
||||
size divided by the number of attention heads divided by 2
|
||||
`low_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
|
||||
`high_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
|
||||
`partial_rotary_factor` (`float`, *optional*, defaults to 0.25):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output projection layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
head_dim (`int`, *optional*, defaults to 256):
|
||||
Projection weights dimension in multi-head attention.
|
||||
linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
|
||||
Kernel size of the convolution used in linear attention layers.
|
||||
linear_key_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each key head in linear attention.
|
||||
linear_value_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each value head in linear attention.
|
||||
linear_num_key_heads (`int`, *optional*, defaults to 16):
|
||||
Number of key heads used in linear attention layers.
|
||||
linear_num_value_heads (`int`, *optional*, defaults to 32):
|
||||
Number of value heads used in linear attention layers.
|
||||
decoder_sparse_step (`int`, *optional*, defaults to 1):
|
||||
The frequency of the MoE layer.
|
||||
moe_intermediate_size (`int`, *optional*, defaults to 512):
|
||||
Intermediate size of the routed expert.
|
||||
shared_expert_intermediate_size (`int`, *optional*, defaults to 512):
|
||||
Intermediate size of the shared expert.
|
||||
num_experts_per_tok (`int`, *optional*, defaults to 10):
|
||||
Number of selected experts.
|
||||
num_experts (`int`, *optional*, defaults to 512):
|
||||
Number of routed experts.
|
||||
norm_topk_prob (`bool`, *optional*, defaults to `True`):
|
||||
Whether to normalize the topk probabilities.
|
||||
output_router_logits (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not the router logits should be returned by the model. Enabling this will also
|
||||
allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
|
||||
router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
|
||||
The aux loss factor for the total loss.
|
||||
mlp_only_layers (`list[int]`, *optional*, defaults to `[]`):
|
||||
Indicate which layers use Qwen3NextMLP rather than Qwen3NextSparseMoeBlock
|
||||
The list contains layer index, from 0 to num_layers-1 if we have num_layers layers
|
||||
If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity.
|
||||
layer_types (`list[str]`, *optional*):
|
||||
Types of each layer (attention or linear).
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen3NextModel, Qwen3NextConfig
|
||||
|
||||
>>> # Initializing a Qwen3Next style configuration
|
||||
>>> configuration = Qwen3NextConfig()
|
||||
|
||||
>>> # Initializing a model from the Qwen3-Next-80B-A3B style configuration
|
||||
>>> model = Qwen3NextModel(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
model_type = "qwen3_next"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_proj": "colwise",
|
||||
"layers.*.self_attn.k_proj": "colwise",
|
||||
"layers.*.self_attn.v_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.mlp.experts.*.gate_proj": "colwise",
|
||||
"layers.*.mlp.experts.*.up_proj": "colwise",
|
||||
"layers.*.mlp.experts.*.down_proj": "rowwise",
|
||||
"layers.*.mlp.shared_experts.gate_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.up_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=151936,
|
||||
hidden_size=2048,
|
||||
intermediate_size=5632,
|
||||
num_hidden_layers=48,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=2,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=32768,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
head_dim=256,
|
||||
linear_conv_kernel_dim=4,
|
||||
linear_key_head_dim=128,
|
||||
linear_value_head_dim=128,
|
||||
linear_num_key_heads=16,
|
||||
linear_num_value_heads=32,
|
||||
decoder_sparse_step=1,
|
||||
moe_intermediate_size=512,
|
||||
shared_expert_intermediate_size=512,
|
||||
num_experts_per_tok=10,
|
||||
num_experts=512,
|
||||
norm_topk_prob=True,
|
||||
output_router_logits=False,
|
||||
router_aux_loss_coef=0.001,
|
||||
mlp_only_layers=None,
|
||||
layer_types=None,
|
||||
**kwargs,
|
||||
):
|
||||
if mlp_only_layers is None:
|
||||
mlp_only_layers = []
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
partial_rotary_factor = kwargs.pop("partial_rotary_factor", 0.25)
|
||||
if "partial_rotary_factor" not in rope_parameters:
|
||||
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
|
||||
self.rope_parameters = rope_parameters
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.head_dim = head_dim
|
||||
|
||||
self.layer_types = layer_types
|
||||
if self.layer_types is None:
|
||||
self.layer_types = [
|
||||
"linear_attention" if bool((i + 1) % 4) else "full_attention"
|
||||
for i in range(self.num_hidden_layers)
|
||||
]
|
||||
self.validate_layer_type()
|
||||
|
||||
# linear attention part
|
||||
self.linear_conv_kernel_dim = linear_conv_kernel_dim
|
||||
self.linear_key_head_dim = linear_key_head_dim
|
||||
self.linear_value_head_dim = linear_value_head_dim
|
||||
self.linear_num_key_heads = linear_num_key_heads
|
||||
self.linear_num_value_heads = linear_num_value_heads
|
||||
|
||||
# MoE arguments
|
||||
self.decoder_sparse_step = decoder_sparse_step
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.shared_expert_intermediate_size = shared_expert_intermediate_size
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_experts = num_experts
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
self.output_router_logits = output_router_logits
|
||||
self.router_aux_loss_coef = router_aux_loss_coef
|
||||
self.mlp_only_layers = mlp_only_layers
|
||||
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
||||
|
||||
|
||||
__all__ = ["Qwen3NextConfig"]
|
||||
@@ -17,7 +17,6 @@ from vllm.transformers_utils.config import (
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.utils.torch_utils import common_broadcastable_dtype
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -41,7 +40,7 @@ class ModelArchConfigConvertorBase:
|
||||
|
||||
This is the only place that decides whether a checkpoint is heterogeneous
|
||||
and how many layers it has, so a convertor can declare variation that
|
||||
Transformers does not express (see `Gemma4ModelArchConfigConvertor`).
|
||||
Transformers does not express.
|
||||
|
||||
Each pair is converted independently and the results are diffed, so a
|
||||
convertor that overrides this must not also collapse a varying field in
|
||||
@@ -745,27 +744,6 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
== "vision"
|
||||
)
|
||||
|
||||
def get_per_layer_hf_configs(
|
||||
self,
|
||||
) -> list[tuple[PreTrainedConfig, PreTrainedConfig]] | None:
|
||||
# Gemma4 uses a larger head dimension, and sometimes more KV heads, on its
|
||||
# full attention layers than on its sliding ones. Transformers >= 5.15.0
|
||||
# says so in the config; before that the values are flat attributes picked
|
||||
# apart by `layer_types`, so build the per-layer configs here instead. Both
|
||||
# then reach the base convertor as ordinary homogeneous configs.
|
||||
if getattr(self.hf_text_config, "is_heterogeneous", False):
|
||||
return super().get_per_layer_hf_configs()
|
||||
text_config = self.hf_text_config
|
||||
if not getattr(text_config, "layer_types", None):
|
||||
return None
|
||||
# `num_hidden_layers` is what the stack is built from, and it is what
|
||||
# Transformers sizes `per_layer_config` by. `layer_types` can be longer,
|
||||
# e.g. when a test truncates the layer count to build a small model.
|
||||
return [
|
||||
(self.hf_config, gemma4_layer_config(text_config, layer_idx))
|
||||
for layer_idx in range(self.get_num_hidden_layers())
|
||||
]
|
||||
|
||||
|
||||
class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def _language_config(self) -> PreTrainedConfig:
|
||||
|
||||
@@ -17,10 +17,8 @@ __all__ = [
|
||||
"FireRedASR2Processor",
|
||||
"FunASRProcessor",
|
||||
"GLM4VProcessor",
|
||||
"Granite4VisionProcessor",
|
||||
"H2OVLProcessor",
|
||||
"Moondream3Processor",
|
||||
"InternVLProcessor",
|
||||
"IsaacProcessor",
|
||||
"KimiAudioProcessor",
|
||||
"KimiK25Processor",
|
||||
@@ -55,9 +53,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
|
||||
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
|
||||
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
|
||||
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
|
||||
"InternVLProcessor": "vllm.transformers_utils.processors.internvl",
|
||||
"IsaacProcessor": "vllm.transformers_utils.processors.isaac",
|
||||
"KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
|
||||
"KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
from transformers import LlavaNextProcessor
|
||||
from transformers.image_processing_utils import select_best_resolution
|
||||
|
||||
|
||||
class Granite4VisionProcessor(LlavaNextProcessor):
|
||||
"""Processor for Granite 4 Vision.
|
||||
|
||||
Extends LlavaNextProcessor to account for the Window Q-Former
|
||||
downsampling when computing the number of image features.
|
||||
|
||||
This processor is needed because the granite4_vision processor type
|
||||
is not yet in the transformers version pinned by vLLM.
|
||||
"""
|
||||
|
||||
model_type = "granite4_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
tokenizer=None,
|
||||
patch_size=None,
|
||||
vision_feature_select_strategy=None,
|
||||
chat_template=None,
|
||||
image_token="<image>",
|
||||
num_additional_image_tokens=0,
|
||||
downsample_rate=None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
image_processor=image_processor,
|
||||
tokenizer=tokenizer,
|
||||
patch_size=patch_size,
|
||||
vision_feature_select_strategy=vision_feature_select_strategy,
|
||||
chat_template=chat_template,
|
||||
image_token=image_token,
|
||||
num_additional_image_tokens=num_additional_image_tokens,
|
||||
)
|
||||
self.downsample_rate = downsample_rate
|
||||
|
||||
def _get_number_of_features(
|
||||
self,
|
||||
orig_height: int,
|
||||
orig_width: int,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> int:
|
||||
image_grid_pinpoints = self.image_processor.image_grid_pinpoints
|
||||
|
||||
height_best_resolution, width_best_resolution = select_best_resolution(
|
||||
[orig_height, orig_width], image_grid_pinpoints
|
||||
)
|
||||
scale_height = height_best_resolution // height
|
||||
scale_width = width_best_resolution // width
|
||||
|
||||
patches_height = height // self.patch_size
|
||||
patches_width = width // self.patch_size
|
||||
if self.downsample_rate is not None:
|
||||
ds_rate = Fraction(self.downsample_rate)
|
||||
patches_height = int(patches_height * ds_rate)
|
||||
patches_width = int(patches_width * ds_rate)
|
||||
|
||||
unpadded_features, newline_features = self._get_unpadded_features(
|
||||
orig_height,
|
||||
orig_width,
|
||||
patches_height,
|
||||
patches_width,
|
||||
scale_height,
|
||||
scale_width,
|
||||
)
|
||||
base_features = (
|
||||
patches_height * patches_width + self.num_additional_image_tokens
|
||||
)
|
||||
return unpadded_features + newline_features + base_features
|
||||
Reference in New Issue
Block a user