Update rope_scaling to rope_parameters in preparation for Transformers v5 (#28542)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -7,8 +7,9 @@ import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from functools import cache, partial
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeVar
|
||||
from typing import Any, Literal, TypeAlias, TypeVar
|
||||
|
||||
import huggingface_hub
|
||||
from huggingface_hub import (
|
||||
@@ -24,7 +25,9 @@ from huggingface_hub.utils import (
|
||||
RepositoryNotFoundError,
|
||||
RevisionNotFoundError,
|
||||
)
|
||||
from packaging.version import Version
|
||||
from transformers import DeepseekV3Config, GenerationConfig, PretrainedConfig
|
||||
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
|
||||
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
||||
from transformers.models.auto.modeling_auto import (
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
|
||||
@@ -390,21 +393,61 @@ def file_or_path_exists(
|
||||
)
|
||||
|
||||
|
||||
def patch_rope_scaling(config: PretrainedConfig) -> None:
|
||||
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
|
||||
"""Some models may have no rope_theta in their config but still use RoPE.
|
||||
This function sets a default rope_theta if it's missing."""
|
||||
if getattr(config, "rope_parameters", None) is None:
|
||||
config.rope_parameters = {"rope_type": "default"}
|
||||
if "rope_theta" not in config.rope_parameters:
|
||||
config.rope_parameters["rope_theta"] = default_theta
|
||||
|
||||
|
||||
def patch_rope_parameters(config: PretrainedConfig) -> None:
|
||||
"""Provide backwards compatibility for RoPE."""
|
||||
text_config = getattr(config, "text_config", None)
|
||||
if text_config is not None:
|
||||
patch_rope_scaling(text_config)
|
||||
# Retrieve rope_parameters differently based on Transformers version
|
||||
if Version(version("transformers")) >= Version("5.0.0.dev0"):
|
||||
from transformers.modeling_rope_utils import RopeParameters
|
||||
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if rope_scaling is not None:
|
||||
patch_rope_scaling_dict(rope_scaling)
|
||||
rope_parameters: RopeParameters | dict[str, RopeParameters] | None = getattr(
|
||||
config, "rope_parameters", None
|
||||
)
|
||||
elif hasattr(config, "rope_parameters"):
|
||||
# We are in Transformers v4 and rope_parameters
|
||||
# has already been patched for this config
|
||||
return
|
||||
else:
|
||||
# Convert Transformers v4 rope_theta and rope_scaling into rope_parameters
|
||||
rope_theta: float | None = getattr(config, "rope_theta", None)
|
||||
rope_scaling: dict | None = getattr(config, "rope_scaling", None)
|
||||
rope_parameters = rope_scaling
|
||||
# Move rope_theta into rope_parameters
|
||||
if rope_theta is not None:
|
||||
rope_parameters = rope_parameters or {"rope_type": "default"}
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
# Add original_max_position_embeddings if present
|
||||
if rope_parameters and (
|
||||
ompe := getattr(config, "original_max_position_embeddings", None)
|
||||
):
|
||||
rope_parameters["original_max_position_embeddings"] = ompe
|
||||
# Write back to config
|
||||
config.rope_parameters = rope_parameters
|
||||
|
||||
# No RoPE parameters to patch
|
||||
if rope_parameters is None:
|
||||
return
|
||||
|
||||
# Handle nested rope_parameters in interleaved sliding attention models
|
||||
if set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
|
||||
for rope_parameters_layer_type in rope_parameters.values():
|
||||
patch_rope_parameters_dict(rope_parameters_layer_type)
|
||||
else:
|
||||
patch_rope_parameters_dict(rope_parameters)
|
||||
|
||||
|
||||
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
|
||||
if "rope_type" in rope_scaling and "type" in rope_scaling:
|
||||
rope_type = rope_scaling["rope_type"]
|
||||
rope_type_legacy = rope_scaling["type"]
|
||||
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
|
||||
if "rope_type" in rope_parameters and "type" in rope_parameters:
|
||||
rope_type = rope_parameters["rope_type"]
|
||||
rope_type_legacy = rope_parameters["type"]
|
||||
if rope_type != rope_type_legacy:
|
||||
raise ValueError(
|
||||
f"Found conflicts between 'rope_type={rope_type}' (modern "
|
||||
@@ -412,28 +455,28 @@ def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
|
||||
"You should only specify one of them."
|
||||
)
|
||||
|
||||
if "rope_type" not in rope_scaling and "type" in rope_scaling:
|
||||
rope_scaling["rope_type"] = rope_scaling["type"]
|
||||
if "rope_type" not in rope_parameters and "type" in rope_parameters:
|
||||
rope_parameters["rope_type"] = rope_parameters["type"]
|
||||
logger.info("Replacing legacy 'type' key with 'rope_type'")
|
||||
|
||||
if "rope_type" not in rope_scaling:
|
||||
raise ValueError("rope_scaling should have a 'rope_type' key")
|
||||
if "rope_type" not in rope_parameters:
|
||||
raise ValueError("rope_parameters should have a 'rope_type' key")
|
||||
|
||||
if rope_scaling["rope_type"] == "su":
|
||||
rope_scaling["rope_type"] = "longrope"
|
||||
if rope_parameters["rope_type"] == "su":
|
||||
rope_parameters["rope_type"] = "longrope"
|
||||
logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
|
||||
elif rope_scaling["rope_type"] == "mrope":
|
||||
assert "mrope_section" in rope_scaling
|
||||
rope_scaling["rope_type"] = "default"
|
||||
elif rope_parameters["rope_type"] == "mrope":
|
||||
assert "mrope_section" in rope_parameters
|
||||
rope_parameters["rope_type"] = "default"
|
||||
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
|
||||
|
||||
|
||||
def _uses_mrope(config: PretrainedConfig) -> bool:
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if rope_scaling is None:
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
if rope_parameters is None:
|
||||
return False
|
||||
|
||||
return "mrope_section" in rope_scaling
|
||||
return "mrope_section" in rope_parameters
|
||||
|
||||
|
||||
def uses_mrope(config: PretrainedConfig) -> bool:
|
||||
@@ -690,7 +733,14 @@ def get_config(
|
||||
logger.debug("Overriding HF config with %s", hf_overrides_fn)
|
||||
config = hf_overrides_fn(config)
|
||||
|
||||
patch_rope_scaling(config)
|
||||
# Exhaustively patch RoPE parameters everywhere they might be
|
||||
patch_rope_parameters(config)
|
||||
patch_rope_parameters(config.get_text_config())
|
||||
SubConfigs: TypeAlias = dict[str, PretrainedConfig]
|
||||
sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
|
||||
if sub_configs:
|
||||
for sub_config in sub_configs:
|
||||
patch_rope_parameters(getattr(config, sub_config))
|
||||
|
||||
if trust_remote_code:
|
||||
maybe_register_config_serialize_by_value()
|
||||
|
||||
@@ -24,7 +24,7 @@ class AfmoeConfig(PretrainedConfig):
|
||||
rms_norm_eps: float = 1e-5,
|
||||
use_cache: bool = True,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_theta: float = 10000.0,
|
||||
rope_parameters: dict | None = None,
|
||||
rope_scaling: dict | None = None,
|
||||
num_experts: int = 64,
|
||||
num_experts_per_tok: int = 6,
|
||||
@@ -56,7 +56,10 @@ class AfmoeConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.rope_scaling = rope_scaling
|
||||
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
|
||||
@@ -85,8 +85,15 @@ class ArcticConfig(PretrainedConfig):
|
||||
The id of the "end-of-sequence" token.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied.
|
||||
rope_theta (`float`, *optional*, defaults to 1000000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_parameters (`dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
||||
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
||||
accordingly.
|
||||
Expected contents:
|
||||
`rope_theta` (`float`): The base period of the RoPE embeddings.
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
||||
'llama3'], with 'default' being the original RoPE implementation.
|
||||
sliding_window (`int`, *optional*):
|
||||
Sliding window attention window size. If not specified, will default to `4096`.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
@@ -132,7 +139,7 @@ class ArcticConfig(PretrainedConfig):
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=1e6,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
sliding_window=None,
|
||||
attention_dropout=0.0,
|
||||
num_experts_per_tok=1,
|
||||
@@ -165,7 +172,10 @@ class ArcticConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
rope_theta = kwargs.pop("rope_theta", 1e6)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
@@ -25,8 +26,7 @@ class FlexOlmoConfig(PretrainedConfig):
|
||||
bos_token_id=None,
|
||||
eos_token_id=100257,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=500000.0,
|
||||
rope_scaling=None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
num_experts_per_tok=5,
|
||||
@@ -62,8 +62,13 @@ class FlexOlmoConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 500000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
@@ -73,5 +78,5 @@ class FlexOlmoConfig(PretrainedConfig):
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
# Validate the correctness of rotary position embeddings parameters
|
||||
# BC: if there is a 'type' field, move it to 'rope_type'.
|
||||
if self.rope_scaling is not None and "type" in self.rope_scaling:
|
||||
self.rope_scaling["rope_type"] = self.rope_scaling["type"]
|
||||
if self.rope_parameters is not None and "type" in self.rope_parameters:
|
||||
self.rope_parameters["rope_type"] = self.rope_parameters["type"]
|
||||
|
||||
@@ -29,8 +29,7 @@ class KimiLinearConfig(PretrainedConfig):
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
rope_theta=10000.0,
|
||||
rope_scaling=None,
|
||||
rope_parameters=None,
|
||||
tie_word_embeddings=False,
|
||||
moe_intermediate_size: int | None = None,
|
||||
moe_renormalize: bool = True,
|
||||
@@ -73,8 +72,13 @@ class KimiLinearConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
@@ -35,8 +36,8 @@ class Lfm2MoeConfig(PretrainedConfig):
|
||||
End of stream token id.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `True`):
|
||||
Whether to tie weight embeddings
|
||||
rope_theta (`float`, *optional*, defaults to 1000000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 128000):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
@@ -100,7 +101,7 @@ class Lfm2MoeConfig(PretrainedConfig):
|
||||
bos_token_id: int = 1,
|
||||
eos_token_id: int = 2,
|
||||
tie_word_embeddings: bool = True,
|
||||
rope_theta: float = 1000000.0,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
max_position_embeddings: int = 128_000,
|
||||
use_cache: bool = True,
|
||||
norm_eps: float = 0.00001,
|
||||
@@ -121,7 +122,10 @@ class Lfm2MoeConfig(PretrainedConfig):
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.rope_theta = rope_theta
|
||||
rope_theta = kwargs.pop("rope_theta", 1000000.0)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.use_cache = use_cache
|
||||
self.norm_eps = norm_eps
|
||||
|
||||
@@ -98,6 +98,6 @@ class MiDashengLMConfig(PretrainedConfig):
|
||||
if text_config
|
||||
else Qwen2_5OmniTextConfig()
|
||||
)
|
||||
self.text_config.rope_scaling = None # uses_mrope is false
|
||||
self.text_config.rope_parameters = None # uses_mrope is false
|
||||
self.audio_token_id = audio_token_id
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -86,13 +86,13 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
|
||||
"apply_scale": "apply_yarn_scaling",
|
||||
}
|
||||
yarn_config = config.get("yarn") or {}
|
||||
config["rope_scaling"] = {
|
||||
config["rope_parameters"] = {
|
||||
"rope_type": "yarn",
|
||||
"mscale_all_dim": 1,
|
||||
}
|
||||
for old_name, new_name in yarn_config_map.items():
|
||||
if old_name in yarn_config:
|
||||
config["rope_scaling"][new_name] = yarn_config.pop(old_name)
|
||||
config["rope_parameters"][new_name] = yarn_config.pop(old_name)
|
||||
|
||||
assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
|
||||
|
||||
|
||||
@@ -88,8 +88,8 @@ class NemotronConfig(PretrainedConfig):
|
||||
End of stream token id.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie weight embeddings
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings.
|
||||
partial_rotary_factor (`float`, *optional*, defaults to 0.5):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
@@ -132,8 +132,7 @@ class NemotronConfig(PretrainedConfig):
|
||||
bos_token_id=2,
|
||||
eos_token_id=3,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=10000.0,
|
||||
rope_scaling=None,
|
||||
rope_parameters=None,
|
||||
partial_rotary_factor=0.5,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
@@ -160,8 +159,13 @@ class NemotronConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.norm_eps = norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
# for backward compatibility
|
||||
partial_rotary_factor = (
|
||||
kwargs.get("rope_percent")
|
||||
@@ -169,7 +173,7 @@ class NemotronConfig(PretrainedConfig):
|
||||
or partial_rotary_factor
|
||||
)
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self._rope_scaling_validation()
|
||||
self._rope_parameters_validation()
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.mlp_bias = mlp_bias
|
||||
@@ -182,31 +186,29 @@ class NemotronConfig(PretrainedConfig):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _rope_scaling_validation(self):
|
||||
def _rope_parameters_validation(self):
|
||||
"""
|
||||
Validate the `rope_scaling` configuration.
|
||||
Validate the `rope_parameters` configuration.
|
||||
"""
|
||||
if self.rope_scaling is None:
|
||||
if self.rope_parameters is None:
|
||||
return
|
||||
|
||||
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
|
||||
rope_type: str | None = self.rope_parameters.get("rope_type", None)
|
||||
factor: float | None = self.rope_parameters.get("factor", None)
|
||||
|
||||
if rope_type not in {"default", "linear", "dynamic"}:
|
||||
raise ValueError(
|
||||
"`rope_scaling` must be a dictionary with two fields, "
|
||||
f"`type` and `factor`, got {self.rope_scaling}"
|
||||
)
|
||||
rope_scaling_type = self.rope_scaling.get("type", None)
|
||||
rope_scaling_factor = self.rope_scaling.get("factor", None)
|
||||
if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s type field must be one of ['linear', "
|
||||
f"'dynamic'], got {rope_scaling_type}"
|
||||
)
|
||||
if (
|
||||
rope_scaling_factor is None
|
||||
or not isinstance(rope_scaling_factor, float)
|
||||
or rope_scaling_factor <= 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s factor field must be a float > 1, got "
|
||||
f"{rope_scaling_factor}"
|
||||
"`rope_type` must be one of ['default', 'linear', 'dynamic'], "
|
||||
f"got {rope_type}"
|
||||
)
|
||||
if rope_type != "default":
|
||||
if factor is None:
|
||||
raise ValueError(
|
||||
"If `rope_type` is not 'default', `rope_parameters` "
|
||||
"must include a `factor` field. Got `None`."
|
||||
)
|
||||
if not isinstance(factor, float) or factor <= 1.0:
|
||||
raise ValueError(
|
||||
"`rope_parameters`'s factor field must be a float > 1, got "
|
||||
f"{factor}"
|
||||
)
|
||||
|
||||
@@ -24,8 +24,7 @@ class Olmo3Config(PretrainedConfig):
|
||||
bos_token_id=None,
|
||||
eos_token_id=50279,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=10000.0,
|
||||
rope_scaling=None,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
rms_norm_eps=1e-5,
|
||||
@@ -63,8 +62,13 @@ class Olmo3Config(PretrainedConfig):
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
|
||||
@@ -66,13 +66,12 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
relevant if `config.is_decoder=True`.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied.
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rope_scaling (`Dict`, *optional*):
|
||||
rope_parameters (`dict`, *optional*):
|
||||
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
||||
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
||||
accordingly.
|
||||
Expected contents:
|
||||
`rope_theta` (`float`): The base period of the RoPE embeddings.
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
||||
'llama3'], with 'default' being the original RoPE implementation.
|
||||
@@ -199,8 +198,7 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=10000.0,
|
||||
rope_scaling=None,
|
||||
rope_parameters=None,
|
||||
partial_rotary_factor=0.25,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
@@ -236,8 +234,13 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
@@ -52,8 +52,7 @@ class Step3TextConfig(PretrainedConfig):
|
||||
moe_intermediate_size: int = 5120,
|
||||
moe_num_experts: int = 48,
|
||||
moe_top_k: int = 3,
|
||||
rope_theta: float = 500000,
|
||||
rope_scaling: dict[str, Any] | None = None,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
max_position_embedding: int = 65536,
|
||||
share_expert_dim: int = 5120,
|
||||
share_q_dim: int = 2048,
|
||||
@@ -130,8 +129,13 @@ class Step3TextConfig(PretrainedConfig):
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.moe_num_experts = moe_num_experts
|
||||
self.moe_top_k = moe_top_k
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 500000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.max_position_embedding = max_position_embedding
|
||||
self.share_expert_dim = share_expert_dim
|
||||
self.share_q_dim = share_q_dim
|
||||
|
||||
Reference in New Issue
Block a user