Access partial_rotary_factor from rope_parameters (#29966)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -25,6 +25,7 @@ from transformers.models.auto.tokenization_auto import get_tokenizer_config
|
||||
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
|
||||
|
||||
@@ -304,7 +305,8 @@ def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> No
|
||||
|
||||
def patch_rope_parameters(config: PretrainedConfig) -> None:
|
||||
"""Provide backwards compatibility for RoPE."""
|
||||
rope_theta = getattr(config, "rope_theta", None)
|
||||
rope_theta_names = ("rope_theta", "rotary_emb_base")
|
||||
rope_theta = getattr_iter(config, rope_theta_names, None)
|
||||
if Version(version("transformers")) < Version("5.0.0.dev0"):
|
||||
# Transformers v4 installed, legacy config fields may be present
|
||||
if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
|
||||
@@ -313,6 +315,12 @@ def patch_rope_parameters(config: PretrainedConfig) -> None:
|
||||
if not hasattr(config, "rope_parameters"):
|
||||
config.rope_parameters = {"rope_type": "default"}
|
||||
config.rope_parameters["rope_theta"] = rope_theta
|
||||
partial_rotary_factor_names = ("partial_rotary_factor", "rotary_pct")
|
||||
partial_rotary_factor = getattr_iter(config, partial_rotary_factor_names, None)
|
||||
if partial_rotary_factor is not None:
|
||||
if not hasattr(config, "rope_parameters"):
|
||||
config.rope_parameters = {"rope_type": "default"}
|
||||
config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
|
||||
elif rope_theta is not None or hasattr(config, "rope_parameters"):
|
||||
# Transformers v5 installed
|
||||
config.standardize_rope_params()
|
||||
|
||||
@@ -89,9 +89,14 @@ class NemotronConfig(PretrainedConfig):
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether to tie weight embeddings
|
||||
rope_parameters (`dict`, *optional*):
|
||||
The parameters of the RoPE embeddings.
|
||||
partial_rotary_factor (`float`, *optional*, defaults to 0.5):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
The parameters of the RoPE embeddings. Expected contents:
|
||||
`rope_theta` (`float`): The base period of the RoPE embeddings.
|
||||
`rope_type` (`str`):
|
||||
The sub-variant of RoPE to use. Can be one of ['default', 'linear',
|
||||
'dynamic', 'yarn', 'longrope', 'llama3'], with 'default' being the
|
||||
original RoPE implementation.
|
||||
`partial_rotary_factor` (`float`, *optional*, defaults to 0.5):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output
|
||||
projection layers during self-attention.
|
||||
@@ -133,7 +138,6 @@ class NemotronConfig(PretrainedConfig):
|
||||
eos_token_id=3,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
partial_rotary_factor=0.5,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
mlp_bias=False,
|
||||
@@ -165,14 +169,16 @@ class NemotronConfig(PretrainedConfig):
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
# for backward compatibility
|
||||
partial_rotary_factor = (
|
||||
kwargs.get("rope_percent")
|
||||
or kwargs.get("rope_percentage")
|
||||
or partial_rotary_factor
|
||||
or kwargs.get("partial_rotary_factor")
|
||||
or 0.5
|
||||
)
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
if "partial_rotary_factor" not in rope_parameters:
|
||||
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
|
||||
self.rope_parameters = rope_parameters
|
||||
self._rope_parameters_validation()
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
@@ -103,8 +103,8 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
|
||||
`high_freq_factor` (`float`, *optional*):
|
||||
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
|
||||
partial_rotary_factor (`float`, *optional*, defaults to 0.25):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
`partial_rotary_factor` (`float`, *optional*, defaults to 0.25):
|
||||
Percentage of the query and keys which will have rotary embedding.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use a bias in the query, key, value and output projection layers during self-attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
@@ -198,7 +198,6 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
partial_rotary_factor=0.25,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
head_dim=256,
|
||||
@@ -239,6 +238,9 @@ class Qwen3NextConfig(PretrainedConfig):
|
||||
rope_theta = kwargs.pop("rope_theta", 10000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
partial_rotary_factor = kwargs.pop("partial_rotary_factor", 0.25)
|
||||
if "partial_rotary_factor" not in rope_parameters:
|
||||
rope_parameters["partial_rotary_factor"] = partial_rotary_factor
|
||||
self.rope_parameters = rope_parameters
|
||||
self.partial_rotary_factor = partial_rotary_factor
|
||||
self.attention_bias = attention_bias
|
||||
|
||||
Reference in New Issue
Block a user