[Bugfix] Align vLLM YaRN with Transformers and stop re-scaling max_model_len (#56446)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a5b714eee4
commit
c191787a68
@@ -1853,6 +1853,54 @@ def test_get_and_verify_max_len_with_nope_layers(
|
|||||||
assert hf_config.rope_parameters["full_attention"] is None
|
assert hf_config.rope_parameters["full_attention"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("rope_type", "factor", "expected_max_len"),
|
||||||
|
[
|
||||||
|
# TeleChat3-36B-Thinking: 32768 already scaled from 8192 by 4
|
||||||
|
("yarn", 4.0, 32768),
|
||||||
|
# sarvam-105b: declares factor 40 but only serves 131072 of it
|
||||||
|
("deepseek_yarn", 40.0, 32768),
|
||||||
|
("deepseek_llama_scaling", 40.0, 32768),
|
||||||
|
# Non-YaRN scaling still multiplies
|
||||||
|
("linear", 4.0, 131072),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_get_and_verify_max_len_yarn_is_already_scaled(
|
||||||
|
rope_type, factor, expected_max_len
|
||||||
|
):
|
||||||
|
"""YaRN variants must not re-apply `factor` to max_position_embeddings.
|
||||||
|
|
||||||
|
Transformers treats max_position_embeddings as the final context length
|
||||||
|
for every YaRN variant, so scaling it again overstates the limit and lets
|
||||||
|
requests past the end of the cos/sin cache.
|
||||||
|
"""
|
||||||
|
from transformers import PretrainedConfig
|
||||||
|
|
||||||
|
from vllm.config.model import _get_and_verify_max_len
|
||||||
|
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||||
|
ModelArchConfigConvertorBase,
|
||||||
|
)
|
||||||
|
|
||||||
|
hf_config = PretrainedConfig(max_position_embeddings=32768)
|
||||||
|
hf_config.rope_parameters = {
|
||||||
|
"rope_type": rope_type,
|
||||||
|
"factor": factor,
|
||||||
|
"original_max_position_embeddings": 8192,
|
||||||
|
}
|
||||||
|
model_arch_config = ModelArchConfigConvertorBase(hf_config, hf_config).convert()
|
||||||
|
|
||||||
|
actual_max_len = _get_and_verify_max_len(
|
||||||
|
hf_config=hf_config,
|
||||||
|
model_arch_config=model_arch_config,
|
||||||
|
tokenizer_config=None,
|
||||||
|
max_model_len=None,
|
||||||
|
disable_sliding_window=False,
|
||||||
|
sliding_window=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert actual_max_len == expected_max_len
|
||||||
|
|
||||||
|
|
||||||
class MockConfig:
|
class MockConfig:
|
||||||
"""Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""
|
"""Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ only get the `eos_token_id` from the tokenizer as defined by
|
|||||||
`BaseRenderer.get_eos_token_id`.
|
`BaseRenderer.get_eos_token_id`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from typing import cast
|
from typing import cast
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
@@ -52,6 +53,31 @@ def test_patch_legacy_rope_type_preserves_nope_layers():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_patch_legacy_rope_type_normalizes_telechat3_yarn():
|
||||||
|
"""TeleChat3's RoPE is YaRN with 0.07 in place of the usual 0.1.
|
||||||
|
|
||||||
|
Encoding that as a precomputed attention_factor keeps the config
|
||||||
|
plain YaRN, which Transformers and every "yarn" guard understand.
|
||||||
|
`mscale` cannot express it: Transformers only applies mscale when
|
||||||
|
mscale_all_dim is also truthy.
|
||||||
|
"""
|
||||||
|
rope_parameters = {
|
||||||
|
"type": "telechat3-yarn",
|
||||||
|
"rope_type": "telechat3-yarn",
|
||||||
|
"factor": 4.0,
|
||||||
|
"original_max_position_embeddings": 8192,
|
||||||
|
}
|
||||||
|
|
||||||
|
patch_legacy_rope_type(rope_parameters)
|
||||||
|
|
||||||
|
assert rope_parameters == {
|
||||||
|
"rope_type": "yarn",
|
||||||
|
"factor": 4.0,
|
||||||
|
"original_max_position_embeddings": 8192,
|
||||||
|
"attention_factor": pytest.approx(0.07 * math.log(4.0) + 1.0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
|
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
|
||||||
layer_types = ["linear_attention", "deepseek_sparse_attention"]
|
layer_types = ["linear_attention", "deepseek_sparse_attention"]
|
||||||
|
|
||||||
|
|||||||
+11
-4
@@ -2465,13 +2465,20 @@ def _get_and_verify_max_len(
|
|||||||
# loading HF config
|
# loading HF config
|
||||||
rope_type = rp["rope_type"]
|
rope_type = rp["rope_type"]
|
||||||
|
|
||||||
if rope_type not in ("su", "longrope", "llama3"):
|
# YaRN variants leave max_position_embeddings already scaled, as
|
||||||
|
# Transformers' _compute_yarn_parameters assumes, so `factor` must
|
||||||
|
# not be applied to it again.
|
||||||
|
if rope_type not in (
|
||||||
|
"su",
|
||||||
|
"longrope",
|
||||||
|
"llama3",
|
||||||
|
"yarn",
|
||||||
|
"deepseek_yarn",
|
||||||
|
"deepseek_llama_scaling",
|
||||||
|
):
|
||||||
# NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
|
# NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
|
||||||
# NOTE: This assumes all layer types have the same scaling factor.
|
# NOTE: This assumes all layer types have the same scaling factor.
|
||||||
scaling_factor = rp.get("factor", scaling_factor)
|
scaling_factor = rp.get("factor", scaling_factor)
|
||||||
|
|
||||||
if rope_type == "yarn":
|
|
||||||
derived_max_model_len = rp["original_max_position_embeddings"]
|
|
||||||
if scaling_factor is None:
|
if scaling_factor is None:
|
||||||
# Fallback the factor to 1.0 if a user assigned `null`
|
# Fallback the factor to 1.0 if a user assigned `null`
|
||||||
logger.warning_once(
|
logger.warning_once(
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ from .mrope import MRotaryEmbedding
|
|||||||
from .mrope_interleaved import MRotaryEmbeddingInterleaved
|
from .mrope_interleaved import MRotaryEmbeddingInterleaved
|
||||||
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
|
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
|
||||||
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
|
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
|
||||||
from .telechat3_scaling_rope import TeleChat3RoPEScaledRotaryEmbedding
|
|
||||||
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
|
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
|
||||||
|
|
||||||
_ROPE_DICT: dict[tuple[Any, ...], RotaryEmbedding] = {}
|
_ROPE_DICT: dict[tuple[Any, ...], RotaryEmbedding] = {}
|
||||||
@@ -246,16 +245,15 @@ def get_rope(
|
|||||||
for k, v in rope_parameters.items()
|
for k, v in rope_parameters.items()
|
||||||
if k
|
if k
|
||||||
in (
|
in (
|
||||||
"extrapolation_factor",
|
|
||||||
"attn_factor",
|
|
||||||
"beta_fast",
|
"beta_fast",
|
||||||
"beta_slow",
|
"beta_slow",
|
||||||
"apply_yarn_scaling",
|
"mscale",
|
||||||
|
"mscale_all_dim",
|
||||||
|
"attention_factor",
|
||||||
"truncate",
|
"truncate",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if "mrope_section" in rope_parameters:
|
if "mrope_section" in rope_parameters:
|
||||||
extra_kwargs.pop("apply_yarn_scaling", None)
|
|
||||||
rotary_emb = MRotaryEmbedding(
|
rotary_emb = MRotaryEmbedding(
|
||||||
head_size,
|
head_size,
|
||||||
rotary_dim,
|
rotary_dim,
|
||||||
@@ -288,8 +286,6 @@ def get_rope(
|
|||||||
for k, v in rope_parameters.items()
|
for k, v in rope_parameters.items()
|
||||||
if k
|
if k
|
||||||
in (
|
in (
|
||||||
"extrapolation_factor",
|
|
||||||
"attn_factor",
|
|
||||||
"beta_fast",
|
"beta_fast",
|
||||||
"beta_slow",
|
"beta_slow",
|
||||||
"mscale",
|
"mscale",
|
||||||
@@ -346,36 +342,6 @@ def get_rope(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError("Pangu mrope lacks necessary parameters.")
|
raise ValueError("Pangu mrope lacks necessary parameters.")
|
||||||
elif scaling_type == "telechat3-yarn":
|
|
||||||
scaling_factor = rope_parameters["factor"]
|
|
||||||
if "original_max_position_embeddings" in rope_parameters:
|
|
||||||
original_max_position = rope_parameters["original_max_position_embeddings"]
|
|
||||||
scaling_factor = max_position / original_max_position
|
|
||||||
else:
|
|
||||||
original_max_position = max_position
|
|
||||||
extra_kwargs = {
|
|
||||||
k: v
|
|
||||||
for k, v in rope_parameters.items()
|
|
||||||
if k
|
|
||||||
in (
|
|
||||||
"extrapolation_factor",
|
|
||||||
"attn_factor",
|
|
||||||
"beta_fast",
|
|
||||||
"beta_slow",
|
|
||||||
"mscale",
|
|
||||||
"mscale_all_dim",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
rotary_emb = TeleChat3RoPEScaledRotaryEmbedding(
|
|
||||||
head_size,
|
|
||||||
rotary_dim,
|
|
||||||
original_max_position,
|
|
||||||
base,
|
|
||||||
is_neox_style,
|
|
||||||
scaling_factor,
|
|
||||||
dtype,
|
|
||||||
**extra_kwargs,
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
|
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
|
||||||
_ROPE_DICT[key] = rotary_emb
|
_ROPE_DICT[key] = rotary_emb
|
||||||
|
|||||||
@@ -70,10 +70,10 @@ def yarn_linear_ramp_mask(
|
|||||||
return ramp_func
|
return ramp_func
|
||||||
|
|
||||||
|
|
||||||
def yarn_get_mscale(scale: float = 1) -> float:
|
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
|
||||||
if scale <= 1:
|
if scale <= 1:
|
||||||
return 1.0
|
return 1.0
|
||||||
return 0.1 * math.log(scale) + 1.0
|
return 0.1 * mscale * math.log(scale) + 1.0
|
||||||
|
|
||||||
|
|
||||||
def _flashinfer_rotary_embedding(
|
def _flashinfer_rotary_embedding(
|
||||||
|
|||||||
@@ -39,8 +39,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
scaling_factor: float,
|
scaling_factor: float,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
*,
|
*,
|
||||||
extrapolation_factor: float = 1,
|
|
||||||
attn_factor: float = 1,
|
|
||||||
beta_fast: int = 32,
|
beta_fast: int = 32,
|
||||||
beta_slow: int = 1,
|
beta_slow: int = 1,
|
||||||
mscale: float = 1,
|
mscale: float = 1,
|
||||||
@@ -48,15 +46,12 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
init_cache: bool = True,
|
init_cache: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.scaling_factor = scaling_factor
|
self.scaling_factor = scaling_factor
|
||||||
self.extrapolation_factor = extrapolation_factor
|
|
||||||
self.attn_factor = attn_factor
|
|
||||||
self.beta_fast = beta_fast
|
self.beta_fast = beta_fast
|
||||||
self.beta_slow = beta_slow
|
self.beta_slow = beta_slow
|
||||||
# Get n-d magnitude scaling corrected for interpolation.
|
# Get n-d magnitude scaling corrected for interpolation.
|
||||||
self.mscale = float(
|
self.mscale = float(
|
||||||
yarn_get_mscale(self.scaling_factor, float(mscale))
|
yarn_get_mscale(self.scaling_factor, float(mscale))
|
||||||
/ yarn_get_mscale(self.scaling_factor, float(mscale_all_dim))
|
/ yarn_get_mscale(self.scaling_factor, float(mscale_all_dim))
|
||||||
* attn_factor
|
|
||||||
)
|
)
|
||||||
self.use_flashinfer = (
|
self.use_flashinfer = (
|
||||||
self.enabled()
|
self.enabled()
|
||||||
@@ -96,10 +91,9 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
self.max_position_embeddings,
|
self.max_position_embeddings,
|
||||||
)
|
)
|
||||||
# Get n-d rotational scaling corrected for extrapolation
|
# Get n-d rotational scaling corrected for extrapolation
|
||||||
inv_freq_mask = (
|
inv_freq_mask = 1 - yarn_linear_ramp_mask(
|
||||||
1
|
low, high, self.rotary_dim // 2, dtype=torch.float
|
||||||
- yarn_linear_ramp_mask(low, high, self.rotary_dim // 2, dtype=torch.float)
|
)
|
||||||
) * self.extrapolation_factor
|
|
||||||
inv_freq = (
|
inv_freq = (
|
||||||
inv_freq_interpolation * (1 - inv_freq_mask)
|
inv_freq_interpolation * (1 - inv_freq_mask)
|
||||||
+ inv_freq_extrapolation * inv_freq_mask
|
+ inv_freq_extrapolation * inv_freq_mask
|
||||||
|
|||||||
@@ -252,21 +252,28 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
# YaRN parameters.
|
# YaRN parameters.
|
||||||
*,
|
*,
|
||||||
scaling_factor: float | None = None,
|
scaling_factor: float | None = None,
|
||||||
extrapolation_factor: float = 1,
|
|
||||||
attn_factor: float = 1,
|
|
||||||
beta_fast: int = 32,
|
beta_fast: int = 32,
|
||||||
beta_slow: int = 1,
|
beta_slow: int = 1,
|
||||||
|
mscale: float | None = None,
|
||||||
|
mscale_all_dim: float | None = None,
|
||||||
|
attention_factor: float | None = None,
|
||||||
truncate: bool = True,
|
truncate: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.scaling_factor = scaling_factor
|
self.scaling_factor = scaling_factor
|
||||||
self.extrapolation_factor = extrapolation_factor
|
|
||||||
self.attn_factor = attn_factor
|
|
||||||
self.beta_fast = beta_fast
|
self.beta_fast = beta_fast
|
||||||
self.beta_slow = beta_slow
|
self.beta_slow = beta_slow
|
||||||
self.truncate = truncate
|
self.truncate = truncate
|
||||||
if self.scaling_factor is not None:
|
if self.scaling_factor is not None:
|
||||||
# Get n-d magnitude scaling corrected for interpolation
|
# Get n-d magnitude scaling corrected for interpolation
|
||||||
self.mscale = float(yarn_get_mscale(self.scaling_factor) * attn_factor)
|
if attention_factor is not None:
|
||||||
|
self.mscale = float(attention_factor)
|
||||||
|
elif mscale and mscale_all_dim:
|
||||||
|
self.mscale = float(
|
||||||
|
yarn_get_mscale(self.scaling_factor, mscale)
|
||||||
|
/ yarn_get_mscale(self.scaling_factor, mscale_all_dim)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.mscale = float(yarn_get_mscale(self.scaling_factor))
|
||||||
else:
|
else:
|
||||||
self.mscale = 1.0
|
self.mscale = 1.0
|
||||||
|
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
import math
|
|
||||||
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from .base import RotaryEmbedding
|
|
||||||
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
|
|
||||||
|
|
||||||
|
|
||||||
class TeleChat3RoPEScaledRotaryEmbedding(YaRNScalingRotaryEmbedding):
|
|
||||||
"""TeleChat3 uses a variant of YaRN method.
|
|
||||||
|
|
||||||
To achieve code reuse as much as possible, we have rewritten the
|
|
||||||
`get_mscale` method in the initialization function
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
head_size: int,
|
|
||||||
rotary_dim: int,
|
|
||||||
max_position_embeddings: int,
|
|
||||||
base: int,
|
|
||||||
is_neox_style: bool,
|
|
||||||
scaling_factor: float,
|
|
||||||
dtype: torch.dtype,
|
|
||||||
*,
|
|
||||||
extrapolation_factor: float = 1,
|
|
||||||
attn_factor: float = 1,
|
|
||||||
beta_fast: int = 32,
|
|
||||||
beta_slow: int = 1,
|
|
||||||
truncate: bool = True,
|
|
||||||
) -> None:
|
|
||||||
self.scaling_factor = scaling_factor
|
|
||||||
self.extrapolation_factor = extrapolation_factor
|
|
||||||
self.attn_factor = attn_factor
|
|
||||||
self.beta_fast = beta_fast
|
|
||||||
self.beta_slow = beta_slow
|
|
||||||
self.truncate = truncate
|
|
||||||
|
|
||||||
def get_mscale(scale, mscale=1):
|
|
||||||
if scale <= 1:
|
|
||||||
return 1.0
|
|
||||||
return 0.07 * mscale * math.log(scale) + 1.0
|
|
||||||
|
|
||||||
self.mscale = float(get_mscale(self.scaling_factor) * attn_factor)
|
|
||||||
# Initialization must be performed after mscale, otherwise mscale is useless
|
|
||||||
RotaryEmbedding.__init__(
|
|
||||||
self,
|
|
||||||
head_size,
|
|
||||||
rotary_dim,
|
|
||||||
max_position_embeddings,
|
|
||||||
base,
|
|
||||||
is_neox_style,
|
|
||||||
dtype,
|
|
||||||
)
|
|
||||||
@@ -23,25 +23,27 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
|
|||||||
scaling_factor: float,
|
scaling_factor: float,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
*,
|
*,
|
||||||
extrapolation_factor: float = 1,
|
|
||||||
attn_factor: float = 1,
|
|
||||||
beta_fast: int = 32,
|
beta_fast: int = 32,
|
||||||
beta_slow: int = 1,
|
beta_slow: int = 1,
|
||||||
apply_yarn_scaling: bool = True,
|
mscale: float | None = None,
|
||||||
|
mscale_all_dim: float | None = None,
|
||||||
|
attention_factor: float | None = None,
|
||||||
truncate: bool = True,
|
truncate: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.scaling_factor = scaling_factor
|
self.scaling_factor = scaling_factor
|
||||||
self.extrapolation_factor = extrapolation_factor
|
|
||||||
self.attn_factor = attn_factor
|
|
||||||
self.beta_fast = beta_fast
|
self.beta_fast = beta_fast
|
||||||
self.beta_slow = beta_slow
|
self.beta_slow = beta_slow
|
||||||
self.truncate = truncate
|
self.truncate = truncate
|
||||||
# Get n-d magnitude scaling corrected for interpolation
|
# Get n-d magnitude scaling corrected for interpolation
|
||||||
self.mscale = (
|
if attention_factor is not None:
|
||||||
float(yarn_get_mscale(self.scaling_factor) * attn_factor)
|
self.mscale = float(attention_factor)
|
||||||
if apply_yarn_scaling
|
elif mscale and mscale_all_dim:
|
||||||
else float(attn_factor)
|
self.mscale = float(
|
||||||
)
|
yarn_get_mscale(self.scaling_factor, mscale)
|
||||||
|
/ yarn_get_mscale(self.scaling_factor, mscale_all_dim)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.mscale = float(yarn_get_mscale(self.scaling_factor))
|
||||||
super().__init__(
|
super().__init__(
|
||||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||||
)
|
)
|
||||||
@@ -62,10 +64,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
|
|||||||
self.truncate,
|
self.truncate,
|
||||||
)
|
)
|
||||||
# Get n-d rotational scaling corrected for extrapolation
|
# Get n-d rotational scaling corrected for extrapolation
|
||||||
inv_freq_mask = (
|
inv_freq_mask = 1 - yarn_linear_ramp_mask(
|
||||||
1
|
low, high, self.rotary_dim // 2, dtype=torch.float
|
||||||
- yarn_linear_ramp_mask(low, high, self.rotary_dim // 2, dtype=torch.float)
|
)
|
||||||
) * self.extrapolation_factor
|
|
||||||
inv_freq = (
|
inv_freq = (
|
||||||
inv_freq_interpolation * (1 - inv_freq_mask)
|
inv_freq_interpolation * (1 - inv_freq_mask)
|
||||||
+ inv_freq_extrapolation * inv_freq_mask
|
+ inv_freq_extrapolation * inv_freq_mask
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
from collections.abc import Callable, Iterator, Sequence
|
from collections.abc import Callable, Iterator, Sequence
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
@@ -539,6 +540,14 @@ def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None:
|
|||||||
)
|
)
|
||||||
rope_parameters["rope_type"] = "default"
|
rope_parameters["rope_type"] = "default"
|
||||||
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
|
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
|
||||||
|
elif rope_parameters["rope_type"] == "telechat3-yarn":
|
||||||
|
# TeleChat3 is YaRN with 0.07 in place of YaRN's 0.1 attention
|
||||||
|
# scaling coefficient. Precompute it so the config is plain YaRN.
|
||||||
|
factor = rope_parameters["factor"]
|
||||||
|
rope_parameters["rope_type"] = "yarn"
|
||||||
|
rope_parameters["attention_factor"] = 0.07 * math.log(factor) + 1.0
|
||||||
|
rope_parameters.pop("type", None)
|
||||||
|
logger.warning("Replacing rope_type 'telechat3-yarn' with 'yarn'")
|
||||||
|
|
||||||
# Handle nested rope_parameters in interleaved sliding attention models
|
# Handle nested rope_parameters in interleaved sliding attention models
|
||||||
if is_rope_parameters_nested(rope_parameters):
|
if is_rope_parameters_nested(rope_parameters):
|
||||||
|
|||||||
@@ -2,9 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from packaging.version import Version
|
|
||||||
from transformers import PretrainedConfig, WhisperConfig
|
from transformers import PretrainedConfig, WhisperConfig
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
|
||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
@@ -119,7 +117,6 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
|
|||||||
"original_max_position_embeddings": ("original_max_position_embeddings", int),
|
"original_max_position_embeddings": ("original_max_position_embeddings", int),
|
||||||
"beta": ("beta_fast", float),
|
"beta": ("beta_fast", float),
|
||||||
"alpha": ("beta_slow", float),
|
"alpha": ("beta_slow", float),
|
||||||
"apply_scale": ("apply_yarn_scaling", bool),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
yarn_config = config.get("yarn") or {}
|
yarn_config = config.get("yarn") or {}
|
||||||
@@ -136,9 +133,10 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
|
|||||||
# Cast to remove Transformers > v5 type warnings
|
# Cast to remove Transformers > v5 type warnings
|
||||||
config["rope_parameters"][new_name] = cast(yarn_config.pop(old_name))
|
config["rope_parameters"][new_name] = cast(yarn_config.pop(old_name))
|
||||||
|
|
||||||
# Ignore apply_yarn_scaling in Transformers > v5 RoPE validation to remove warnings
|
# `apply_scale: false` means no magnitude correction, which Transformers
|
||||||
if Version(TRANSFORMERS_VERSION) >= Version("5.3.0.dev0"):
|
# spells as an explicit attention_factor of 1.
|
||||||
config["ignore_keys_at_rope_validation"] = {"apply_yarn_scaling"}
|
if not yarn_config.pop("apply_scale", True):
|
||||||
|
config["rope_parameters"]["attention_factor"] = 1.0
|
||||||
|
|
||||||
assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
|
assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user