[Bugfix] Align vLLM YaRN with Transformers and stop re-scaling max_model_len (#56446)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-11 13:01:38 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent a5b714eee4
commit c191787a68
11 changed files with 133 additions and 134 deletions
@@ -24,7 +24,6 @@ from .mrope import MRotaryEmbedding
from .mrope_interleaved import MRotaryEmbeddingInterleaved
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
from .telechat3_scaling_rope import TeleChat3RoPEScaledRotaryEmbedding
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
_ROPE_DICT: dict[tuple[Any, ...], RotaryEmbedding] = {}
@@ -246,16 +245,15 @@ def get_rope(
for k, v in rope_parameters.items()
if k
in (
"extrapolation_factor",
"attn_factor",
"beta_fast",
"beta_slow",
"apply_yarn_scaling",
"mscale",
"mscale_all_dim",
"attention_factor",
"truncate",
)
}
if "mrope_section" in rope_parameters:
extra_kwargs.pop("apply_yarn_scaling", None)
rotary_emb = MRotaryEmbedding(
head_size,
rotary_dim,
@@ -288,8 +286,6 @@ def get_rope(
for k, v in rope_parameters.items()
if k
in (
"extrapolation_factor",
"attn_factor",
"beta_fast",
"beta_slow",
"mscale",
@@ -346,36 +342,6 @@ def get_rope(
)
else:
raise ValueError("Pangu mrope lacks necessary parameters.")
elif scaling_type == "telechat3-yarn":
scaling_factor = rope_parameters["factor"]
if "original_max_position_embeddings" in rope_parameters:
original_max_position = rope_parameters["original_max_position_embeddings"]
scaling_factor = max_position / original_max_position
else:
original_max_position = max_position
extra_kwargs = {
k: v
for k, v in rope_parameters.items()
if k
in (
"extrapolation_factor",
"attn_factor",
"beta_fast",
"beta_slow",
"mscale",
"mscale_all_dim",
)
}
rotary_emb = TeleChat3RoPEScaledRotaryEmbedding(
head_size,
rotary_dim,
original_max_position,
base,
is_neox_style,
scaling_factor,
dtype,
**extra_kwargs,
)
else:
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
_ROPE_DICT[key] = rotary_emb
@@ -70,10 +70,10 @@ def yarn_linear_ramp_mask(
return ramp_func
def yarn_get_mscale(scale: float = 1) -> float:
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
if scale <= 1:
return 1.0
return 0.1 * math.log(scale) + 1.0
return 0.1 * mscale * math.log(scale) + 1.0
def _flashinfer_rotary_embedding(
@@ -39,8 +39,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
scaling_factor: float,
dtype: torch.dtype,
*,
extrapolation_factor: float = 1,
attn_factor: float = 1,
beta_fast: int = 32,
beta_slow: int = 1,
mscale: float = 1,
@@ -48,15 +46,12 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
init_cache: bool = True,
) -> None:
self.scaling_factor = scaling_factor
self.extrapolation_factor = extrapolation_factor
self.attn_factor = attn_factor
self.beta_fast = beta_fast
self.beta_slow = beta_slow
# Get n-d magnitude scaling corrected for interpolation.
self.mscale = float(
yarn_get_mscale(self.scaling_factor, float(mscale))
/ yarn_get_mscale(self.scaling_factor, float(mscale_all_dim))
* attn_factor
)
self.use_flashinfer = (
self.enabled()
@@ -96,10 +91,9 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
self.max_position_embeddings,
)
# Get n-d rotational scaling corrected for extrapolation
inv_freq_mask = (
1
- yarn_linear_ramp_mask(low, high, self.rotary_dim // 2, dtype=torch.float)
) * self.extrapolation_factor
inv_freq_mask = 1 - yarn_linear_ramp_mask(
low, high, self.rotary_dim // 2, dtype=torch.float
)
inv_freq = (
inv_freq_interpolation * (1 - inv_freq_mask)
+ inv_freq_extrapolation * inv_freq_mask
@@ -252,21 +252,28 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
# YaRN parameters.
*,
scaling_factor: float | None = None,
extrapolation_factor: float = 1,
attn_factor: float = 1,
beta_fast: int = 32,
beta_slow: int = 1,
mscale: float | None = None,
mscale_all_dim: float | None = None,
attention_factor: float | None = None,
truncate: bool = True,
) -> None:
self.scaling_factor = scaling_factor
self.extrapolation_factor = extrapolation_factor
self.attn_factor = attn_factor
self.beta_fast = beta_fast
self.beta_slow = beta_slow
self.truncate = truncate
if self.scaling_factor is not None:
# Get n-d magnitude scaling corrected for interpolation
self.mscale = float(yarn_get_mscale(self.scaling_factor) * attn_factor)
if attention_factor is not None:
self.mscale = float(attention_factor)
elif mscale and mscale_all_dim:
self.mscale = float(
yarn_get_mscale(self.scaling_factor, mscale)
/ yarn_get_mscale(self.scaling_factor, mscale_all_dim)
)
else:
self.mscale = float(yarn_get_mscale(self.scaling_factor))
else:
self.mscale = 1.0
@@ -1,57 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import torch
from .base import RotaryEmbedding
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
class TeleChat3RoPEScaledRotaryEmbedding(YaRNScalingRotaryEmbedding):
"""TeleChat3 uses a variant of YaRN method.
To achieve code reuse as much as possible, we have rewritten the
`get_mscale` method in the initialization function
"""
def __init__(
self,
head_size: int,
rotary_dim: int,
max_position_embeddings: int,
base: int,
is_neox_style: bool,
scaling_factor: float,
dtype: torch.dtype,
*,
extrapolation_factor: float = 1,
attn_factor: float = 1,
beta_fast: int = 32,
beta_slow: int = 1,
truncate: bool = True,
) -> None:
self.scaling_factor = scaling_factor
self.extrapolation_factor = extrapolation_factor
self.attn_factor = attn_factor
self.beta_fast = beta_fast
self.beta_slow = beta_slow
self.truncate = truncate
def get_mscale(scale, mscale=1):
if scale <= 1:
return 1.0
return 0.07 * mscale * math.log(scale) + 1.0
self.mscale = float(get_mscale(self.scaling_factor) * attn_factor)
# Initialization must be performed after mscale, otherwise mscale is useless
RotaryEmbedding.__init__(
self,
head_size,
rotary_dim,
max_position_embeddings,
base,
is_neox_style,
dtype,
)
@@ -23,25 +23,27 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
scaling_factor: float,
dtype: torch.dtype,
*,
extrapolation_factor: float = 1,
attn_factor: float = 1,
beta_fast: int = 32,
beta_slow: int = 1,
apply_yarn_scaling: bool = True,
mscale: float | None = None,
mscale_all_dim: float | None = None,
attention_factor: float | None = None,
truncate: bool = True,
) -> None:
self.scaling_factor = scaling_factor
self.extrapolation_factor = extrapolation_factor
self.attn_factor = attn_factor
self.beta_fast = beta_fast
self.beta_slow = beta_slow
self.truncate = truncate
# Get n-d magnitude scaling corrected for interpolation
self.mscale = (
float(yarn_get_mscale(self.scaling_factor) * attn_factor)
if apply_yarn_scaling
else float(attn_factor)
)
if attention_factor is not None:
self.mscale = float(attention_factor)
elif mscale and mscale_all_dim:
self.mscale = float(
yarn_get_mscale(self.scaling_factor, mscale)
/ yarn_get_mscale(self.scaling_factor, mscale_all_dim)
)
else:
self.mscale = float(yarn_get_mscale(self.scaling_factor))
super().__init__(
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
)
@@ -62,10 +64,9 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
self.truncate,
)
# Get n-d rotational scaling corrected for extrapolation
inv_freq_mask = (
1
- yarn_linear_ramp_mask(low, high, self.rotary_dim // 2, dtype=torch.float)
) * self.extrapolation_factor
inv_freq_mask = 1 - yarn_linear_ramp_mask(
low, high, self.rotary_dim // 2, dtype=torch.float
)
inv_freq = (
inv_freq_interpolation * (1 - inv_freq_mask)
+ inv_freq_extrapolation * inv_freq_mask