Fix error in Dynamic NTK scaling (#41277)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Signed-off-by: Max de Bayser <maxdebayser@gmail.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Max de Bayser
2026-05-19 17:27:54 -04:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> wang.yuqi
parent 1242196295
commit 117afeea46
4 changed files with 45 additions and 152 deletions
@@ -211,10 +211,14 @@ def get_rope(
)
elif "factor" in rope_parameters:
scaling_factor = rope_parameters["factor"]
max_trained_positions = rope_parameters.get(
"max_trained_positions", max_position
)
rotary_emb = DynamicNTKScalingRotaryEmbedding(
head_size,
rotary_dim,
max_position,
max_trained_positions,
base,
is_neox_style,
scaling_factor,
@@ -38,12 +38,14 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
head_size: int,
rotary_dim: int,
max_position_embeddings: int,
max_trained_positions: int,
base: float,
is_neox_style: bool,
scaling_factor: float,
dtype: torch.dtype,
) -> None:
self.scaling_factor = scaling_factor
self.max_trained_positions = max_trained_positions
super().__init__(
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
)
@@ -53,13 +55,16 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
# maximum length before applying the rope scaling.
# Thus, the maximum length after applying the rope scaling is
# self.max_position_embeddings * self.scaling_factor.
max_len = self.max_position_embeddings * self.scaling_factor
base = self.base * (
(self.scaling_factor * max_len / self.max_position_embeddings)
(
self.scaling_factor
* self.max_position_embeddings
/ self.max_trained_positions
)
- (self.scaling_factor - 1)
) ** (self.rotary_dim / (self.rotary_dim - 2))
inv_freq = self._compute_inv_freq(base)
t = torch.arange(max_len, dtype=torch.float)
t = torch.arange(self.max_position_embeddings, dtype=torch.float)
freqs = torch.einsum("i,j -> ij", t, inv_freq)
cos = freqs.cos()