Fix error in Dynamic NTK scaling (#41277)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
wang.yuqi
parent
1242196295
commit
117afeea46
@@ -211,10 +211,14 @@ def get_rope(
|
||||
)
|
||||
elif "factor" in rope_parameters:
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
max_trained_positions = rope_parameters.get(
|
||||
"max_trained_positions", max_position
|
||||
)
|
||||
rotary_emb = DynamicNTKScalingRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
max_trained_positions,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_factor,
|
||||
|
||||
@@ -38,12 +38,14 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
max_trained_positions: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
self.scaling_factor = scaling_factor
|
||||
self.max_trained_positions = max_trained_positions
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
@@ -53,13 +55,16 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
# maximum length before applying the rope scaling.
|
||||
# Thus, the maximum length after applying the rope scaling is
|
||||
# self.max_position_embeddings * self.scaling_factor.
|
||||
max_len = self.max_position_embeddings * self.scaling_factor
|
||||
base = self.base * (
|
||||
(self.scaling_factor * max_len / self.max_position_embeddings)
|
||||
(
|
||||
self.scaling_factor
|
||||
* self.max_position_embeddings
|
||||
/ self.max_trained_positions
|
||||
)
|
||||
- (self.scaling_factor - 1)
|
||||
) ** (self.rotary_dim / (self.rotary_dim - 2))
|
||||
inv_freq = self._compute_inv_freq(base)
|
||||
t = torch.arange(max_len, dtype=torch.float)
|
||||
t = torch.arange(self.max_position_embeddings, dtype=torch.float)
|
||||
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos()
|
||||
|
||||
Reference in New Issue
Block a user