[Bugfix] Fix repeated DSv4 RoPE cache initialization (#41148)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
11b69129e2
commit
9d8ad5b408
@@ -45,6 +45,7 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
||||
beta_slow: int = 1,
|
||||
mscale: float = 1,
|
||||
mscale_all_dim: float = 0,
|
||||
init_cache: bool = True,
|
||||
) -> None:
|
||||
self.scaling_factor = scaling_factor
|
||||
self.extrapolation_factor = extrapolation_factor
|
||||
@@ -65,7 +66,13 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
||||
and head_size in [64, 128, 256, 512]
|
||||
)
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position_embeddings,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
init_cache=init_cache,
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
|
||||
@@ -211,7 +218,9 @@ class DeepseekV4ScalingRotaryEmbedding(DeepseekScalingRotaryEmbedding):
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# Avoid compute cache repeatedly
|
||||
kwargs.pop("init_cache", None)
|
||||
super().__init__(*args, **kwargs, init_cache=False)
|
||||
cache_fp32 = self._compute_cos_sin_cache()
|
||||
self.register_buffer("cos_sin_cache", cache_fp32, persistent=False)
|
||||
|
||||
|
||||
@@ -1027,7 +1027,6 @@ class DeepseekV4Attention(nn.Module):
|
||||
max_position=self.max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=False,
|
||||
dtype=config.torch_dtype,
|
||||
)
|
||||
|
||||
self.indexer = None
|
||||
|
||||
Reference in New Issue
Block a user