[Model] Add Gemma 2 (#5908)
This commit is contained in:
@@ -610,6 +610,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
|
||||
return query.flatten(-2), key.flatten(-2)
|
||||
|
||||
|
||||
class GemmaRotaryEmbedding(RotaryEmbedding):
|
||||
|
||||
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
|
||||
# https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
|
||||
inv_freq = 1.0 / (base**(
|
||||
torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
|
||||
self.rotary_dim))
|
||||
return inv_freq
|
||||
|
||||
|
||||
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user