[Model] Add Gemma 2 (#5908)

This commit is contained in:
Woosuk Kwon
2024-06-27 13:33:56 -07:00
committed by GitHub
parent 736ed38849
commit 79c92c7c8a
9 changed files with 499 additions and 9 deletions
@@ -610,6 +610,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
return query.flatten(-2), key.flatten(-2)
class GemmaRotaryEmbedding(RotaryEmbedding):
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
# https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
inv_freq = 1.0 / (base**(
torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
self.rotary_dim))
return inv_freq
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}