[Model] Adding support for MSFT Phi-3.5-MoE (#7729)
Co-authored-by: Your Name <you@example.com> Co-authored-by: Zeqi Lin <zelin@microsoft.com> Co-authored-by: Zeqi Lin <Zeqi.Lin@microsoft.com>
This commit is contained in:
co-authored by
Your Name
Zeqi Lin
Zeqi Lin
parent
2684efc467
commit
1248e8506a
@@ -503,8 +503,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
|
||||
dtype: torch.dtype,
|
||||
short_factor: List[float],
|
||||
long_factor: List[float],
|
||||
short_mscale: float = 1.0,
|
||||
long_mscale: float = 1.0,
|
||||
short_mscale: Optional[float] = None,
|
||||
long_mscale: Optional[float] = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -523,18 +523,22 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
|
||||
self.base = base
|
||||
self.short_factor = short_factor
|
||||
self.long_factor = long_factor
|
||||
self.short_mscale = short_mscale
|
||||
self.long_mscale = long_mscale
|
||||
|
||||
scale = (self.max_position_embeddings /
|
||||
self.original_max_position_embeddings)
|
||||
|
||||
scale = self.max_position_embeddings / \
|
||||
self.original_max_position_embeddings
|
||||
if scale <= 1.0:
|
||||
self.scaling_factor = 1.0
|
||||
scaling_factor = 1.0
|
||||
else:
|
||||
self.scaling_factor = math.sqrt(
|
||||
scaling_factor = math.sqrt(
|
||||
1 + math.log(scale) /
|
||||
math.log(self.original_max_position_embeddings))
|
||||
if short_mscale is None:
|
||||
short_mscale = scaling_factor
|
||||
if long_mscale is None:
|
||||
long_mscale = scaling_factor
|
||||
|
||||
self.short_mscale = short_mscale
|
||||
self.long_mscale = long_mscale
|
||||
|
||||
short_cache = self._compute_cos_sin_cache(
|
||||
original_max_position_embeddings, short_factor, short_mscale)
|
||||
@@ -571,8 +575,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
|
||||
inv_freq = self._compute_inv_freq(rescale_factors)
|
||||
t = torch.arange(max_position_embeddings, dtype=torch.float)
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos() * mscale * self.scaling_factor
|
||||
sin = freqs.sin() * mscale * self.scaling_factor
|
||||
cos = freqs.cos() * mscale
|
||||
sin = freqs.sin() * mscale
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
|
||||
Reference in New Issue
Block a user