[CI] Make mistral tests pass (#4596)

This commit is contained in:
SangBin Cho
2024-05-08 08:44:35 -07:00
committed by GitHub
parent d7740ea4dc
commit f6a593093a
5 changed files with 85 additions and 19 deletions
@@ -109,7 +109,7 @@ class RotaryEmbedding(nn.Module):
key_pass = key[..., self.rotary_dim:]
self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
positions.device)
positions.device, dtype=query.dtype)
cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
if offsets is not None else positions]
cos, sin = cos_sin.chunk(2, dim=-1)
@@ -143,7 +143,8 @@ class RotaryEmbedding(nn.Module):
key: torch.Tensor,
offsets: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
dtype=query.dtype)
# ops.rotary_embedding()/batched_rotary_embedding()
# are in-place operations that update the query and key tensors.
if offsets is not None: