[ROCm]: Enable customop and rope+kvcache fusion for AITER RoPE (#35180)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
This commit is contained in:
@@ -47,15 +47,20 @@ class RotaryEmbeddingBase(CustomOp):
|
||||
if not hasattr(self, "use_flashinfer"):
|
||||
self.use_flashinfer = False
|
||||
|
||||
self.use_aiter = (
|
||||
self.enabled() and rocm_aiter_ops.is_triton_rotary_embed_enabled()
|
||||
)
|
||||
if self.use_aiter:
|
||||
self.rocm_aiter_triton_rotary_embedding = (
|
||||
rocm_aiter_ops.get_triton_rotary_embedding_op()
|
||||
)
|
||||
|
||||
if init_cache:
|
||||
cache = self._compute_cos_sin_cache()
|
||||
if not self.use_flashinfer:
|
||||
cache = cache.to(dtype)
|
||||
self.cos_sin_cache: torch.Tensor
|
||||
self.register_buffer("cos_sin_cache", cache, persistent=False)
|
||||
self.is_rocm_triton_rotary_embed_enabled = (
|
||||
rocm_aiter_ops.is_triton_rotary_embed_enabled()
|
||||
)
|
||||
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
is_neox_style=self.is_neox_style,
|
||||
@@ -231,15 +236,14 @@ class RotaryEmbedding(RotaryEmbeddingBase):
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if self.is_rocm_triton_rotary_embed_enabled:
|
||||
if self.use_aiter:
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
rocm_aiter_ops.triton_rotary_embed(
|
||||
self.rocm_aiter_triton_rotary_embedding(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
cos_sin_cache,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
cos_sin_cache,
|
||||
self.is_neox_style,
|
||||
)
|
||||
return query, key
|
||||
|
||||
Reference in New Issue
Block a user