[ROCm] AITER fused RoPE+KVCache (#33443)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Signed-off-by: charlifu <charlifu@amd.com> Signed-off-by: Rohan Potdar <66227218+Rohan138@users.noreply.github.com> Co-authored-by: charlifu <charlifu@amd.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
charlifu
Luka Govedič
Douglas Lehr
parent
95642441d0
commit
2ff4e51152
@@ -1518,6 +1518,45 @@ class rocm_aiter_ops:
|
||||
query = query.view(query_shape)
|
||||
key = key.view(key_shape)
|
||||
|
||||
@staticmethod
|
||||
def triton_rope_and_cache(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
flash_layout: bool,
|
||||
apply_scale: bool,
|
||||
):
|
||||
from aiter.ops.triton.fused_kv_cache import fused_qk_rope_reshape_and_cache
|
||||
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
fused_qk_rope_reshape_and_cache(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
layer_slot_mapping,
|
||||
positions,
|
||||
cos,
|
||||
sin,
|
||||
k_scale,
|
||||
v_scale,
|
||||
is_neox,
|
||||
flash_layout=flash_layout,
|
||||
apply_scale=apply_scale,
|
||||
q_out=query,
|
||||
k_out=key,
|
||||
output_zeros=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def batched_gemm_a16wfp4(
|
||||
X: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user