[ROCm][Perf] Extend QK-norm/RoPE/KV-cache fusion to MRoPE (#50212)
Signed-off-by: Vorapol Assavasangthong <Vorapol.Assavasangthong@amd.com> Co-authored-by: Santosh Hiremath <Santosh.Hiremath@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Santosh Hiremath
Douglas Lehr
parent
0549e8d0ab
commit
b715699df4
@@ -3362,6 +3362,115 @@ class rocm_aiter_ops:
|
||||
rotary_dim=kernel_rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fused_qk_norm_mrope_and_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
mrope_section: list[int],
|
||||
is_interleaved: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
block_size: int,
|
||||
x: int,
|
||||
rotary_dim: int = 0,
|
||||
) -> None:
|
||||
from aiter.ops.fused_qk_norm_mrope_cache_quant import (
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle,
|
||||
)
|
||||
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
qkv.size(0),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_k,
|
||||
head_dim,
|
||||
is_neox,
|
||||
mrope_section,
|
||||
is_interleaved,
|
||||
rms_norm_eps,
|
||||
q_out,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slot_mapping,
|
||||
k_scale,
|
||||
v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
block_size,
|
||||
x,
|
||||
rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
mrope_section: list[int],
|
||||
is_interleaved: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
rotary_dim: int = 0,
|
||||
) -> None:
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
rocm_aiter_ops.fused_qk_norm_mrope_and_cache(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=num_heads_q,
|
||||
num_heads_k=num_heads_k,
|
||||
head_dim=head_dim,
|
||||
is_neox=is_neox,
|
||||
mrope_section=mrope_section,
|
||||
is_interleaved=is_interleaved,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_cache=key_cache,
|
||||
v_cache=value_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
block_size=key_cache.shape[1],
|
||||
x=16 // key_cache.element_size(),
|
||||
rotary_dim=rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def triton_rope_and_cache(
|
||||
query: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user