Fuse RoPE and MLA KV-cache write (#25774)
Signed-off-by: Patryk Saffer <patryk.saffer99@gmail.com> Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai> Co-authored-by: Patryk Saffer <patryk.saffer99@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Patryk Saffer
Luka Govedič
Michael Goin
parent
e45946bd91
commit
80fead8bf6
@@ -2418,6 +2418,32 @@ def concat_and_cache_mla(
|
||||
)
|
||||
|
||||
|
||||
def concat_and_cache_mla_rope_fused(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.concat_and_cache_mla_rope_fused(
|
||||
positions,
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
)
|
||||
|
||||
|
||||
def swap_blocks(
|
||||
src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user