Fuse RoPE and MLA KV-cache write (#25774)

Signed-off-by: Patryk Saffer <patryk.saffer99@gmail.com>
Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai>
Co-authored-by: Patryk Saffer <patryk.saffer99@gmail.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
PatrykSaffer
2026-01-09 19:18:37 -08:00
committed by GitHub
co-authored by Patryk Saffer Luka Govedič Michael Goin
parent e45946bd91
commit 80fead8bf6
6 changed files with 490 additions and 0 deletions
+26
View File
@@ -2418,6 +2418,32 @@ def concat_and_cache_mla(
)
def concat_and_cache_mla_rope_fused(
positions: torch.Tensor,
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c: torch.Tensor,
cos_sin_cache: torch.Tensor,
is_neox: bool,
slot_mapping: torch.Tensor,
kv_cache: torch.Tensor,
kv_cache_dtype: str,
kv_cache_scale: torch.Tensor,
) -> None:
torch.ops._C_cache_ops.concat_and_cache_mla_rope_fused(
positions,
q_pe,
k_pe,
kv_c,
cos_sin_cache,
is_neox,
slot_mapping,
kv_cache,
kv_cache_dtype,
kv_cache_scale,
)
def swap_blocks(
src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor
) -> None: