[Feature] Support Decode Context Parallel (DCP) for MLA (#23734)
Signed-off-by: hongchao <hongchao@msh.team> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: hongchao <hongchao@msh.team> Co-authored-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
co-authored by
hongchao
youkaichao
parent
3c529fc994
commit
ac201a0eaf
@@ -1625,20 +1625,6 @@ def concat_and_cache_mla(
|
||||
scale)
|
||||
|
||||
|
||||
def cp_fused_concat_and_cache_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
cp_local_token_select_indices: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
scale: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.cp_fused_concat_and_cache_mla(
|
||||
kv_c, k_pe, cp_local_token_select_indices, kv_cache, slot_mapping,
|
||||
kv_cache_dtype, scale)
|
||||
|
||||
|
||||
def copy_blocks(key_caches: list[torch.Tensor],
|
||||
value_caches: list[torch.Tensor],
|
||||
block_mapping: torch.Tensor) -> None:
|
||||
|
||||
Reference in New Issue
Block a user