[Kernel] cuda kernels for upcoming decode context parallel feature (#23791)
Co-authored-by: hongchao <hongchao@msh.team>
This commit is contained in:
@@ -1625,6 +1625,20 @@ def concat_and_cache_mla(
|
||||
scale)
|
||||
|
||||
|
||||
def cp_fused_concat_and_cache_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
cp_local_token_select_indices: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
scale: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.cp_fused_concat_and_cache_mla(
|
||||
kv_c, k_pe, cp_local_token_select_indices, kv_cache, slot_mapping,
|
||||
kv_cache_dtype, scale)
|
||||
|
||||
|
||||
def copy_blocks(key_caches: list[torch.Tensor],
|
||||
value_caches: list[torch.Tensor],
|
||||
block_mapping: torch.Tensor) -> None:
|
||||
@@ -1662,6 +1676,16 @@ def gather_and_maybe_dequant_cache(
|
||||
scale, seq_starts)
|
||||
|
||||
|
||||
def cp_gather_cache(src_cache: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
batch_size: int,
|
||||
seq_starts: Optional[torch.Tensor] = None) -> None:
|
||||
torch.ops._C_cache_ops.cp_gather_cache(src_cache, dst, block_table,
|
||||
cu_seq_lens, batch_size, seq_starts)
|
||||
|
||||
|
||||
def get_device_attribute(attribute: int, device: int) -> int:
|
||||
return torch.ops._C_cuda_utils.get_device_attribute(attribute, device)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user