Add gather_indexer_k_quant_cache kernel (#25931)
Signed-off-by: Barry Kang <43644113+Barry-Delaney@users.noreply.github.com> Signed-off-by: Simon Mo <simon.mo@hey.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Simon Mo
Yongye Zhu
Chen Zhang
parent
cd9890544b
commit
127c8b782a
@@ -2108,6 +2108,18 @@ def indexer_k_quant_and_cache(
|
||||
)
|
||||
|
||||
|
||||
def cp_gather_indexer_k_quant_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
dst_k: torch.Tensor,
|
||||
dst_scale: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
|
||||
)
|
||||
|
||||
|
||||
def get_device_attribute(attribute: int, device: int) -> int:
|
||||
return torch.ops._C_cuda_utils.get_device_attribute(attribute, device)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user