[XPU] Enable multiple key kernels for sparse attention (#37888)
Signed-off-by: Xiaochang Wu <xiaochang.wu@intel.com> Signed-off-by: Wu, Xiaochang <xiaochang.wu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
18a27cc9a3
commit
8c8b1825eb
@@ -2826,6 +2826,50 @@ def indexer_k_quant_and_cache(
|
||||
)
|
||||
|
||||
|
||||
def top_k_per_row_prefill(
|
||||
logits: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
raw_topk_indices: torch.Tensor,
|
||||
num_rows: int,
|
||||
stride0: int,
|
||||
stride1: int,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
raw_topk_indices,
|
||||
num_rows,
|
||||
stride0,
|
||||
stride1,
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
|
||||
def top_k_per_row_decode(
|
||||
logits: torch.Tensor,
|
||||
next_n: int,
|
||||
seq_lens: torch.Tensor,
|
||||
raw_topk_indices: torch.Tensor,
|
||||
num_rows: int,
|
||||
stride0: int,
|
||||
stride1: int,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
raw_topk_indices,
|
||||
num_rows,
|
||||
stride0,
|
||||
stride1,
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
|
||||
def cp_gather_indexer_k_quant_cache(
|
||||
kv_cache: torch.Tensor,
|
||||
dst_k: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user