[NVIDIA] Support Cutlass MLA for Blackwell GPUs (#16032)
Signed-off-by: kaixih <kaixih@nvidia.com>
This commit is contained in:
@@ -1525,3 +1525,12 @@ def flash_mla_with_kvcache(
|
||||
num_splits,
|
||||
)
|
||||
return out, softmax_lse
|
||||
|
||||
|
||||
def cutlass_mla_decode(out: torch.Tensor, q_nope: torch.Tensor,
|
||||
q_pe: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor,
|
||||
seq_lens: torch.Tensor, page_table: torch.Tensor,
|
||||
scale: float) -> torch.Tensor:
|
||||
torch.ops._C.cutlass_mla_decode(out, q_nope, q_pe, kv_c_and_k_pe_cache,
|
||||
seq_lens, page_table, scale)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user