[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2 (#34917)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
This commit is contained in:
@@ -74,6 +74,12 @@ void indexer_k_quant_and_cache(
|
||||
int64_t quant_block_size, // quantization block size
|
||||
const std::string& scale_fmt);
|
||||
|
||||
// Concatenate query nope and rope for MLA/DSA attention
|
||||
void concat_mla_q(
|
||||
torch::Tensor& ql_nope, // [num_tokens, num_heads, nope_dim]
|
||||
torch::Tensor& q_pe, // [num_tokens, num_heads, rope_dim]
|
||||
torch::Tensor& q_out); // [num_tokens, num_heads, nope_dim + rope_dim]
|
||||
|
||||
// Extract function to gather quantized K cache
|
||||
void cp_gather_indexer_k_quant_cache(
|
||||
const torch::Tensor& kv_cache, // [num_blocks, block_size, cache_stride]
|
||||
|
||||
Reference in New Issue
Block a user