[Kernel] support merge_attn_states CUDA kernel, 3x speedup (#16173)

Signed-off-by: DefTruth <qiustudent_r@163.com>
This commit is contained in:
DefTruth
2025-04-11 06:50:50 -06:00
committed by GitHub
parent 51baa9c333
commit e9528f6dc6
10 changed files with 519 additions and 4 deletions
+9
View File
@@ -52,6 +52,15 @@ void paged_attention_v2(
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
const int64_t blocksparse_head_sliding_step);
#ifndef USE_ROCM
void merge_attn_states(torch::Tensor& output,
std::optional<torch::Tensor> output_lse,
const torch::Tensor& prefix_output,
const torch::Tensor& prefix_lse,
const torch::Tensor& suffix_output,
const torch::Tensor& suffix_lse);
#endif
void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
double epsilon);