[10b/n] Migrate custom all-reduce, DeepSeek V4 fused MLA, MiniMax reduce-RMS, and MXFP8 MoE to libtorch stable ABI (#44365)
Signed-off-by: Chris Leonard <chleonar@redhat.com> Signed-off-by: Shengqi Chen <harry-chen@outlook.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Shengqi Chen
parent
0a5cbf633e
commit
59d0236193
@@ -337,6 +337,24 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"bool is_neox, Tensor position_ids, "
|
||||
"int forced_token_heads_per_warp=-1) -> ()");
|
||||
|
||||
ops.def(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert("
|
||||
"Tensor q_in, Tensor kv, Tensor! k_cache, "
|
||||
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
|
||||
"int q_head_padded, float eps, int cache_block_size) -> Tensor");
|
||||
|
||||
#ifndef USE_ROCM
|
||||
ops.def(
|
||||
"minimax_allreduce_rms("
|
||||
"Tensor input, Tensor norm_weight, Tensor workspace, "
|
||||
"int rank, int nranks, float eps) -> Tensor");
|
||||
ops.def(
|
||||
"minimax_allreduce_rms_qk("
|
||||
"Tensor qkv, Tensor norm_weight_q, Tensor norm_weight_k, "
|
||||
"Tensor workspace, int q_size, int kv_size, int rank, int nranks, "
|
||||
"float eps) -> (Tensor, Tensor)");
|
||||
#endif
|
||||
|
||||
// Apply repetition penalties to logits in-place.
|
||||
ops.def(
|
||||
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
|
||||
@@ -571,6 +589,12 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
// Positional encoding kernels (shared CUDA/ROCm)
|
||||
ops.impl("rotary_embedding", TORCH_BOX(&rotary_embedding));
|
||||
ops.impl("fused_qk_norm_rope", TORCH_BOX(&fused_qk_norm_rope));
|
||||
ops.impl("fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert",
|
||||
TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert));
|
||||
#ifndef USE_ROCM
|
||||
ops.impl("minimax_allreduce_rms", TORCH_BOX(&minimax_allreduce_rms));
|
||||
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
|
||||
#endif
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
ops.impl("apply_repetition_penalties_",
|
||||
@@ -725,6 +749,45 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
"dst_scale, Tensor block_table, Tensor cu_seq_lens) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ar) {
|
||||
custom_ar.def(
|
||||
"init_custom_ar(int[] ipc_tensors, Tensor rank_data, "
|
||||
"int rank, bool fully_connected) -> int");
|
||||
custom_ar.def(
|
||||
"all_reduce(int fa, Tensor inp, Tensor! out, int reg_buffer, "
|
||||
"int reg_buffer_sz_bytes) -> ()");
|
||||
custom_ar.def("dispose(int fa) -> ()");
|
||||
custom_ar.def("meta_size() -> int");
|
||||
custom_ar.def("register_buffer(int fa, int[] ipc_tensors) -> ()");
|
||||
custom_ar.def("get_graph_buffer_ipc_meta(int fa) -> (int[], int[])");
|
||||
custom_ar.def(
|
||||
"register_graph_buffers(int fa, int[][] handles, int[][] offsets) -> ()");
|
||||
custom_ar.def("allocate_shared_buffer_and_handle(int size) -> (int, Tensor)");
|
||||
custom_ar.def("open_mem_handle(Tensor mem_handle) -> int");
|
||||
custom_ar.def("free_shared_buffer(int ptr) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ar) {
|
||||
custom_ar.impl("init_custom_ar", TORCH_BOX(&init_custom_ar));
|
||||
custom_ar.impl("all_reduce", TORCH_BOX(&all_reduce));
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CPU, custom_ar) {
|
||||
custom_ar.impl("open_mem_handle", TORCH_BOX(&open_mem_handle));
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CompositeExplicitAutograd, custom_ar) {
|
||||
custom_ar.impl("dispose", TORCH_BOX(&dispose));
|
||||
custom_ar.impl("meta_size", TORCH_BOX(&meta_size));
|
||||
custom_ar.impl("register_buffer", TORCH_BOX(®ister_buffer));
|
||||
custom_ar.impl("get_graph_buffer_ipc_meta",
|
||||
TORCH_BOX(&get_graph_buffer_ipc_meta));
|
||||
custom_ar.impl("register_graph_buffers", TORCH_BOX(®ister_graph_buffers));
|
||||
custom_ar.impl("allocate_shared_buffer_and_handle",
|
||||
TORCH_BOX(&allocate_shared_buffer_and_handle));
|
||||
custom_ar.impl("free_shared_buffer", TORCH_BOX(&free_shared_buffer));
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CPU, ops) {
|
||||
ops.impl("swap_blocks_batch", TORCH_BOX(&swap_blocks_batch));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user