[Perf] Optimize moe permute by pre-allocate buffer, 9~14% kernel performance improvement (#43014)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-05-28 06:18:26 -07:00
committed by GitHub
parent 02606b0b09
commit 64e1218673
8 changed files with 480 additions and 92 deletions
+13
View File
@@ -100,13 +100,26 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
"expert_first_token_offset, Tensor! inv_permuted_idx, Tensor! "
"permuted_idx)->()");
m.def(
"moe_permute_with_scratch(Tensor input, Tensor topk_ids,"
"Tensor token_expert_indices, Tensor? expert_map, int n_expert,"
"int n_local_expert,"
"int topk, Tensor! permuted_input, Tensor! "
"expert_first_token_offset, Tensor! inv_permuted_idx, Tensor! "
"permuted_idx, Tensor! sort_workspace, Tensor! permuted_experts_id, "
"Tensor! sorted_row_idx, Tensor! topk_ids_for_sort)->()");
m.def(
"moe_unpermute(Tensor permuted_hidden_states, Tensor topk_weights,"
"Tensor inv_permuted_idx, Tensor? expert_first_token_offset, "
"int topk, Tensor! hidden_states)->()");
m.def("moe_permute_unpermute_supported() -> bool");
m.def(
"moe_permute_sort_workspace_size(int num_expanded_rows, int n_expert) -> "
"int");
m.impl("moe_permute_unpermute_supported", &moe_permute_unpermute_supported);
m.impl("moe_permute_sort_workspace_size", &moe_permute_sort_workspace_size);
// Row shuffle for MoE
m.def(