[Perf] Optimize moe permute by pre-allocate buffer, 9~14% kernel performance improvement (#43014)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-05-28 06:18:26 -07:00
committed by GitHub
parent 02606b0b09
commit 64e1218673
8 changed files with 480 additions and 92 deletions
@@ -17,7 +17,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
moe_permute,
moe_permute_unpermute_supported,
moe_unpermute,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
@@ -73,6 +75,7 @@ def run_cutlass_moe_fp8(
per_out_ch: bool,
use_batched_format: bool,
topk_weights: torch.Tensor | None,
permute_scratch: MoEPermuteScratch | None,
):
a1q = hidden_states
@@ -198,6 +201,7 @@ def run_cutlass_moe_fp8(
local_E,
expert_map,
permuted_hidden_states=a1q_perm,
scratch=permute_scratch,
)
# swap_ab is a CUTLASS grouped-GEMM optimization (M <= 64 reduces padding).
swap_ab = a1q.size(0) <= 64
@@ -291,6 +295,7 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
self.ab_strides2 = ab_strides2
self.c_strides1 = c_strides1
self.c_strides2 = ab_strides1_c_strides2
self._permute_scratch: MoEPermuteScratch | None = None
@staticmethod
def _supports_current_device() -> bool:
@@ -324,6 +329,17 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
# Let PrepareAndFinalize::finalize() decide the impl.
return TopKWeightAndReduceDelegate()
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
return self._permute_scratch
def apply(
self,
output: torch.Tensor,
@@ -379,6 +395,7 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
self.per_out_ch_quant,
use_batched_format,
topk_weights,
self._get_permute_scratch(),
)
@@ -1121,6 +1138,7 @@ def run_cutlass_moe_w4a8_fp8(
use_batched_format: bool,
topk_weights: torch.Tensor | None,
group_size: int,
permute_scratch: MoEPermuteScratch | None,
):
a1q = hidden_states
M = a1q.size(0)
@@ -1176,6 +1194,7 @@ def run_cutlass_moe_w4a8_fp8(
local_E,
expert_map,
permuted_hidden_states=a1q_perm,
scratch=permute_scratch,
)
# for RS gemm SwapAB is always enabled (swap logical M, N in the problem shape).
ops.get_cutlass_moe_mm_problem_sizes_from_expert_offsets(
@@ -1266,6 +1285,7 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
self.s_strides2[:, 0] = k
self.group_size = group_size
self._permute_scratch: MoEPermuteScratch | None = None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -1330,6 +1350,17 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype:
return self.out_dtype if self.out_dtype is not None else act_dtype
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
)
return self._permute_scratch
def workspace_shapes(
self,
M: int,
@@ -1409,4 +1440,5 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
use_batched_format,
topk_weights,
self.group_size,
self._get_permute_scratch(),
)
@@ -26,7 +26,9 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
)
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import moe_fused_mul_sum
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
moe_permute,
moe_permute_unpermute_supported,
moe_unpermute,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
@@ -85,6 +87,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
self._permute_scratch: MoEPermuteScratch | None = None
def init_humming_moe(self):
self.compute_config = {
@@ -110,6 +113,19 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
if self._permute_scratch is None and moe_permute_unpermute_supported():
self._permute_scratch = MoEPermuteScratch(
max_num_tokens=self.moe_config.max_num_tokens,
topk=self.moe_config.experts_per_token,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
)
return self._permute_scratch
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
num_tokens = topk_ids.size(0)
ctx = get_forward_context()
@@ -598,6 +614,7 @@ class HummingGroupedExperts(HummingExpertsBase):
n_expert=self.global_num_experts,
n_local_expert=self.num_experts,
expert_map=self.layer.expert_map,
scratch=self._get_permute_scratch(),
)
inputs, input_scale = HummingMethod.may_quant_input(
@@ -1,9 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
import torch
@dataclass
class MoEPermuteScratch:
# Reused metadata buffers for repeated grouped-MoE permutes.
max_num_tokens: int
topk: int
num_experts: int
num_local_experts: int
device: torch.device
hidden_size: int | None = None
hidden_dtype: torch.dtype | None = None
token_expert_indices: torch.Tensor = field(init=False)
expert_first_token_offset: torch.Tensor = field(init=False)
permuted_idx: torch.Tensor = field(init=False)
inv_permuted_idx: torch.Tensor = field(init=False)
permuted_hidden_states: torch.Tensor | None = field(init=False, default=None)
sort_workspace: torch.Tensor = field(init=False)
permuted_experts_id: torch.Tensor = field(init=False)
sorted_row_idx: torch.Tensor = field(init=False)
topk_ids_int32: torch.Tensor = field(init=False)
topk_ids_for_sort: torch.Tensor = field(init=False)
max_expanded_rows: int = field(init=False)
def __post_init__(self) -> None:
assert self.max_num_tokens > 0
assert self.topk > 0
assert self.num_experts > 0
assert self.num_local_experts > 0
if self.hidden_size is None:
assert self.hidden_dtype is None
else:
assert self.hidden_dtype is not None
self.max_expanded_rows = self.max_num_tokens * self.topk
self.token_expert_indices = torch.arange(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.expert_first_token_offset = torch.empty(
self.num_local_experts + 1, dtype=torch.int64, device=self.device
)
self.permuted_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.inv_permuted_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
if self.hidden_size is not None:
hidden_numel = self.max_expanded_rows * self.hidden_size
self.permuted_hidden_states = torch.empty(
hidden_numel, dtype=self.hidden_dtype, device=self.device
)
self.permuted_experts_id = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.sorted_row_idx = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.topk_ids_int32 = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
self.topk_ids_for_sort = torch.empty(
self.max_expanded_rows, dtype=torch.int32, device=self.device
)
sorter_size = torch.ops._moe_C.moe_permute_sort_workspace_size(
self.max_expanded_rows, self.num_experts
)
self.sort_workspace = torch.empty(
sorter_size, dtype=torch.int8, device=self.device
)
def validate(self, hidden_states: torch.Tensor, topk_ids: torch.Tensor) -> None:
n_token, n_hidden = hidden_states.shape
assert hidden_states.device == self.device
assert topk_ids.device == self.device
assert n_token <= self.max_num_tokens
assert topk_ids.size(1) == self.topk
assert topk_ids.size(0) == n_token
if self.hidden_size is not None:
assert n_hidden == self.hidden_size
assert hidden_states.dtype == self.hidden_dtype
assert self.permuted_hidden_states is not None
def token_expert_indices_view(self, n_token: int) -> torch.Tensor:
return self.token_expert_indices[: n_token * self.topk].view(n_token, self.topk)
def prepare_topk_ids(self, topk_ids: torch.Tensor) -> torch.Tensor:
if topk_ids.dtype == torch.int32:
return topk_ids
numel = topk_ids.numel()
topk_ids_int32 = self.topk_ids_int32[:numel].view_as(topk_ids)
topk_ids_int32.copy_(topk_ids)
return topk_ids_int32
def moe_permute(
hidden_states: torch.Tensor,
a1q_scale: torch.Tensor | None,
@@ -12,6 +107,7 @@ def moe_permute(
n_local_expert: int = -1,
expert_map: torch.Tensor | None = None,
permuted_hidden_states: torch.Tensor | None = None,
scratch: MoEPermuteScratch | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
This function expands and permutes activation to gather uncontinuous tokens
@@ -45,46 +141,92 @@ def moe_permute(
if n_local_expert == -1:
n_local_expert = n_expert
if permuted_hidden_states is None:
permuted_hidden_states = torch.empty(
(permuted_row_size, n_hidden),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
if scratch is None:
permuted_hidden_states = torch.empty(
(permuted_row_size, n_hidden),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
else:
scratch.validate(hidden_states, topk_ids)
hidden_numel = permuted_row_size * n_hidden
scratch_hidden_states = scratch.permuted_hidden_states
assert scratch_hidden_states is not None
permuted_hidden_states = scratch_hidden_states[:hidden_numel].view(
permuted_row_size, n_hidden
)
assert permuted_hidden_states.size() == (permuted_row_size, n_hidden), (
f"Expected permuted hidden states to be {(permuted_row_size, n_hidden)}"
f" but got {permuted_hidden_states.size()}"
)
token_expert_indices = torch.arange(
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
).reshape((n_token, topk))
if scratch is None:
token_expert_indices = torch.arange(
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
).reshape((n_token, topk))
expert_first_token_offset = torch.empty(
n_local_expert + 1, dtype=torch.int64, device=hidden_states.device
)
permuted_idx = torch.full(
(permuted_row_size,),
n_token * topk,
dtype=torch.int32,
device=hidden_states.device,
)
inv_permuted_idx = torch.empty(
(n_token, topk), dtype=torch.int32, device=hidden_states.device
)
topk_ids = topk_ids.to(torch.int32)
torch.ops._moe_C.moe_permute(
hidden_states,
topk_ids,
token_expert_indices,
expert_map,
n_expert,
n_local_expert,
topk,
permuted_hidden_states,
expert_first_token_offset,
inv_permuted_idx,
permuted_idx,
)
expert_first_token_offset = torch.empty(
n_local_expert + 1, dtype=torch.int64, device=hidden_states.device
)
permuted_idx = torch.full(
(permuted_row_size,),
n_token * topk,
dtype=torch.int32,
device=hidden_states.device,
)
inv_permuted_idx = torch.empty(
(n_token, topk), dtype=torch.int32, device=hidden_states.device
)
topk_ids_int32 = topk_ids.to(torch.int32)
torch.ops._moe_C.moe_permute(
hidden_states,
topk_ids_int32,
token_expert_indices,
expert_map,
n_expert,
n_local_expert,
topk,
permuted_hidden_states,
expert_first_token_offset,
inv_permuted_idx,
permuted_idx,
)
else:
scratch.validate(hidden_states, topk_ids)
assert n_expert == scratch.num_experts
assert n_local_expert == scratch.num_local_experts
token_expert_indices = scratch.token_expert_indices_view(n_token)
expert_first_token_offset = scratch.expert_first_token_offset
permuted_idx = scratch.permuted_idx[:permuted_row_size]
permuted_idx.fill_(permuted_row_size)
inv_permuted_idx = scratch.inv_permuted_idx[:permuted_row_size].view(
n_token, topk
)
permuted_experts_id = scratch.permuted_experts_id[:permuted_row_size].view(
n_token, topk
)
sorted_row_idx = scratch.sorted_row_idx[:permuted_row_size].view(n_token, topk)
topk_ids_for_sort = scratch.topk_ids_for_sort[:permuted_row_size].view(
n_token, topk
)
topk_ids_int32 = scratch.prepare_topk_ids(topk_ids)
torch.ops._moe_C.moe_permute_with_scratch(
hidden_states,
topk_ids_int32,
token_expert_indices,
expert_map,
n_expert,
n_local_expert,
topk,
permuted_hidden_states,
expert_first_token_offset,
inv_permuted_idx,
permuted_idx,
scratch.sort_workspace,
permuted_experts_id,
sorted_row_idx,
topk_ids_for_sort,
)
if a1q_scale is not None and a1q_scale.dim() > 1:
a1q_scale = a1q_scale[permuted_idx.clamp(max=n_token * topk - 1) // topk]