[Perf] Optimize moe permute by pre-allocate buffer, 9~14% kernel performance improvement (#43014)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -17,7 +17,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
@@ -73,6 +75,7 @@ def run_cutlass_moe_fp8(
|
||||
per_out_ch: bool,
|
||||
use_batched_format: bool,
|
||||
topk_weights: torch.Tensor | None,
|
||||
permute_scratch: MoEPermuteScratch | None,
|
||||
):
|
||||
a1q = hidden_states
|
||||
|
||||
@@ -198,6 +201,7 @@ def run_cutlass_moe_fp8(
|
||||
local_E,
|
||||
expert_map,
|
||||
permuted_hidden_states=a1q_perm,
|
||||
scratch=permute_scratch,
|
||||
)
|
||||
# swap_ab is a CUTLASS grouped-GEMM optimization (M <= 64 reduces padding).
|
||||
swap_ab = a1q.size(0) <= 64
|
||||
@@ -291,6 +295,7 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
self.ab_strides2 = ab_strides2
|
||||
self.c_strides1 = c_strides1
|
||||
self.c_strides2 = ab_strides1_c_strides2
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
@@ -324,6 +329,17 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
# Let PrepareAndFinalize::finalize() decide the impl.
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
@@ -379,6 +395,7 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
self.per_out_ch_quant,
|
||||
use_batched_format,
|
||||
topk_weights,
|
||||
self._get_permute_scratch(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1121,6 +1138,7 @@ def run_cutlass_moe_w4a8_fp8(
|
||||
use_batched_format: bool,
|
||||
topk_weights: torch.Tensor | None,
|
||||
group_size: int,
|
||||
permute_scratch: MoEPermuteScratch | None,
|
||||
):
|
||||
a1q = hidden_states
|
||||
M = a1q.size(0)
|
||||
@@ -1176,6 +1194,7 @@ def run_cutlass_moe_w4a8_fp8(
|
||||
local_E,
|
||||
expert_map,
|
||||
permuted_hidden_states=a1q_perm,
|
||||
scratch=permute_scratch,
|
||||
)
|
||||
# for RS gemm SwapAB is always enabled (swap logical M, N in the problem shape).
|
||||
ops.get_cutlass_moe_mm_problem_sizes_from_expert_offsets(
|
||||
@@ -1266,6 +1285,7 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
self.s_strides2[:, 0] = k
|
||||
|
||||
self.group_size = group_size
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -1330,6 +1350,17 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
def workspace_dtype(self, act_dtype: torch.dtype) -> torch.dtype:
|
||||
return self.out_dtype if self.out_dtype is not None else act_dtype
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -1409,4 +1440,5 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
use_batched_format,
|
||||
topk_weights,
|
||||
self.group_size,
|
||||
self._get_permute_scratch(),
|
||||
)
|
||||
|
||||
@@ -26,7 +26,9 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import moe_fused_mul_sum
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
@@ -85,6 +87,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
def init_humming_moe(self):
|
||||
self.compute_config = {
|
||||
@@ -110,6 +113,19 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
|
||||
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
return self._permute_scratch
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
num_tokens = topk_ids.size(0)
|
||||
ctx = get_forward_context()
|
||||
@@ -598,6 +614,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
n_expert=self.global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=self.layer.expert_map,
|
||||
scratch=self._get_permute_scratch(),
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
|
||||
@@ -1,9 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoEPermuteScratch:
|
||||
# Reused metadata buffers for repeated grouped-MoE permutes.
|
||||
max_num_tokens: int
|
||||
topk: int
|
||||
num_experts: int
|
||||
num_local_experts: int
|
||||
device: torch.device
|
||||
hidden_size: int | None = None
|
||||
hidden_dtype: torch.dtype | None = None
|
||||
token_expert_indices: torch.Tensor = field(init=False)
|
||||
expert_first_token_offset: torch.Tensor = field(init=False)
|
||||
permuted_idx: torch.Tensor = field(init=False)
|
||||
inv_permuted_idx: torch.Tensor = field(init=False)
|
||||
permuted_hidden_states: torch.Tensor | None = field(init=False, default=None)
|
||||
sort_workspace: torch.Tensor = field(init=False)
|
||||
permuted_experts_id: torch.Tensor = field(init=False)
|
||||
sorted_row_idx: torch.Tensor = field(init=False)
|
||||
topk_ids_int32: torch.Tensor = field(init=False)
|
||||
topk_ids_for_sort: torch.Tensor = field(init=False)
|
||||
max_expanded_rows: int = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert self.max_num_tokens > 0
|
||||
assert self.topk > 0
|
||||
assert self.num_experts > 0
|
||||
assert self.num_local_experts > 0
|
||||
if self.hidden_size is None:
|
||||
assert self.hidden_dtype is None
|
||||
else:
|
||||
assert self.hidden_dtype is not None
|
||||
|
||||
self.max_expanded_rows = self.max_num_tokens * self.topk
|
||||
self.token_expert_indices = torch.arange(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.expert_first_token_offset = torch.empty(
|
||||
self.num_local_experts + 1, dtype=torch.int64, device=self.device
|
||||
)
|
||||
self.permuted_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.inv_permuted_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if self.hidden_size is not None:
|
||||
hidden_numel = self.max_expanded_rows * self.hidden_size
|
||||
self.permuted_hidden_states = torch.empty(
|
||||
hidden_numel, dtype=self.hidden_dtype, device=self.device
|
||||
)
|
||||
self.permuted_experts_id = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.sorted_row_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.topk_ids_int32 = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.topk_ids_for_sort = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
sorter_size = torch.ops._moe_C.moe_permute_sort_workspace_size(
|
||||
self.max_expanded_rows, self.num_experts
|
||||
)
|
||||
self.sort_workspace = torch.empty(
|
||||
sorter_size, dtype=torch.int8, device=self.device
|
||||
)
|
||||
|
||||
def validate(self, hidden_states: torch.Tensor, topk_ids: torch.Tensor) -> None:
|
||||
n_token, n_hidden = hidden_states.shape
|
||||
assert hidden_states.device == self.device
|
||||
assert topk_ids.device == self.device
|
||||
assert n_token <= self.max_num_tokens
|
||||
assert topk_ids.size(1) == self.topk
|
||||
assert topk_ids.size(0) == n_token
|
||||
if self.hidden_size is not None:
|
||||
assert n_hidden == self.hidden_size
|
||||
assert hidden_states.dtype == self.hidden_dtype
|
||||
assert self.permuted_hidden_states is not None
|
||||
|
||||
def token_expert_indices_view(self, n_token: int) -> torch.Tensor:
|
||||
return self.token_expert_indices[: n_token * self.topk].view(n_token, self.topk)
|
||||
|
||||
def prepare_topk_ids(self, topk_ids: torch.Tensor) -> torch.Tensor:
|
||||
if topk_ids.dtype == torch.int32:
|
||||
return topk_ids
|
||||
numel = topk_ids.numel()
|
||||
topk_ids_int32 = self.topk_ids_int32[:numel].view_as(topk_ids)
|
||||
topk_ids_int32.copy_(topk_ids)
|
||||
return topk_ids_int32
|
||||
|
||||
|
||||
def moe_permute(
|
||||
hidden_states: torch.Tensor,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
@@ -12,6 +107,7 @@ def moe_permute(
|
||||
n_local_expert: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
permuted_hidden_states: torch.Tensor | None = None,
|
||||
scratch: MoEPermuteScratch | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
This function expands and permutes activation to gather uncontinuous tokens
|
||||
@@ -45,46 +141,92 @@ def moe_permute(
|
||||
if n_local_expert == -1:
|
||||
n_local_expert = n_expert
|
||||
if permuted_hidden_states is None:
|
||||
permuted_hidden_states = torch.empty(
|
||||
(permuted_row_size, n_hidden),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
if scratch is None:
|
||||
permuted_hidden_states = torch.empty(
|
||||
(permuted_row_size, n_hidden),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
else:
|
||||
scratch.validate(hidden_states, topk_ids)
|
||||
hidden_numel = permuted_row_size * n_hidden
|
||||
scratch_hidden_states = scratch.permuted_hidden_states
|
||||
assert scratch_hidden_states is not None
|
||||
permuted_hidden_states = scratch_hidden_states[:hidden_numel].view(
|
||||
permuted_row_size, n_hidden
|
||||
)
|
||||
assert permuted_hidden_states.size() == (permuted_row_size, n_hidden), (
|
||||
f"Expected permuted hidden states to be {(permuted_row_size, n_hidden)}"
|
||||
f" but got {permuted_hidden_states.size()}"
|
||||
)
|
||||
|
||||
token_expert_indices = torch.arange(
|
||||
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
|
||||
).reshape((n_token, topk))
|
||||
if scratch is None:
|
||||
token_expert_indices = torch.arange(
|
||||
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
|
||||
).reshape((n_token, topk))
|
||||
|
||||
expert_first_token_offset = torch.empty(
|
||||
n_local_expert + 1, dtype=torch.int64, device=hidden_states.device
|
||||
)
|
||||
permuted_idx = torch.full(
|
||||
(permuted_row_size,),
|
||||
n_token * topk,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
inv_permuted_idx = torch.empty(
|
||||
(n_token, topk), dtype=torch.int32, device=hidden_states.device
|
||||
)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
torch.ops._moe_C.moe_permute(
|
||||
hidden_states,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
expert_map,
|
||||
n_expert,
|
||||
n_local_expert,
|
||||
topk,
|
||||
permuted_hidden_states,
|
||||
expert_first_token_offset,
|
||||
inv_permuted_idx,
|
||||
permuted_idx,
|
||||
)
|
||||
expert_first_token_offset = torch.empty(
|
||||
n_local_expert + 1, dtype=torch.int64, device=hidden_states.device
|
||||
)
|
||||
permuted_idx = torch.full(
|
||||
(permuted_row_size,),
|
||||
n_token * topk,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
inv_permuted_idx = torch.empty(
|
||||
(n_token, topk), dtype=torch.int32, device=hidden_states.device
|
||||
)
|
||||
topk_ids_int32 = topk_ids.to(torch.int32)
|
||||
torch.ops._moe_C.moe_permute(
|
||||
hidden_states,
|
||||
topk_ids_int32,
|
||||
token_expert_indices,
|
||||
expert_map,
|
||||
n_expert,
|
||||
n_local_expert,
|
||||
topk,
|
||||
permuted_hidden_states,
|
||||
expert_first_token_offset,
|
||||
inv_permuted_idx,
|
||||
permuted_idx,
|
||||
)
|
||||
else:
|
||||
scratch.validate(hidden_states, topk_ids)
|
||||
assert n_expert == scratch.num_experts
|
||||
assert n_local_expert == scratch.num_local_experts
|
||||
token_expert_indices = scratch.token_expert_indices_view(n_token)
|
||||
expert_first_token_offset = scratch.expert_first_token_offset
|
||||
permuted_idx = scratch.permuted_idx[:permuted_row_size]
|
||||
permuted_idx.fill_(permuted_row_size)
|
||||
inv_permuted_idx = scratch.inv_permuted_idx[:permuted_row_size].view(
|
||||
n_token, topk
|
||||
)
|
||||
permuted_experts_id = scratch.permuted_experts_id[:permuted_row_size].view(
|
||||
n_token, topk
|
||||
)
|
||||
sorted_row_idx = scratch.sorted_row_idx[:permuted_row_size].view(n_token, topk)
|
||||
topk_ids_for_sort = scratch.topk_ids_for_sort[:permuted_row_size].view(
|
||||
n_token, topk
|
||||
)
|
||||
topk_ids_int32 = scratch.prepare_topk_ids(topk_ids)
|
||||
torch.ops._moe_C.moe_permute_with_scratch(
|
||||
hidden_states,
|
||||
topk_ids_int32,
|
||||
token_expert_indices,
|
||||
expert_map,
|
||||
n_expert,
|
||||
n_local_expert,
|
||||
topk,
|
||||
permuted_hidden_states,
|
||||
expert_first_token_offset,
|
||||
inv_permuted_idx,
|
||||
permuted_idx,
|
||||
scratch.sort_workspace,
|
||||
permuted_experts_id,
|
||||
sorted_row_idx,
|
||||
topk_ids_for_sort,
|
||||
)
|
||||
|
||||
if a1q_scale is not None and a1q_scale.dim() > 1:
|
||||
a1q_scale = a1q_scale[permuted_idx.clamp(max=n_token * topk - 1) // topk]
|
||||
|
||||
Reference in New Issue
Block a user