[Bugfix][MoE] Size the permute scratch from its inputs (#60447)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
@@ -304,7 +305,6 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
self.ab_strides2 = ab_strides2
|
||||
self.c_strides1 = c_strides1
|
||||
self.c_strides2 = ab_strides1_c_strides2
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
@@ -335,15 +335,13 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
return get_moe_permute_scratch(
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -1296,7 +1294,6 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
self.s_strides2[:, 0] = k
|
||||
|
||||
self.group_size = group_size
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -1357,15 +1354,13 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
return self.out_dtype if self.out_dtype is not None else act_dtype
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
return get_moe_permute_scratch(
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
|
||||
@@ -258,25 +258,14 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _get_permute_scratch(
|
||||
self, topk: int, indices_only: bool = False
|
||||
) -> MoEPermuteScratch | None:
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
return get_moe_permute_scratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=None if indices_only else self.moe_config.hidden_dim,
|
||||
hidden_dtype=None if indices_only else self.moe_config.in_dtype,
|
||||
)
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
@@ -911,14 +900,18 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
)
|
||||
buffers["output"] = output
|
||||
|
||||
scratch = self._get_permute_scratch()
|
||||
if a1q_scale is None:
|
||||
scratch = self._get_permute_scratch(topk_ids.size(1), indices_only=True)
|
||||
assert scratch is not None
|
||||
scatter_metadata = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
expert_offsets, scatter_idx = scatter_metadata
|
||||
inv_perm = scatter_idx.flatten()
|
||||
num_valid_tokens = expert_offsets[-1:] if expert_map is not None else None
|
||||
else:
|
||||
# Prequantized inputs skip w13 quantization, so its buffer is free
|
||||
# to hold the permuted hidden states.
|
||||
permuted_hidden_states = buffers["quanted_gate_up_input"]
|
||||
assert permuted_hidden_states.dtype == hidden_states.dtype
|
||||
hidden_states, a1q_scale, expert_offsets, inv_perm, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=a1q_scale,
|
||||
@@ -926,7 +919,8 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=expert_map,
|
||||
scratch=self._get_permute_scratch(topk_ids.size(1)),
|
||||
permuted_hidden_states=permuted_hidden_states,
|
||||
scratch=scratch,
|
||||
)
|
||||
scatter_idx = None
|
||||
num_valid_tokens = None
|
||||
|
||||
Reference in New Issue
Block a user