[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tyler Michael Smith
parent
67e5b0acc9
commit
ac2f0ea82c
@@ -137,6 +137,7 @@ def FusedMoEFactory(
|
||||
runner_args: dict[str, Any] | None = None,
|
||||
routed_experts_cls: type[RoutedExperts] | None = None,
|
||||
routed_experts_args: dict[str, Any] | None = None,
|
||||
skip_padding: bool = False,
|
||||
) -> MoERunner:
|
||||
"""Factory function for creating MoE execution pipeline.
|
||||
|
||||
@@ -213,6 +214,7 @@ def FusedMoEFactory(
|
||||
runner_args: Additional arguments for runner constructor
|
||||
routed_experts_cls: Custom RoutedExperts class (None = use default)
|
||||
routed_experts_args: Additional arguments for routed_experts constructor
|
||||
skip_padding: Whether grouped routing should invalidate padding rows.
|
||||
|
||||
Returns:
|
||||
MoERunner: Configured MoE execution pipeline ready for forward passes
|
||||
@@ -308,6 +310,7 @@ def FusedMoEFactory(
|
||||
else 1.0,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
skip_padding=(skip_padding and moe_parallel_config.use_deepep_v2_kernels),
|
||||
# Fused shared-expert slot weight. With apply_routed_scale_to_output
|
||||
# the runner scales the combined output by routed_scaling_factor, so
|
||||
# the shared slot weight must be 1/routed_scaling_factor for its net
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm import envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
@@ -258,6 +259,7 @@ class GroupedTopKRouter(BaseRouter):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
skip_padding: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
@@ -271,6 +273,7 @@ class GroupedTopKRouter(BaseRouter):
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
self.skip_padding = skip_padding
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
@@ -346,4 +349,15 @@ class GroupedTopKRouter(BaseRouter):
|
||||
e_score_correction_bias=self.e_score_correction_bias,
|
||||
)
|
||||
|
||||
if (
|
||||
self.skip_padding
|
||||
and envs.VLLM_MOE_SKIP_PADDING
|
||||
and is_forward_context_available()
|
||||
):
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
is_padding = is_padding[: topk_ids.shape[0]].unsqueeze(1)
|
||||
topk_weights = topk_weights.masked_fill(is_padding, 0)
|
||||
topk_ids = topk_ids.masked_fill(is_padding, -1)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -65,6 +65,7 @@ def create_fused_moe_router(
|
||||
# Deepseek V4 vision routing bias parameters
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
skip_padding: bool = False,
|
||||
) -> FusedMoERouter:
|
||||
"""Factory function to create the appropriate FusedMoERouter subclass based on
|
||||
the provided parameters.
|
||||
@@ -91,6 +92,7 @@ def create_fused_moe_router(
|
||||
topk_group: Top-k within each group (for grouped routing)
|
||||
scoring_func: Scoring function to use ("softmax" or "sigmoid")
|
||||
num_fused_shared_experts: Number of fused shared experts (for ROCm AITER)
|
||||
skip_padding: Whether to invalidate routes for padding rows
|
||||
|
||||
Grouped topk and fused topk bias arguments:
|
||||
routed_scaling_factor: Scaling factor for routed weights
|
||||
@@ -209,6 +211,7 @@ def create_fused_moe_router(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
skip_padding=skip_padding,
|
||||
)
|
||||
# Otherwise fall through to the non-grouped chain below.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user