[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
Elvir Crnčević
2026-09-17 19:36:19 +00:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Tyler Michael Smith
parent 67e5b0acc9
commit ac2f0ea82c
12 changed files with 224 additions and 6 deletions
@@ -137,6 +137,7 @@ def FusedMoEFactory(
runner_args: dict[str, Any] | None = None,
routed_experts_cls: type[RoutedExperts] | None = None,
routed_experts_args: dict[str, Any] | None = None,
skip_padding: bool = False,
) -> MoERunner:
"""Factory function for creating MoE execution pipeline.
@@ -213,6 +214,7 @@ def FusedMoEFactory(
runner_args: Additional arguments for runner constructor
routed_experts_cls: Custom RoutedExperts class (None = use default)
routed_experts_args: Additional arguments for routed_experts constructor
skip_padding: Whether grouped routing should invalidate padding rows.
Returns:
MoERunner: Configured MoE execution pipeline ready for forward passes
@@ -308,6 +310,7 @@ def FusedMoEFactory(
else 1.0,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=num_fused_shared_experts,
skip_padding=(skip_padding and moe_parallel_config.use_deepep_v2_kernels),
# Fused shared-expert slot weight. With apply_routed_scale_to_output
# the runner scales the combined output by routed_scaling_factor, so
# the shared slot weight must be 1/routed_scaling_factor for its net
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
from vllm import envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
@@ -258,6 +259,7 @@ class GroupedTopKRouter(BaseRouter):
e_score_correction_bias: torch.Tensor | None = None,
num_fused_shared_experts: int = 0,
eplb_state: EplbLayerState | None = None,
skip_padding: bool = False,
):
super().__init__(
top_k=top_k,
@@ -271,6 +273,7 @@ class GroupedTopKRouter(BaseRouter):
self.routed_scaling_factor = routed_scaling_factor
self.e_score_correction_bias = e_score_correction_bias
self.num_fused_shared_experts = num_fused_shared_experts
self.skip_padding = skip_padding
@property
def routing_method_type(self) -> RoutingMethodType:
@@ -346,4 +349,15 @@ class GroupedTopKRouter(BaseRouter):
e_score_correction_bias=self.e_score_correction_bias,
)
if (
self.skip_padding
and envs.VLLM_MOE_SKIP_PADDING
and is_forward_context_available()
):
is_padding = get_forward_context().is_padding
if is_padding is not None:
is_padding = is_padding[: topk_ids.shape[0]].unsqueeze(1)
topk_weights = topk_weights.masked_fill(is_padding, 0)
topk_ids = topk_ids.masked_fill(is_padding, -1)
return topk_weights, topk_ids
@@ -65,6 +65,7 @@ def create_fused_moe_router(
# Deepseek V4 vision routing bias parameters
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
skip_padding: bool = False,
) -> FusedMoERouter:
"""Factory function to create the appropriate FusedMoERouter subclass based on
the provided parameters.
@@ -91,6 +92,7 @@ def create_fused_moe_router(
topk_group: Top-k within each group (for grouped routing)
scoring_func: Scoring function to use ("softmax" or "sigmoid")
num_fused_shared_experts: Number of fused shared experts (for ROCm AITER)
skip_padding: Whether to invalidate routes for padding rows
Grouped topk and fused topk bias arguments:
routed_scaling_factor: Scaling factor for routed weights
@@ -209,6 +211,7 @@ def create_fused_moe_router(
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=num_fused_shared_experts,
skip_padding=skip_padding,
)
# Otherwise fall through to the non-grouped chain below.