[Optimization] Skip DP padding tokens in MoE (#46428)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Woosuk Kwon
2026-06-23 14:54:46 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 84f13374b3
commit 0a3e2dbc09
9 changed files with 154 additions and 1 deletions
@@ -10,6 +10,7 @@ from typing import final
import torch
import vllm.envs as envs
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
@@ -1133,6 +1134,22 @@ class FusedMoEKernelModularImpl:
The _prepare method is a wrapper around self.prepare_finalize.prepare
that handles DBO and async.
"""
# Skip cudagraph/DP padding tokens uniformly across all a2a backends:
# forcing padded rows' expert ids to -1 makes every prepare_finalize drop
# them (not dispatched / not computed by the experts). The V2 model runner
# marks them in forward_context.is_padding; it is None for runners that do
# not populate it, leaving topk_ids unchanged.
# Gated by VLLM_MOE_SKIP_PADDING (off by default) because this requires the
# experts kernel to treat topk_id == -1 as a skip sentinel, which not all
# MoE backends support yet.
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
n = topk_ids.shape[0]
# TODO: Properly support DBO (padding lives at the batch tail).
topk_ids = torch.where(is_padding[:n].unsqueeze(1), -1, topk_ids)
if not self.prepare_finalize.supports_async():
# We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize