[Optimization] Skip DP padding tokens in MoE (#46428)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
84f13374b3
commit
0a3e2dbc09
@@ -10,6 +10,7 @@ from typing import final
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
@@ -1133,6 +1134,22 @@ class FusedMoEKernelModularImpl:
|
||||
The _prepare method is a wrapper around self.prepare_finalize.prepare
|
||||
that handles DBO and async.
|
||||
"""
|
||||
# Skip cudagraph/DP padding tokens uniformly across all a2a backends:
|
||||
# forcing padded rows' expert ids to -1 makes every prepare_finalize drop
|
||||
# them (not dispatched / not computed by the experts). The V2 model runner
|
||||
# marks them in forward_context.is_padding; it is None for runners that do
|
||||
# not populate it, leaving topk_ids unchanged.
|
||||
# Gated by VLLM_MOE_SKIP_PADDING (off by default) because this requires the
|
||||
# experts kernel to treat topk_id == -1 as a skip sentinel, which not all
|
||||
# MoE backends support yet.
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
n = topk_ids.shape[0]
|
||||
# TODO: Properly support DBO (padding lives at the batch tail).
|
||||
topk_ids = torch.where(is_padding[:n].unsqueeze(1), -1, topk_ids)
|
||||
|
||||
if not self.prepare_finalize.supports_async():
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
# support async prepare/finalize
|
||||
|
||||
Reference in New Issue
Block a user