skip cudagraph/DP padding in topk (#48979)

Signed-off-by: gnovack <novackgm@gmail.com>
This commit is contained in:
gnovack
2026-07-21 15:20:30 -07:00
committed by GitHub
parent 08e5067561
commit 85f638a2b8
10 changed files with 302 additions and 99 deletions
@@ -10,7 +10,6 @@ from typing import final
import torch
import vllm.envs as envs
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
@@ -1194,21 +1193,6 @@ class FusedMoEKernelModularImpl:
The _prepare method is a wrapper around self.prepare_finalize.prepare
that handles DBO and async.
"""
# Skip cudagraph/DP padding tokens uniformly across all a2a backends:
# forcing padded rows' expert ids to -1 makes every prepare_finalize drop
# them (not dispatched / not computed by the experts). The V2 model runner
# marks them in forward_context.is_padding; it is None for runners that do
# not populate it, leaving topk_ids unchanged.
# Gated by VLLM_MOE_SKIP_PADDING (off by default) because this requires the
# experts kernel to treat topk_id == -1 as a skip sentinel, which not all
# MoE backends support yet.
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
n = topk_ids.shape[0]
# TODO: Properly support DBO (padding lives at the batch tail).
topk_ids = torch.where(is_padding[:n].unsqueeze(1), -1, topk_ids)
if not self.prepare_finalize.supports_async():
# We shouldn't be running an a2a kernel that doesn't
@@ -9,6 +9,7 @@ import vllm._custom_ops as ops
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
@@ -20,6 +21,13 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
)
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
return is_padding[:num_tokens] if is_padding is not None else None
return None
def vllm_topk_softmax(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
@@ -35,6 +43,7 @@ def vllm_topk_softmax(
gating_output,
renormalize,
e_score_correction_bias,
is_padding=_get_padding_mask(topk_indices.shape[0]),
)
return topk_weights, topk_indices
@@ -57,6 +66,7 @@ def vllm_topk_sigmoid(
renormalize,
e_score_correction_bias,
routed_scaling_factor,
is_padding=_get_padding_mask(topk_indices.shape[0]),
)
return topk_weights, topk_indices
@@ -144,6 +154,7 @@ def vllm_topk_softplus_sqrt(
e_score_correction_bias,
input_tokens,
hash_indices_table,
is_padding=_get_padding_mask(topk_indices.shape[0]),
)
return topk_weights, topk_indices
@@ -5,8 +5,10 @@ from collections.abc import Callable
import torch
import vllm._custom_ops as ops
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
@@ -14,6 +16,13 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
return is_padding[:num_tokens] if is_padding is not None else None
return None
def vllm_topk_softmax(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
@@ -27,6 +36,7 @@ def vllm_topk_softmax(
token_expert_indices,
gating_output,
renormalize,
is_padding=_get_padding_mask(topk_indices.shape[0]),
)
return topk_weights, topk_indices
@@ -45,6 +55,7 @@ def vllm_topk_sigmoid(
token_expert_indices,
gating_output,
renormalize,
is_padding=_get_padding_mask(topk_indices.shape[0]),
)
return topk_weights, topk_indices