skip cudagraph/DP padding in topk (#48979)
Signed-off-by: gnovack <novackgm@gmail.com>
This commit is contained in:
@@ -10,7 +10,6 @@ from typing import final
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
@@ -1194,21 +1193,6 @@ class FusedMoEKernelModularImpl:
|
||||
The _prepare method is a wrapper around self.prepare_finalize.prepare
|
||||
that handles DBO and async.
|
||||
"""
|
||||
# Skip cudagraph/DP padding tokens uniformly across all a2a backends:
|
||||
# forcing padded rows' expert ids to -1 makes every prepare_finalize drop
|
||||
# them (not dispatched / not computed by the experts). The V2 model runner
|
||||
# marks them in forward_context.is_padding; it is None for runners that do
|
||||
# not populate it, leaving topk_ids unchanged.
|
||||
# Gated by VLLM_MOE_SKIP_PADDING (off by default) because this requires the
|
||||
# experts kernel to treat topk_id == -1 as a skip sentinel, which not all
|
||||
# MoE backends support yet.
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
n = topk_ids.shape[0]
|
||||
# TODO: Properly support DBO (padding lives at the batch tail).
|
||||
topk_ids = torch.where(is_padding[:n].unsqueeze(1), -1, topk_ids)
|
||||
|
||||
if not self.prepare_finalize.supports_async():
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
|
||||
@@ -9,6 +9,7 @@ import vllm._custom_ops as ops
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
@@ -20,6 +21,13 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
|
||||
)
|
||||
|
||||
|
||||
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
return is_padding[:num_tokens] if is_padding is not None else None
|
||||
return None
|
||||
|
||||
|
||||
def vllm_topk_softmax(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
@@ -35,6 +43,7 @@ def vllm_topk_softmax(
|
||||
gating_output,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
@@ -57,6 +66,7 @@ def vllm_topk_sigmoid(
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
@@ -144,6 +154,7 @@ def vllm_topk_softplus_sqrt(
|
||||
e_score_correction_bias,
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
|
||||
@@ -5,8 +5,10 @@ from collections.abc import Callable
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
@@ -14,6 +16,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
|
||||
|
||||
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
return is_padding[:num_tokens] if is_padding is not None else None
|
||||
return None
|
||||
|
||||
|
||||
def vllm_topk_softmax(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
@@ -27,6 +36,7 @@ def vllm_topk_softmax(
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
@@ -45,6 +55,7 @@ def vllm_topk_sigmoid(
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
is_padding=_get_padding_mask(topk_indices.shape[0]),
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
|
||||
Reference in New Issue
Block a user