[Optimization] Skip DP padding tokens in MoE (#46428)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Woosuk Kwon
2026-06-23 14:54:46 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 84f13374b3
commit 0a3e2dbc09
9 changed files with 154 additions and 1 deletions
+80 -1
View File
@@ -46,7 +46,8 @@ def test_deepseek_v4_mega_moe_ue8m0_uint8_to_float():
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership(): def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
vllm_config = SimpleNamespace( vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4) scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
) )
experts = DeepseekV4MegaMoEExperts( experts = DeepseekV4MegaMoEExperts(
vllm_config, vllm_config,
@@ -182,3 +183,81 @@ def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact():
fused_topk_weights.view(torch.uint8), fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8), ref_topk_weights.view(torch.uint8),
) )
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding():
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
num_tokens = 7
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(1)
hidden_states = torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.bfloat16,
generator=generator,
)
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
is_padding = torch.tensor(
[False, True, False, False, True, False, True],
device=device,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_idx[is_padding] = -1
ref_topk_weights = topk_weights.clone()
ref_topk_weights[is_padding] = 0.0
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
is_padding=is_padding,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
+6
View File
@@ -185,6 +185,7 @@ if TYPE_CHECKING:
"relax", "relax",
] = "relax" ] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_MOE_SKIP_PADDING: bool = False
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
@@ -1469,6 +1470,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool( "VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1")) int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
), ),
# Skip cudagraph/DP padding tokens in the MoE path by forcing their expert
# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
# treats topk_id == -1 as a skip sentinel; off by default because not all
# kernels support it yet.
"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "0"))),
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers. # Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper). # This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool( "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
+9
View File
@@ -147,6 +147,11 @@ class ForwardContext:
ubatch_slices: UBatchSlices | None = None ubatch_slices: UBatchSlices | None = None
# Boolean mask over the token axis: True for padding rows that are not real
# tokens. Consumers can use it to skip work for padded tokens. None when
# the producer does not set it.
is_padding: torch.Tensor | None = None
# If True, bypass the compiled model call, e.g. by using .forward() directly # If True, bypass the compiled model call, e.g. by using .forward() directly
skip_compiled: bool = False skip_compiled: bool = False
@@ -211,6 +216,7 @@ def create_forward_context(
slot_mapping: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None, slot_mapping: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None,
additional_kwargs: dict[str, Any] | None = None, additional_kwargs: dict[str, Any] | None = None,
skip_compiled: bool = False, skip_compiled: bool = False,
is_padding: torch.Tensor | None = None,
): ):
if vllm_config.compilation_config.fast_moe_cold_start: if vllm_config.compilation_config.fast_moe_cold_start:
all_moe_layers = vllm_config.compilation_config.static_all_moe_layers all_moe_layers = vllm_config.compilation_config.static_all_moe_layers
@@ -228,6 +234,7 @@ def create_forward_context(
ubatch_slices=ubatch_slices, ubatch_slices=ubatch_slices,
skip_compiled=skip_compiled, skip_compiled=skip_compiled,
additional_kwargs=additional_kwargs or {}, additional_kwargs=additional_kwargs or {},
is_padding=is_padding,
) )
@@ -257,6 +264,7 @@ def set_forward_context(
ubatch_slices: UBatchSlices | None = None, ubatch_slices: UBatchSlices | None = None,
slot_mapping: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None, slot_mapping: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None = None,
skip_compiled: bool = False, skip_compiled: bool = False,
is_padding: torch.Tensor | None = None,
): ):
"""A context manager that stores the current forward context, """A context manager that stores the current forward context,
can be attention metadata, etc. can be attention metadata, etc.
@@ -316,6 +324,7 @@ def set_forward_context(
slot_mapping, slot_mapping,
additional_kwargs, additional_kwargs,
skip_compiled, skip_compiled,
is_padding=is_padding,
) )
try: try:
@@ -10,6 +10,7 @@ from typing import final
import torch import torch
import vllm.envs as envs import vllm.envs as envs
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import ( from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation, MoEActivation,
@@ -1133,6 +1134,22 @@ class FusedMoEKernelModularImpl:
The _prepare method is a wrapper around self.prepare_finalize.prepare The _prepare method is a wrapper around self.prepare_finalize.prepare
that handles DBO and async. that handles DBO and async.
""" """
# Skip cudagraph/DP padding tokens uniformly across all a2a backends:
# forcing padded rows' expert ids to -1 makes every prepare_finalize drop
# them (not dispatched / not computed by the experts). The V2 model runner
# marks them in forward_context.is_padding; it is None for runners that do
# not populate it, leaving topk_ids unchanged.
# Gated by VLLM_MOE_SKIP_PADDING (off by default) because this requires the
# experts kernel to treat topk_id == -1 as a skip sentinel, which not all
# MoE backends support yet.
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
n = topk_ids.shape[0]
# TODO: Properly support DBO (padding lives at the batch tail).
topk_ids = torch.where(is_padding[:n].unsqueeze(1), -1, topk_ids)
if not self.prepare_finalize.supports_async(): if not self.prepare_finalize.supports_async():
# We shouldn't be running an a2a kernel that doesn't # We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize # support async prepare/finalize
+10
View File
@@ -8,6 +8,7 @@ import regex as re
import torch import torch
import torch.nn as nn import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.distributed import ( from vllm.distributed import (
get_ep_group, get_ep_group,
@@ -16,6 +17,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size, get_tensor_model_parallel_world_size,
) )
from vllm.distributed.eplb.eplb_state import EplbLayerState from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.kernels.mhc.tilelang import ( from vllm.model_executor.kernels.mhc.tilelang import (
hc_head_fused_kernel_tilelang, hc_head_fused_kernel_tilelang,
mhc_fused_post_pre_tilelang, mhc_fused_post_pre_tilelang,
@@ -442,6 +444,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
symm_buffer = self.get_symm_buffer() symm_buffer = self.get_symm_buffer()
num_tokens = hidden_states.shape[0] num_tokens = hidden_states.shape[0]
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
is_padding = is_padding[:num_tokens]
# EPLB: map logical expert IDs to physical replicas and record load. # EPLB: map logical expert IDs to physical replicas and record load.
eplb_state = self.eplb_state eplb_state = self.eplb_state
@@ -449,6 +456,8 @@ class DeepseekV4MegaMoEExperts(nn.Module):
assert eplb_state.expert_load_view is not None assert eplb_state.expert_load_view is not None
assert eplb_state.logical_replica_count is not None assert eplb_state.logical_replica_count is not None
assert eplb_state.should_record_tensor is not None assert eplb_state.should_record_tensor is not None
if is_padding is not None:
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
topk_ids = eplb_map_to_physical_and_record( topk_ids = eplb_map_to_physical_and_record(
topk_ids=topk_ids, topk_ids=topk_ids,
expert_load_view=eplb_state.expert_load_view, expert_load_view=eplb_state.expert_load_view,
@@ -465,6 +474,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
symm_buffer.x_sf[:num_tokens], symm_buffer.x_sf[:num_tokens],
symm_buffer.topk_idx[:num_tokens], symm_buffer.topk_idx[:num_tokens],
symm_buffer.topk_weights[:num_tokens], symm_buffer.topk_weights[:num_tokens],
is_padding=is_padding,
) )
# This method must have been already called during the weight loading phase. # This method must have been already called during the weight loading phase.
@@ -19,6 +19,7 @@ def _prepare_megamoe_inputs_kernel(
x_sf, x_sf,
topk_ids, topk_ids,
topk_weights, topk_weights,
is_padding,
topk_idx_out, topk_idx_out,
topk_weights_out, topk_weights_out,
hidden_stride_m: tl.constexpr, hidden_stride_m: tl.constexpr,
@@ -31,6 +32,7 @@ def _prepare_megamoe_inputs_kernel(
topk_ids_stride_k: tl.constexpr, topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr, topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr, topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr, topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr, topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr, topk_weights_out_stride_m: tl.constexpr,
@@ -85,12 +87,16 @@ def _prepare_megamoe_inputs_kernel(
if k_block_id == 0: if k_block_id == 0:
topk_offsets = tl.arange(0, BLOCK_TOPK) topk_offsets = tl.arange(0, BLOCK_TOPK)
topk_mask = topk_offsets < top_k topk_mask = topk_offsets < top_k
token_is_padding = False
if is_padding is not None:
token_is_padding = tl.load(is_padding + token_id * is_padding_stride_m)
ids = tl.load( ids = tl.load(
topk_ids + token_id * topk_ids_stride_m + topk_offsets * topk_ids_stride_k, topk_ids + token_id * topk_ids_stride_m + topk_offsets * topk_ids_stride_k,
mask=topk_mask, mask=topk_mask,
other=0, other=0,
).to(tl.int64) ).to(tl.int64)
ids = tl.where(token_is_padding, -1, ids)
tl.store( tl.store(
topk_idx_out topk_idx_out
+ token_id * topk_idx_stride_m + token_id * topk_idx_stride_m
@@ -106,6 +112,7 @@ def _prepare_megamoe_inputs_kernel(
mask=topk_mask, mask=topk_mask,
other=0.0, other=0.0,
) )
weights = tl.where(token_is_padding, 0.0, weights)
tl.store( tl.store(
topk_weights_out topk_weights_out
+ token_id * topk_weights_out_stride_m + token_id * topk_weights_out_stride_m
@@ -123,6 +130,7 @@ def prepare_megamoe_inputs(
x_sf: torch.Tensor, x_sf: torch.Tensor,
topk_idx_out: torch.Tensor, topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor, topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
) -> None: ) -> None:
num_tokens, hidden_size = hidden_states.shape num_tokens, hidden_size = hidden_states.shape
if num_tokens == 0: if num_tokens == 0:
@@ -142,12 +150,14 @@ def prepare_megamoe_inputs(
block_k = 128 block_k = 128
grid = (num_tokens, triton.cdiv(hidden_size, block_k)) grid = (num_tokens, triton.cdiv(hidden_size, block_k))
block_topk = triton.next_power_of_2(top_k) block_topk = triton.next_power_of_2(top_k)
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
_prepare_megamoe_inputs_kernel[grid]( _prepare_megamoe_inputs_kernel[grid](
hidden_states, hidden_states,
x_fp8, x_fp8,
x_sf, x_sf,
topk_ids, topk_ids,
topk_weights, topk_weights,
is_padding,
topk_idx_out, topk_idx_out,
topk_weights_out, topk_weights_out,
hidden_states.stride(0), hidden_states.stride(0),
@@ -160,6 +170,7 @@ def prepare_megamoe_inputs(
topk_ids.stride(1), topk_ids.stride(1),
topk_weights.stride(0), topk_weights.stride(0),
topk_weights.stride(1), topk_weights.stride(1),
padding_stride_m,
topk_idx_out.stride(0), topk_idx_out.stride(0),
topk_idx_out.stride(1), topk_idx_out.stride(1),
topk_weights_out.stride(0), topk_weights_out.stride(0),
+4
View File
@@ -471,6 +471,9 @@ class ModelCudaGraphManager(CudaGraphManager):
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
) )
# Capture with dummy rows marked as padding.
input_buffers.is_padding.fill_(True)
def forward_fn(cg_mode: CUDAGraphMode) -> None: def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None batch_descriptor = None
if cg_mode == CUDAGraphMode.PIECEWISE: if cg_mode == CUDAGraphMode.PIECEWISE:
@@ -488,6 +491,7 @@ class ModelCudaGraphManager(CudaGraphManager):
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings, slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor, batch_descriptor=batch_descriptor,
is_padding=input_buffers.is_padding[:num_tokens],
): ):
if cg_mode == CUDAGraphMode.PIECEWISE: if cg_mode == CUDAGraphMode.PIECEWISE:
# PIECEWISE graph (compiled PW or breakable, chosen inside # PIECEWISE graph (compiled PW or breakable, chosen inside
+7
View File
@@ -22,6 +22,7 @@ class InputBuffers:
self.input_ids = torch.zeros(max_num_tokens, dtype=torch.int32, device=device) self.input_ids = torch.zeros(max_num_tokens, dtype=torch.int32, device=device)
self.positions = torch.zeros(max_num_tokens, dtype=torch.int64, device=device) self.positions = torch.zeros(max_num_tokens, dtype=torch.int64, device=device)
self.is_padding = torch.zeros(max_num_tokens, dtype=torch.bool, device=device)
self.query_start_loc = torch.zeros( self.query_start_loc = torch.zeros(
max_num_reqs + 1, dtype=torch.int32, device=device max_num_reqs + 1, dtype=torch.int32, device=device
) )
@@ -83,6 +84,8 @@ class InputBatch:
input_ids: torch.Tensor input_ids: torch.Tensor
# [num_tokens_after_padding] # [num_tokens_after_padding]
positions: torch.Tensor positions: torch.Tensor
# [num_tokens_after_padding]
is_padding: torch.Tensor
# [total_num_logits] # [total_num_logits]
logits_indices: torch.Tensor logits_indices: torch.Tensor
@@ -134,6 +137,9 @@ class InputBatch:
input_ids = input_buffers.input_ids[:num_tokens].zero_() input_ids = input_buffers.input_ids[:num_tokens].zero_()
positions = input_buffers.positions[:num_tokens].zero_() positions = input_buffers.positions[:num_tokens].zero_()
input_buffers.is_padding[:num_tokens].fill_(True)
is_padding = input_buffers.is_padding[:num_tokens]
logits_indices = query_start_loc[1:] - 1 logits_indices = query_start_loc[1:] - 1
cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32) cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32)
cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32) cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32)
@@ -164,6 +170,7 @@ class InputBatch:
max_seq_len_np=None, max_seq_len_np=None,
input_ids=input_ids, input_ids=input_ids,
positions=positions, positions=positions,
is_padding=is_padding,
logits_indices=logits_indices, logits_indices=logits_indices,
cu_num_logits=cu_num_logits, cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np, cu_num_logits_np=cu_num_logits_np,
+10
View File
@@ -27,6 +27,7 @@ import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.counter import compilation_counter from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode from vllm.config.compilation import CUDAGraphMode
@@ -847,6 +848,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens = scheduler_output.total_num_scheduled_tokens num_tokens = scheduler_output.total_num_scheduled_tokens
num_tokens_after_padding = batch_desc.num_tokens num_tokens_after_padding = batch_desc.num_tokens
assert num_tokens > 0 assert num_tokens > 0
if envs.VLLM_MOE_SKIP_PADDING:
# Mark trailing cudagraph-padding rows so kernels can skip work for
# them when supported.
self.input_buffers.is_padding[:num_tokens].fill_(False)
self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_(
True
)
num_tokens_per_req = scheduler_output.num_scheduled_tokens num_tokens_per_req = scheduler_output.num_scheduled_tokens
num_reqs = len(num_tokens_per_req) num_reqs = len(num_tokens_per_req)
@@ -1001,6 +1009,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_seq_len_np=max_seq_len_np, max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding],
is_padding=self.input_buffers.is_padding[:num_tokens_after_padding],
logits_indices=logits_indices, logits_indices=logits_indices,
cu_num_logits=cu_num_logits, cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np, cu_num_logits_np=cu_num_logits_np,
@@ -1277,6 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
batch_descriptor=batch_descriptor, batch_descriptor=batch_descriptor,
slot_mapping=slot_mappings_by_layer, slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled, skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
): ):
self.kv_connector.pre_forward(scheduler_output) self.kv_connector.pre_forward(scheduler_output)
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: