[Optimization] Skip DP padding tokens in MoE (#46428)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Woosuk Kwon
2026-06-23 14:54:46 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 84f13374b3
commit 0a3e2dbc09
9 changed files with 154 additions and 1 deletions
+10
View File
@@ -27,6 +27,7 @@ import numpy as np
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
@@ -847,6 +848,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens = scheduler_output.total_num_scheduled_tokens
num_tokens_after_padding = batch_desc.num_tokens
assert num_tokens > 0
if envs.VLLM_MOE_SKIP_PADDING:
# Mark trailing cudagraph-padding rows so kernels can skip work for
# them when supported.
self.input_buffers.is_padding[:num_tokens].fill_(False)
self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_(
True
)
num_tokens_per_req = scheduler_output.num_scheduled_tokens
num_reqs = len(num_tokens_per_req)
@@ -1001,6 +1009,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
is_padding=self.input_buffers.is_padding[:num_tokens_after_padding],
logits_indices=logits_indices,
cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np,
@@ -1277,6 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
batch_descriptor=batch_descriptor,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(scheduler_output)
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: