[Optimization] Skip DP padding tokens in MoE (#46428)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
84f13374b3
commit
0a3e2dbc09
@@ -27,6 +27,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -847,6 +848,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
num_tokens_after_padding = batch_desc.num_tokens
|
||||
assert num_tokens > 0
|
||||
if envs.VLLM_MOE_SKIP_PADDING:
|
||||
# Mark trailing cudagraph-padding rows so kernels can skip work for
|
||||
# them when supported.
|
||||
self.input_buffers.is_padding[:num_tokens].fill_(False)
|
||||
self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_(
|
||||
True
|
||||
)
|
||||
num_tokens_per_req = scheduler_output.num_scheduled_tokens
|
||||
num_reqs = len(num_tokens_per_req)
|
||||
|
||||
@@ -1001,6 +1009,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_seq_len_np=max_seq_len_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
is_padding=self.input_buffers.is_padding[:num_tokens_after_padding],
|
||||
logits_indices=logits_indices,
|
||||
cu_num_logits=cu_num_logits,
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
@@ -1277,6 +1286,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
batch_descriptor=batch_descriptor,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
|
||||
Reference in New Issue
Block a user