[Optimization] Skip DP padding tokens in MoE (#46428)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
84f13374b3
commit
0a3e2dbc09
@@ -8,6 +8,7 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
@@ -16,6 +17,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
mhc_fused_post_pre_tilelang,
|
||||
@@ -442,6 +444,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
is_padding = is_padding[:num_tokens]
|
||||
|
||||
# EPLB: map logical expert IDs to physical replicas and record load.
|
||||
eplb_state = self.eplb_state
|
||||
@@ -449,6 +456,8 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
assert eplb_state.expert_load_view is not None
|
||||
assert eplb_state.logical_replica_count is not None
|
||||
assert eplb_state.should_record_tensor is not None
|
||||
if is_padding is not None:
|
||||
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
|
||||
topk_ids = eplb_map_to_physical_and_record(
|
||||
topk_ids=topk_ids,
|
||||
expert_load_view=eplb_state.expert_load_view,
|
||||
@@ -465,6 +474,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
symm_buffer.x_sf[:num_tokens],
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
)
|
||||
|
||||
# This method must have been already called during the weight loading phase.
|
||||
|
||||
@@ -19,6 +19,7 @@ def _prepare_megamoe_inputs_kernel(
|
||||
x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
@@ -31,6 +32,7 @@ def _prepare_megamoe_inputs_kernel(
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
@@ -85,12 +87,16 @@ def _prepare_megamoe_inputs_kernel(
|
||||
if k_block_id == 0:
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = topk_offsets < top_k
|
||||
token_is_padding = False
|
||||
if is_padding is not None:
|
||||
token_is_padding = tl.load(is_padding + token_id * is_padding_stride_m)
|
||||
|
||||
ids = tl.load(
|
||||
topk_ids + token_id * topk_ids_stride_m + topk_offsets * topk_ids_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ids = tl.where(token_is_padding, -1, ids)
|
||||
tl.store(
|
||||
topk_idx_out
|
||||
+ token_id * topk_idx_stride_m
|
||||
@@ -106,6 +112,7 @@ def _prepare_megamoe_inputs_kernel(
|
||||
mask=topk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weights = tl.where(token_is_padding, 0.0, weights)
|
||||
tl.store(
|
||||
topk_weights_out
|
||||
+ token_id * topk_weights_out_stride_m
|
||||
@@ -123,6 +130,7 @@ def prepare_megamoe_inputs(
|
||||
x_sf: torch.Tensor,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
@@ -142,12 +150,14 @@ def prepare_megamoe_inputs(
|
||||
block_k = 128
|
||||
grid = (num_tokens, triton.cdiv(hidden_size, block_k))
|
||||
block_topk = triton.next_power_of_2(top_k)
|
||||
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
|
||||
_prepare_megamoe_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x_fp8,
|
||||
x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_states.stride(0),
|
||||
@@ -160,6 +170,7 @@ def prepare_megamoe_inputs(
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
padding_stride_m,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
|
||||
Reference in New Issue
Block a user