[MoE Refactor] Remove MoE DP chunking (#39107)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-04-14 09:48:05 -04:00
committed by GitHub
co-authored by Robert Shaw
parent f7e62e3d66
commit e1e318af01
20 changed files with 76 additions and 389 deletions
+2 -12
View File
@@ -6,8 +6,7 @@ from typing import Union
import torch
import vllm.envs as envs
from vllm.config import ParallelConfig
from vllm.config import ParallelConfig, SchedulerConfig
from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -937,15 +936,6 @@ class FusedMoEParallelConfig:
all2all_backend: str # all2all backend for MoE communication
enable_eplb: bool # whether to enable expert load balancing
@property
def use_dp_chunking(self) -> bool:
return (
self.use_deepep_ll_kernels
or self.use_mori_kernels
or self.use_fi_nvl_two_sided_kernels
or self.use_nixl_ep_kernels
) and envs.VLLM_ENABLE_MOE_DP_CHUNK
@property
def is_sequence_parallel(self) -> bool:
return self.sp_size > 1
@@ -1184,7 +1174,7 @@ class FusedMoEConfig:
intermediate_size_per_partition_unpadded: int | None = None
moe_backend: str = "auto"
max_num_tokens: int = envs.VLLM_MOE_DP_CHUNK_SIZE
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
has_bias: bool = False
is_act_and_mul: bool = True
is_lora_enabled: bool = False
@@ -8,7 +8,6 @@ from typing import Literal, cast, get_args, overload
import torch
from torch.nn.parameter import UninitializedParameter
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.parallel import ExpertPlacementStrategy
@@ -479,7 +478,7 @@ class FusedMoE(CustomOp):
in_dtype=moe_in_dtype,
moe_backend=vllm_config.kernel_config.moe_backend,
router_logits_dtype=router_logits_dtype,
max_num_tokens=envs.VLLM_MOE_DP_CHUNK_SIZE,
max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
has_bias=has_bias,
is_act_and_mul=is_act_and_mul,
is_lora_enabled=vllm_config.lora_config is not None,
@@ -1,243 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.forward_context import (
get_forward_context,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.runner.moe_runner_base import MoERunnerBase
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from vllm.v1.worker.workspace import current_workspace_manager
class ChunkingMoERunner(MoERunnerBase):
"""
MoE runner wrapper that adds chunked processing to any MoERunnerBase.
This runner wraps an inner MoERunnerBase and overrides _forward_impl to
process large batches by breaking them into smaller chunks. Each chunk
is delegated to the inner runner's _forward_impl, making chunking
composable with any runner implementation.
All MoERunnerBase state (moe_config, router, quant_method, etc.) is
transparently delegated to the inner runner via __getattr__.
ChunkingMoERunner only owns chunking-specific state: the pre-allocated
workspace buffers and the reduce_results override.
Key behaviors:
- Pre-allocates workspace tensors for CUDA graph compatibility
- Processes chunks via inner._forward_impl per chunk
- Never reduces results (reduce_results always returns False)
"""
def __init__(self, inner: MoERunnerBase):
# Assert that _maybe_dispatch/_maybe_combine will be nops.
assert inner.moe_config.pcp_size == 1
# Skip MoERunnerBase.__init__ — all state is delegated to inner
# via __getattr__. Only chunking-specific state lives here.
self._inner = inner
# Pre-allocated staging buffers. These need to exist ahead of time
# due to CUDA graph construction needing fixed buffer addresses.
self.batched_hidden_states, self.batched_router_logits = (
self._init_dp_chunking()
)
def __getattr__(self, name):
# Delegate attribute access to the inner runner. This is only
# called when normal lookup (instance __dict__, class MRO) fails,
# so ChunkingMoERunner's own attributes and methods take priority.
return getattr(self._inner, name)
@property
def shared_experts(self) -> SharedExperts | None:
return self._inner.shared_experts
# TODO(bnell): temporary hack, do not call this method.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self._inner._replace_quant_method(quant_method)
self.quant_method = quant_method
def is_internal_router(self) -> bool:
return self._inner.gate is not None
# Reducing results when chunking is handled by the MK finalize operations
# when DP chunking is enabled..
# This will be removed by #35949
@property
def reduce_results(self) -> bool:
return False
def _init_dp_chunking(self) -> list[torch.Tensor]:
states_shape: tuple[int, ...]
logits_shape: tuple[int, ...]
moe = self.moe_config
if self.enable_dbo:
states_shape = (2, moe.max_num_tokens, self.moe_config.hidden_dim)
logits_shape = (2, moe.max_num_tokens, self.moe_config.num_logical_experts)
else:
states_shape = (moe.max_num_tokens, self.moe_config.hidden_dim)
logits_shape = (moe.max_num_tokens, self.moe_config.num_logical_experts)
# Does this need some kind of profiling run check like modular_kernel.py?
return current_workspace_manager().get_simultaneous(
(states_shape, moe.in_dtype),
(logits_shape, moe.router_logits_dtype),
)
def _allocate_dp_chunking_outputs(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
) -> tuple[torch.Tensor | None, torch.Tensor]:
# Assert the inputs are of the proper type and shape.
assert self.batched_hidden_states is not None
assert self.batched_router_logits is not None
assert self.batched_hidden_states.dtype == hidden_states.dtype, (
f"{self.batched_hidden_states.dtype} == {hidden_states.dtype}"
)
assert self.batched_router_logits.dtype == router_logits.dtype, (
f"{self.batched_router_logits.dtype} == {router_logits.dtype}"
)
# Check size compatibility.
assert self.batched_hidden_states.size(-1) == hidden_states.size(-1)
assert self.batched_router_logits.size(-1) == router_logits.size(-1)
final_fused_hidden_states = torch.empty_like(hidden_states)
if self.shared_experts is not None:
if shared_experts_input is not None:
final_shared_hidden_states = torch.empty_like(shared_experts_input)
else:
final_shared_hidden_states = torch.empty_like(hidden_states)
else:
final_shared_hidden_states = None
return final_shared_hidden_states, final_fused_hidden_states
def _slice_and_copy_input(
self,
out_slice: torch.Tensor,
orig: torch.Tensor | None,
start: int,
end: int,
) -> torch.Tensor:
assert orig is not None
slice_size = end - start
orig_slice = orig[start:end, :]
if self.enable_dbo:
assert out_slice.dim() == 3
batch_buffer_idx = dbo_current_ubatch_id()
out_slice = out_slice[batch_buffer_idx, :]
assert out_slice.size(0) >= slice_size
out_slice = out_slice[:slice_size, :]
out_slice.copy_(orig_slice, non_blocking=True)
return out_slice
def _forward_impl(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
final_shared_hidden_states, final_fused_hidden_states = (
self._allocate_dp_chunking_outputs(
hidden_states, router_logits, shared_experts_input
)
)
ctx = get_forward_context()
# flashinfer_cutlass_kernels can handle: optional DP + TP/EP
max_tokens_across_dispatchers = ctx.dp_metadata.max_tokens_across_dp_cpu
moe_dp_chunk_size_per_rank = self.moe_config.max_num_tokens
# If the input to the MoE is sequence parallel then divide by sp_size
# to find the maximum number of tokens for any individual dispatcher.
if self.moe_config.is_sequence_parallel:
max_tokens_across_dispatchers = cdiv(
max_tokens_across_dispatchers, self.moe_config.sp_size
)
num_tokens = hidden_states.size(0)
for chunk_idx, chunk_start_ in enumerate(
range(0, max_tokens_across_dispatchers, moe_dp_chunk_size_per_rank)
):
chunk_start = chunk_start_
chunk_end = min(
chunk_start + moe_dp_chunk_size_per_rank, max_tokens_across_dispatchers
)
# clamp start and end
chunk_start = min(chunk_start, num_tokens - 1)
chunk_end = min(chunk_end, num_tokens)
chunk_sizes = ctx.dp_metadata.chunked_sizes(
self.moe_config.sp_size, moe_dp_chunk_size_per_rank, chunk_idx
)
with chunk_sizes:
hidden_states_chunk = self._slice_and_copy_input(
self.batched_hidden_states,
hidden_states,
chunk_start,
chunk_end,
)
router_logits_chunk = self._slice_and_copy_input(
self.batched_router_logits,
router_logits,
chunk_start,
chunk_end,
)
shared_experts_input_chunk = (
shared_experts_input[chunk_start:chunk_end, :]
if shared_experts_input is not None
else None
)
# Delegate per-chunk computation to the inner runner.
chunk_result = self._inner._forward_impl(
layer=layer,
hidden_states=hidden_states_chunk,
router_logits=router_logits_chunk,
shared_experts_input=shared_experts_input_chunk,
)
# Store outputs
# TODO(bnell): document when chunk_start >= num_tokens
if chunk_start < num_tokens:
if self.shared_experts is not None:
assert isinstance(chunk_result, tuple)
shared_output_chunk, hidden_states_chunk = chunk_result
final_fused_hidden_states[chunk_start:chunk_end, :].copy_(
hidden_states_chunk, non_blocking=True
)
assert shared_output_chunk is not None
assert final_shared_hidden_states is not None
final_shared_hidden_states[chunk_start:chunk_end, :].copy_(
shared_output_chunk, non_blocking=True
)
else:
assert isinstance(chunk_result, torch.Tensor)
final_fused_hidden_states[chunk_start:chunk_end, :].copy_(
chunk_result, non_blocking=True
)
if self.shared_experts is None:
return final_fused_hidden_states
else:
assert final_shared_hidden_states is not None
return (final_shared_hidden_states, final_fused_hidden_states)
@@ -12,9 +12,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
from vllm.model_executor.layers.fused_moe.runner.chunking_moe_runner import (
ChunkingMoERunner,
)
from vllm.model_executor.layers.fused_moe.runner.default_moe_runner import (
DefaultMoERunner,
)
@@ -35,7 +32,7 @@ def create_moe_runner(
reduce_results: bool,
enable_dbo: bool,
) -> MoERunner:
runner = DefaultMoERunner(
return DefaultMoERunner(
layer_name,
moe_config,
router,
@@ -46,6 +43,3 @@ def create_moe_runner(
reduce_results,
enable_dbo,
)
if moe_config.moe_parallel_config.use_dp_chunking:
return ChunkingMoERunner(runner)
return runner
@@ -69,7 +69,6 @@ class SharedExperts:
self._moe_config = moe_config
self._quant_method = quant_method
self._reduce_results = reduce_results
self._use_dp_chunking = moe_config.moe_parallel_config.use_dp_chunking
# Allow disabling of the separate shared experts stream for
# debug purposes.
@@ -87,20 +86,6 @@ class SharedExperts:
"Enabled separate cuda stream for MoE shared_experts", scope="local"
)
@property
def _use_external_experts(self) -> bool:
if self._use_dp_chunking:
return False
# Disable shared expert overlap if:
# - we are using eplb with non-default backend, because of correctness issues
# - we are using flashinfer with DP, since there nothing to gain
backend = self._moe_config.moe_parallel_config.all2all_backend
return (
self._moe_config.moe_parallel_config.enable_eplb
and backend != "allgather_reducescatter"
) or self._moe_config.moe_parallel_config.use_fi_nvl_two_sided_kernels
def _determine_shared_experts_order(
self,
hidden_states: torch.Tensor,
@@ -110,7 +95,6 @@ class SharedExperts:
should_run_shared_in_aux_stream = (
current_platform.is_cuda()
and not self._use_dp_chunking
and self._stream is not None
and hidden_states.shape[0]
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
@@ -1502,9 +1502,9 @@ class QuarkOCP_MX_MoEMethod_OSS(QuarkOCP_MX_MoEMethod):
layer.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
# FIXME warp need to be adjusted based on batch size
# only apply to batched mode
# only apply to batched mode
if self.moe.use_ep:
num_warps = 4 if envs.VLLM_MOE_DP_CHUNK_SIZE <= 512 else 8
num_warps = 4 if self.moe.max_num_tokens <= 512 else 8
else:
num_warps = 8