[MoE Refactor] Remove MoE DP chunking (#39107)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -6,8 +6,7 @@ from typing import Union
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config import ParallelConfig, SchedulerConfig
|
||||
from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -937,15 +936,6 @@ class FusedMoEParallelConfig:
|
||||
all2all_backend: str # all2all backend for MoE communication
|
||||
enable_eplb: bool # whether to enable expert load balancing
|
||||
|
||||
@property
|
||||
def use_dp_chunking(self) -> bool:
|
||||
return (
|
||||
self.use_deepep_ll_kernels
|
||||
or self.use_mori_kernels
|
||||
or self.use_fi_nvl_two_sided_kernels
|
||||
or self.use_nixl_ep_kernels
|
||||
) and envs.VLLM_ENABLE_MOE_DP_CHUNK
|
||||
|
||||
@property
|
||||
def is_sequence_parallel(self) -> bool:
|
||||
return self.sp_size > 1
|
||||
@@ -1184,7 +1174,7 @@ class FusedMoEConfig:
|
||||
intermediate_size_per_partition_unpadded: int | None = None
|
||||
|
||||
moe_backend: str = "auto"
|
||||
max_num_tokens: int = envs.VLLM_MOE_DP_CHUNK_SIZE
|
||||
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
|
||||
has_bias: bool = False
|
||||
is_act_and_mul: bool = True
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
@@ -8,7 +8,6 @@ from typing import Literal, cast, get_args, overload
|
||||
import torch
|
||||
from torch.nn.parameter import UninitializedParameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.parallel import ExpertPlacementStrategy
|
||||
@@ -479,7 +478,7 @@ class FusedMoE(CustomOp):
|
||||
in_dtype=moe_in_dtype,
|
||||
moe_backend=vllm_config.kernel_config.moe_backend,
|
||||
router_logits_dtype=router_logits_dtype,
|
||||
max_num_tokens=envs.VLLM_MOE_DP_CHUNK_SIZE,
|
||||
max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
has_bias=has_bias,
|
||||
is_act_and_mul=is_act_and_mul,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
|
||||
@@ -1,243 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import (
|
||||
get_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner_base import MoERunnerBase
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
|
||||
class ChunkingMoERunner(MoERunnerBase):
|
||||
"""
|
||||
MoE runner wrapper that adds chunked processing to any MoERunnerBase.
|
||||
|
||||
This runner wraps an inner MoERunnerBase and overrides _forward_impl to
|
||||
process large batches by breaking them into smaller chunks. Each chunk
|
||||
is delegated to the inner runner's _forward_impl, making chunking
|
||||
composable with any runner implementation.
|
||||
|
||||
All MoERunnerBase state (moe_config, router, quant_method, etc.) is
|
||||
transparently delegated to the inner runner via __getattr__.
|
||||
ChunkingMoERunner only owns chunking-specific state: the pre-allocated
|
||||
workspace buffers and the reduce_results override.
|
||||
|
||||
Key behaviors:
|
||||
- Pre-allocates workspace tensors for CUDA graph compatibility
|
||||
- Processes chunks via inner._forward_impl per chunk
|
||||
- Never reduces results (reduce_results always returns False)
|
||||
"""
|
||||
|
||||
def __init__(self, inner: MoERunnerBase):
|
||||
# Assert that _maybe_dispatch/_maybe_combine will be nops.
|
||||
assert inner.moe_config.pcp_size == 1
|
||||
|
||||
# Skip MoERunnerBase.__init__ — all state is delegated to inner
|
||||
# via __getattr__. Only chunking-specific state lives here.
|
||||
self._inner = inner
|
||||
|
||||
# Pre-allocated staging buffers. These need to exist ahead of time
|
||||
# due to CUDA graph construction needing fixed buffer addresses.
|
||||
self.batched_hidden_states, self.batched_router_logits = (
|
||||
self._init_dp_chunking()
|
||||
)
|
||||
|
||||
def __getattr__(self, name):
|
||||
# Delegate attribute access to the inner runner. This is only
|
||||
# called when normal lookup (instance __dict__, class MRO) fails,
|
||||
# so ChunkingMoERunner's own attributes and methods take priority.
|
||||
return getattr(self._inner, name)
|
||||
|
||||
@property
|
||||
def shared_experts(self) -> SharedExperts | None:
|
||||
return self._inner.shared_experts
|
||||
|
||||
# TODO(bnell): temporary hack, do not call this method.
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
self._inner._replace_quant_method(quant_method)
|
||||
self.quant_method = quant_method
|
||||
|
||||
def is_internal_router(self) -> bool:
|
||||
return self._inner.gate is not None
|
||||
|
||||
# Reducing results when chunking is handled by the MK finalize operations
|
||||
# when DP chunking is enabled..
|
||||
# This will be removed by #35949
|
||||
@property
|
||||
def reduce_results(self) -> bool:
|
||||
return False
|
||||
|
||||
def _init_dp_chunking(self) -> list[torch.Tensor]:
|
||||
states_shape: tuple[int, ...]
|
||||
logits_shape: tuple[int, ...]
|
||||
|
||||
moe = self.moe_config
|
||||
|
||||
if self.enable_dbo:
|
||||
states_shape = (2, moe.max_num_tokens, self.moe_config.hidden_dim)
|
||||
logits_shape = (2, moe.max_num_tokens, self.moe_config.num_logical_experts)
|
||||
else:
|
||||
states_shape = (moe.max_num_tokens, self.moe_config.hidden_dim)
|
||||
logits_shape = (moe.max_num_tokens, self.moe_config.num_logical_experts)
|
||||
|
||||
# Does this need some kind of profiling run check like modular_kernel.py?
|
||||
return current_workspace_manager().get_simultaneous(
|
||||
(states_shape, moe.in_dtype),
|
||||
(logits_shape, moe.router_logits_dtype),
|
||||
)
|
||||
|
||||
def _allocate_dp_chunking_outputs(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
# Assert the inputs are of the proper type and shape.
|
||||
assert self.batched_hidden_states is not None
|
||||
assert self.batched_router_logits is not None
|
||||
|
||||
assert self.batched_hidden_states.dtype == hidden_states.dtype, (
|
||||
f"{self.batched_hidden_states.dtype} == {hidden_states.dtype}"
|
||||
)
|
||||
assert self.batched_router_logits.dtype == router_logits.dtype, (
|
||||
f"{self.batched_router_logits.dtype} == {router_logits.dtype}"
|
||||
)
|
||||
|
||||
# Check size compatibility.
|
||||
assert self.batched_hidden_states.size(-1) == hidden_states.size(-1)
|
||||
assert self.batched_router_logits.size(-1) == router_logits.size(-1)
|
||||
|
||||
final_fused_hidden_states = torch.empty_like(hidden_states)
|
||||
if self.shared_experts is not None:
|
||||
if shared_experts_input is not None:
|
||||
final_shared_hidden_states = torch.empty_like(shared_experts_input)
|
||||
else:
|
||||
final_shared_hidden_states = torch.empty_like(hidden_states)
|
||||
else:
|
||||
final_shared_hidden_states = None
|
||||
|
||||
return final_shared_hidden_states, final_fused_hidden_states
|
||||
|
||||
def _slice_and_copy_input(
|
||||
self,
|
||||
out_slice: torch.Tensor,
|
||||
orig: torch.Tensor | None,
|
||||
start: int,
|
||||
end: int,
|
||||
) -> torch.Tensor:
|
||||
assert orig is not None
|
||||
slice_size = end - start
|
||||
orig_slice = orig[start:end, :]
|
||||
if self.enable_dbo:
|
||||
assert out_slice.dim() == 3
|
||||
batch_buffer_idx = dbo_current_ubatch_id()
|
||||
out_slice = out_slice[batch_buffer_idx, :]
|
||||
|
||||
assert out_slice.size(0) >= slice_size
|
||||
out_slice = out_slice[:slice_size, :]
|
||||
out_slice.copy_(orig_slice, non_blocking=True)
|
||||
return out_slice
|
||||
|
||||
def _forward_impl(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
final_shared_hidden_states, final_fused_hidden_states = (
|
||||
self._allocate_dp_chunking_outputs(
|
||||
hidden_states, router_logits, shared_experts_input
|
||||
)
|
||||
)
|
||||
|
||||
ctx = get_forward_context()
|
||||
# flashinfer_cutlass_kernels can handle: optional DP + TP/EP
|
||||
max_tokens_across_dispatchers = ctx.dp_metadata.max_tokens_across_dp_cpu
|
||||
moe_dp_chunk_size_per_rank = self.moe_config.max_num_tokens
|
||||
|
||||
# If the input to the MoE is sequence parallel then divide by sp_size
|
||||
# to find the maximum number of tokens for any individual dispatcher.
|
||||
if self.moe_config.is_sequence_parallel:
|
||||
max_tokens_across_dispatchers = cdiv(
|
||||
max_tokens_across_dispatchers, self.moe_config.sp_size
|
||||
)
|
||||
|
||||
num_tokens = hidden_states.size(0)
|
||||
for chunk_idx, chunk_start_ in enumerate(
|
||||
range(0, max_tokens_across_dispatchers, moe_dp_chunk_size_per_rank)
|
||||
):
|
||||
chunk_start = chunk_start_
|
||||
chunk_end = min(
|
||||
chunk_start + moe_dp_chunk_size_per_rank, max_tokens_across_dispatchers
|
||||
)
|
||||
# clamp start and end
|
||||
chunk_start = min(chunk_start, num_tokens - 1)
|
||||
chunk_end = min(chunk_end, num_tokens)
|
||||
chunk_sizes = ctx.dp_metadata.chunked_sizes(
|
||||
self.moe_config.sp_size, moe_dp_chunk_size_per_rank, chunk_idx
|
||||
)
|
||||
with chunk_sizes:
|
||||
hidden_states_chunk = self._slice_and_copy_input(
|
||||
self.batched_hidden_states,
|
||||
hidden_states,
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
)
|
||||
|
||||
router_logits_chunk = self._slice_and_copy_input(
|
||||
self.batched_router_logits,
|
||||
router_logits,
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
)
|
||||
|
||||
shared_experts_input_chunk = (
|
||||
shared_experts_input[chunk_start:chunk_end, :]
|
||||
if shared_experts_input is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Delegate per-chunk computation to the inner runner.
|
||||
chunk_result = self._inner._forward_impl(
|
||||
layer=layer,
|
||||
hidden_states=hidden_states_chunk,
|
||||
router_logits=router_logits_chunk,
|
||||
shared_experts_input=shared_experts_input_chunk,
|
||||
)
|
||||
|
||||
# Store outputs
|
||||
# TODO(bnell): document when chunk_start >= num_tokens
|
||||
if chunk_start < num_tokens:
|
||||
if self.shared_experts is not None:
|
||||
assert isinstance(chunk_result, tuple)
|
||||
shared_output_chunk, hidden_states_chunk = chunk_result
|
||||
final_fused_hidden_states[chunk_start:chunk_end, :].copy_(
|
||||
hidden_states_chunk, non_blocking=True
|
||||
)
|
||||
assert shared_output_chunk is not None
|
||||
assert final_shared_hidden_states is not None
|
||||
final_shared_hidden_states[chunk_start:chunk_end, :].copy_(
|
||||
shared_output_chunk, non_blocking=True
|
||||
)
|
||||
else:
|
||||
assert isinstance(chunk_result, torch.Tensor)
|
||||
final_fused_hidden_states[chunk_start:chunk_end, :].copy_(
|
||||
chunk_result, non_blocking=True
|
||||
)
|
||||
|
||||
if self.shared_experts is None:
|
||||
return final_fused_hidden_states
|
||||
else:
|
||||
assert final_shared_hidden_states is not None
|
||||
return (final_shared_hidden_states, final_fused_hidden_states)
|
||||
@@ -12,9 +12,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
|
||||
FusedMoERouter,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.chunking_moe_runner import (
|
||||
ChunkingMoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.default_moe_runner import (
|
||||
DefaultMoERunner,
|
||||
)
|
||||
@@ -35,7 +32,7 @@ def create_moe_runner(
|
||||
reduce_results: bool,
|
||||
enable_dbo: bool,
|
||||
) -> MoERunner:
|
||||
runner = DefaultMoERunner(
|
||||
return DefaultMoERunner(
|
||||
layer_name,
|
||||
moe_config,
|
||||
router,
|
||||
@@ -46,6 +43,3 @@ def create_moe_runner(
|
||||
reduce_results,
|
||||
enable_dbo,
|
||||
)
|
||||
if moe_config.moe_parallel_config.use_dp_chunking:
|
||||
return ChunkingMoERunner(runner)
|
||||
return runner
|
||||
|
||||
@@ -69,7 +69,6 @@ class SharedExperts:
|
||||
self._moe_config = moe_config
|
||||
self._quant_method = quant_method
|
||||
self._reduce_results = reduce_results
|
||||
self._use_dp_chunking = moe_config.moe_parallel_config.use_dp_chunking
|
||||
|
||||
# Allow disabling of the separate shared experts stream for
|
||||
# debug purposes.
|
||||
@@ -87,20 +86,6 @@ class SharedExperts:
|
||||
"Enabled separate cuda stream for MoE shared_experts", scope="local"
|
||||
)
|
||||
|
||||
@property
|
||||
def _use_external_experts(self) -> bool:
|
||||
if self._use_dp_chunking:
|
||||
return False
|
||||
|
||||
# Disable shared expert overlap if:
|
||||
# - we are using eplb with non-default backend, because of correctness issues
|
||||
# - we are using flashinfer with DP, since there nothing to gain
|
||||
backend = self._moe_config.moe_parallel_config.all2all_backend
|
||||
return (
|
||||
self._moe_config.moe_parallel_config.enable_eplb
|
||||
and backend != "allgather_reducescatter"
|
||||
) or self._moe_config.moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
|
||||
def _determine_shared_experts_order(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -110,7 +95,6 @@ class SharedExperts:
|
||||
|
||||
should_run_shared_in_aux_stream = (
|
||||
current_platform.is_cuda()
|
||||
and not self._use_dp_chunking
|
||||
and self._stream is not None
|
||||
and hidden_states.shape[0]
|
||||
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
||||
|
||||
@@ -1502,9 +1502,9 @@ class QuarkOCP_MX_MoEMethod_OSS(QuarkOCP_MX_MoEMethod):
|
||||
layer.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
|
||||
|
||||
# FIXME warp need to be adjusted based on batch size
|
||||
# only apply to batched mode
|
||||
# only apply to batched mode
|
||||
if self.moe.use_ep:
|
||||
num_warps = 4 if envs.VLLM_MOE_DP_CHUNK_SIZE <= 512 else 8
|
||||
num_warps = 4 if self.moe.max_num_tokens <= 512 else 8
|
||||
else:
|
||||
num_warps = 8
|
||||
|
||||
|
||||
Reference in New Issue
Block a user