[WideEP] Integrate DeepEP v2 (#41183)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tyler Michael Smith
2026-06-08 18:07:29 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 05cb606cad
commit e2f993dc41
22 changed files with 1409 additions and 17 deletions
@@ -29,7 +29,12 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two
FlashInferNVLinkTwoSidedPrepareAndFinalize,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep, has_mori, has_nixl_ep
from vllm.utils.import_utils import (
has_deep_ep,
has_deep_ep_v2,
has_mori,
has_nixl_ep,
)
logger = init_logger(__name__)
@@ -40,6 +45,8 @@ if current_platform.is_cuda_alike():
DEEPEP_QUANT_BLOCK_SHAPE,
DeepEPLLPrepareAndFinalize,
)
if has_deep_ep_v2():
from .prepare_finalize.deepep_v2 import DeepEPV2PrepareAndFinalize
if has_mori():
from .prepare_finalize.mori import MoriPrepareAndFinalize
if has_nixl_ep():
@@ -94,6 +101,11 @@ def maybe_roundup_layer_hidden_size(
hidden_size
)
if moe_parallel_config.use_deepep_v2_kernels:
hidden_size = DeepEPV2PrepareAndFinalize.maybe_roundup_layer_hidden_size(
hidden_size, act_dtype
)
if moe_parallel_config.use_nixl_ep_kernels:
hidden_size = NixlEPPrepareAndFinalize.maybe_roundup_layer_hidden_size(
hidden_size
@@ -193,6 +205,36 @@ def maybe_make_prepare_finalize(
physical_to_global=physical_to_global,
local_expert_global_ids=local_expert_global_ids,
)
elif moe.use_deepep_v2_kernels:
assert moe.dp_size == all2all_manager.dp_world_size
use_fp8_dispatch = (
quant_config is not None
and quant_config.quant_dtype == current_platform.fp8_dtype()
and quant_config.is_block_quantized
)
all_to_all_args = dict(
num_max_tokens_per_rank=moe.max_num_tokens,
hidden=moe.hidden_dim,
num_topk=moe.experts_per_token,
num_experts=moe.num_experts,
use_fp8_dispatch=use_fp8_dispatch,
)
handle = all2all_manager.get_handle(all_to_all_args)
vllm_config = get_current_vllm_config()
use_cudagraph = not vllm_config.model_config.enforce_eager
prepare_finalize = DeepEPV2PrepareAndFinalize(
buffer=handle,
num_dispatchers=all2all_manager.world_size,
dp_size=all2all_manager.dp_world_size,
rank_expert_offset=all2all_manager.rank * moe.num_local_experts,
num_experts=moe.num_experts,
num_topk=moe.experts_per_token,
use_fp8_dispatch=use_fp8_dispatch,
use_cudagraph=use_cudagraph,
)
elif moe.use_mori_kernels:
assert quant_config is not None
@@ -1070,6 +1070,10 @@ class FusedMoEParallelConfig:
def use_nixl_ep_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "nixl_ep"
@property
def use_deepep_v2_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
@staticmethod
def flatten_tp_across_dp_and_pcp(
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
@@ -1394,6 +1398,10 @@ class FusedMoEConfig:
def use_nixl_ep_kernels(self):
return self.moe_parallel_config.use_nixl_ep_kernels
@property
def use_deepep_v2_kernels(self):
return self.moe_parallel_config.use_deepep_v2_kernels
@property
def needs_round_robin_routing_tables(self):
return self.moe_parallel_config.needs_round_robin_routing_tables
@@ -94,6 +94,14 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
Fp8 TRTLLM-Gen MoE kernels. Supports modular interface.
"""
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
or moe_parallel_config.use_ag_rs_all2all_kernels
or moe_parallel_config.use_deepep_v2_kernels
) and not moe_parallel_config.enable_eplb
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
@@ -193,7 +201,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
gemm2_weights=w2,
gemm2_weights_scale=self.quant_config.w2_scale,
num_experts=global_num_experts,
top_k=self.topk,
top_k=topk_ids.size(1),
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
@@ -85,6 +85,18 @@ def _get_priority_backends(
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
# With DeepEP v2 contiguous layout (do_expand=False), tensors are
# worst-case allocated with padding. TrtLLM's tile-level skipping
# avoids wasted compute on padding rows; other backends process all rows.
if (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and moe_config.moe_parallel_config.use_deepep_v2_kernels
and activation_key == kFp8Dynamic128Sym
and weight_key == kFp8Static128BlockSym
):
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.FLASHINFER_TRTLLM)
# On Hopper for Block Fp8, prefer Triton for TP and FI CUTLASS for EP.
if (
current_platform.is_cuda()
@@ -0,0 +1,394 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import deep_ep
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceContiguous,
TopKWeightAndReduceDelegate,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.utils.math_utils import round_up
from vllm.v1.worker.ubatching import (
dbo_current_ubatch_id,
)
class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""
Prepare/Finalize using DeepEP v2 ElasticBuffer (unified API).
Supports two modes controlled by the `use_cudagraph` constructor arg:
**Decode mode (use_cudagraph=True):**
- do_expand=False, do_cpu_sync=False
- Tokens returned in original order with recv_topk_idx (global IDs)
- Worst-case tensor allocation; padding rows zeroed via
handle.psum_num_recv_tokens_per_scaleup_rank
- Fully cudagraph-capturable
- Expert kernel sorts internally (expert_tokens_meta=None)
**Prefill mode (use_cudagraph=False):**
- do_expand=True, do_cpu_sync=True
- Per-expert-contiguous layout; exact memory allocation
- Saves GPU memory (no worst-case allocation)
- Not cudagraph-capturable (CPU polling), but prefill doesn't
use cudagraphs anyway
- Provides expert_tokens_meta for efficient batched expert kernels
Both modes use async_with_compute_stream=False (synchronous from
caller's perspective). The ElasticBuffer handles comm internally.
"""
@staticmethod
def maybe_roundup_layer_hidden_size(hidden_size: int, dtype: torch.dtype) -> int:
hidden_size_bytes = hidden_size * dtype.itemsize
xfer_atom_size = 512 # 32 * 16 (size(int4))
if hidden_size_bytes % xfer_atom_size == 0:
return hidden_size
hidden_size_bytes = round_up(hidden_size_bytes, xfer_atom_size)
return hidden_size_bytes // dtype.itemsize
def __init__(
self,
buffer: deep_ep.ElasticBuffer,
num_dispatchers: int,
dp_size: int,
rank_expert_offset: int,
num_experts: int,
num_topk: int,
use_fp8_dispatch: bool = False,
use_cudagraph: bool = False,
):
super().__init__()
self.buffer = buffer
self.num_dispatchers_ = num_dispatchers
self.dp_size = dp_size
self.rank_expert_offset = rank_expert_offset
self.num_experts = num_experts
self.num_topk = num_topk
self.use_fp8_dispatch = use_fp8_dispatch
self.use_cudagraph = use_cudagraph
# DBO microbatching: one handle slot per micro-batch.
self.handles: list[deep_ep.EPHandle | None] = [None, None]
def num_dispatchers(self) -> int:
return self.num_dispatchers_
def output_is_reduced(self) -> bool:
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
def max_num_tokens_per_rank(self) -> int | None:
return None
def topk_indices_dtype(self) -> torch.dtype | None:
return torch.int64
def _do_dispatch(
self,
tokens: torch.Tensor,
token_scales: torch.Tensor | None,
rank_topk_ids: torch.Tensor,
rank_topk_weights: torch.Tensor,
num_experts: int,
a1_scale: torch.Tensor | None,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool,
) -> Callable:
has_scales = token_scales is not None
token_data = tokens
if has_scales:
token_data = (tokens, token_scales)
# Decode: do_expand=False + do_cpu_sync=False (cudagraph-safe)
# Prefill: do_expand=True + do_cpu_sync=True (memory-efficient)
do_expand = not self.use_cudagraph
do_cpu_sync = not self.use_cudagraph
(
recv_x,
recv_topk_idx,
recv_topk_weights,
handle,
event,
) = self.buffer.dispatch(
x=token_data,
topk_idx=rank_topk_ids,
topk_weights=rank_topk_weights,
num_experts=num_experts,
do_expand=do_expand,
do_cpu_sync=do_cpu_sync,
async_with_compute_stream=False,
)
a2a_idx = dbo_current_ubatch_id()
self.handles[a2a_idx] = handle
return lambda: self._receiver(
event,
has_scales,
recv_x,
recv_topk_idx,
num_experts,
handle.num_recv_tokens_per_expert_list,
recv_topk_weights,
handle.psum_num_recv_tokens_per_scaleup_rank,
a1_scale,
quant_config,
defer_input_quant=defer_input_quant,
)
def _receiver(
self,
event: deep_ep.EventOverlap,
has_scales: bool,
recv_x: tuple[torch.Tensor, torch.Tensor] | torch.Tensor,
recv_topk_idx: torch.Tensor | None,
num_experts: int,
recv_expert_num_tokens: list[int],
recv_topk_weights: torch.Tensor | None,
psum_recv_per_rank: torch.Tensor,
a1_scale: torch.Tensor | None,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool,
) -> mk.PrepareResultType:
if event.event is not None:
event.current_stream_wait()
if isinstance(recv_x, tuple):
expert_x, expert_x_scale = recv_x
else:
expert_x, expert_x_scale = recv_x, None
if recv_topk_idx is None:
# do_expand=True (prefill mode): build topk_ids from
# per-expert token counts.
total_tokens = sum(recv_expert_num_tokens)
if total_tokens > 0:
recv_topk_idx = torch.empty(
total_tokens,
dtype=torch.int64,
device=expert_x.device,
)
offset = 0
for i, count in enumerate(recv_expert_num_tokens):
if count > 0:
recv_topk_idx[offset : offset + count].fill_(
i + self.rank_expert_offset
)
offset += count
else:
recv_topk_idx = torch.empty(
0,
dtype=torch.int64,
device=expert_x.device,
)
recv_topk_idx = recv_topk_idx.unsqueeze(1)
else:
# do_expand=False (decode/cudagraph mode): recv_topk_idx has
# LOCAL expert IDs (-1 for non-local and padding rows).
# Convert valid local IDs to global. Rows with -1 are
# skipped by expert kernels (TrtLLM tile-level skipping,
# DeepGemm is_computation_valid), so no need to zero
# hidden states, scales, or weights for padding rows.
valid_mask = recv_topk_idx >= 0
recv_topk_idx = torch.where(
valid_mask,
recv_topk_idx + self.rank_expert_offset,
recv_topk_idx,
)
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
recv_topk_weights = recv_topk_weights.unsqueeze(1)
expert_tokens_meta = mk.ExpertTokensMetadata.make_from_list(
recv_expert_num_tokens,
device=expert_x.device,
)
if not quant_config.is_block_quantized and not defer_input_quant:
expert_x_scale = None
if expert_x.numel() != 0:
expert_x, expert_x_scale = moe_kernel_quantize_input(
expert_x,
a1_scale,
quant_dtype=quant_config.quant_dtype,
per_act_token_quant=False,
block_shape=quant_config.block_shape,
is_scale_swizzled=quant_config.is_scale_swizzled,
)
return (
expert_x,
expert_x_scale,
expert_tokens_meta,
recv_topk_idx,
recv_topk_weights,
)
def supports_async(self) -> bool:
return True
def prepare_async(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.ReceiverType:
if apply_router_weight_on_input:
topk = topk_ids.size(1)
assert topk == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
a1 = a1 * topk_weights.to(a1.dtype)
if quant_config.is_block_quantized and not defer_input_quant:
a1q, a1q_scale = moe_kernel_quantize_input(
a1,
quant_config.a1_scale,
quant_dtype=quant_config.quant_dtype,
per_act_token_quant=quant_config.per_act_token_quant,
block_shape=quant_config.block_shape,
)
if a1q_scale is not None and a1q_scale.numel() == 1:
a1q_scale = a1q_scale.view(1, 1)
a1_post_scale = None
else:
a1q = a1
a1q_scale = None
a1_post_scale = (
quant_config.a1_gscale
if quant_config.quant_dtype == "nvfp4"
else quant_config.a1_scale
)
return self._do_dispatch(
tokens=a1q,
token_scales=a1q_scale,
rank_topk_ids=topk_ids,
rank_topk_weights=topk_weights,
num_experts=num_experts,
a1_scale=a1_post_scale,
quant_config=quant_config,
defer_input_quant=defer_input_quant,
)
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
receiver = self.prepare_async(
a1,
topk_weights,
topk_ids,
num_experts,
expert_map,
apply_router_weight_on_input,
quant_config,
defer_input_quant,
)
return receiver()
def _finalize(
self,
output: torch.Tensor,
fused_expert_output: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
do_async: bool,
) -> Callable | None:
a2a_idx = dbo_current_ubatch_id()
handle = self.handles[a2a_idx]
assert handle is not None
if fused_expert_output.numel() != 0:
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate):
weight_and_reduce_impl = TopKWeightAndReduceContiguous()
fused_expert_output = weight_and_reduce_impl.apply(
output=None,
fused_expert_output=fused_expert_output,
topk_weights=topk_weights,
topk_ids=topk_ids,
apply_router_weight_on_input=apply_router_weight_on_input,
)
if fused_expert_output.dtype != torch.bfloat16:
raise ValueError(
f"DeepEP v2 combine requires bfloat16 input, "
f"got {fused_expert_output.dtype}"
)
combined_x, _, event = self.buffer.combine(
x=fused_expert_output,
handle=handle,
topk_weights=None,
async_with_compute_stream=False,
)
output.copy_(combined_x, non_blocking=True)
return None
def finalize_async(
self,
output: torch.Tensor,
fused_expert_output: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> Callable:
self._finalize(
output,
fused_expert_output,
topk_weights,
topk_ids,
apply_router_weight_on_input,
weight_and_reduce_impl,
False,
)
return lambda: None
def finalize(
self,
output: torch.Tensor,
fused_expert_output: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> None:
self._finalize(
output,
fused_expert_output,
topk_weights,
topk_ids,
apply_router_weight_on_input,
weight_and_reduce_impl,
False,
)
@@ -1038,6 +1038,7 @@ class RoutedExperts(PluggableLayer):
"e_score_correction_bias",
"w13_input_scale",
"w2_input_scale",
"hash_indices_table",
}
# Parameters of non-expert submodules that live inside runner (RoutedExperts).
@@ -168,6 +168,14 @@ def fused_topk_bias(
hash_indices_table: torch.Tensor | None = None,
routed_scaling_factor: float = 1.0,
):
# The topk kernel dispatches dtype based on topk_ids (set by
# indices_type) and assumes input_tokens/hash_indices_table match.
if indices_type is not None:
if input_tokens is not None and input_tokens.dtype != indices_type:
input_tokens = input_tokens.to(dtype=indices_type)
if hash_indices_table is not None and hash_indices_table.dtype != indices_type:
hash_indices_table = hash_indices_table.to(dtype=indices_type)
if not rocm_aiter_ops.is_fused_moe_enabled():
assert hidden_states.size(0) == gating_output.size(0), (
"Number of tokens mismatch"
@@ -489,6 +489,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self.w13_precision_config = None
self.w2_precision_config = None
@property
def supports_eplb(self) -> bool:
return True
@property
def skip_forward_padding(self) -> bool:
# SM100_FI_MXFP4_MXFP8_TRTLLM supports padding with mxfp8 quant