[Kernel] Fuse DeepEP-v2 decode globalize+align into one launch (#53911)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
This commit is contained in:
Elvir Crnčević
2026-10-13 11:10:38 -04:00
committed by GitHub
parent 669f85e686
commit b9b401b2ef
10 changed files with 686 additions and 16 deletions
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
fused_globalize_align_block_size,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
@@ -710,16 +713,45 @@ class HummingIndexedExperts(HummingExpertsBase):
num_sms = num_compute_units(topk_ids.get_device())
use_scatter = topk_ids.numel() > 2 * num_sms
alignment = moe_align_block_size(
topk_ids=topk_ids,
block_size=moe_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
return_scatter_idx=use_scatter,
# When DeepEP-v2 deferred globalization, fuse globalize + align + sort
# into one launch (topk_ids globalized in place); else align only.
rank_expert_offset = (
expert_tokens_meta.rank_expert_offset
if expert_tokens_meta is not None
else None
)
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
scatter_idx = alignment[3] if len(alignment) == 4 else None
psum = (
expert_tokens_meta.psum_recv_per_rank
if expert_tokens_meta is not None
else None
)
if (
current_platform.is_cuda()
and rank_expert_offset is not None
and psum is not None
):
alignment = fused_globalize_align_block_size(
recv_topk_idx=topk_ids,
psum_recv_per_rank=psum,
rank_expert_offset=rank_expert_offset,
global_num_experts=self.global_num_experts,
local_num_experts=self.num_experts,
block_size=moe_block_size,
return_scatter_idx=use_scatter,
)
_, sorted_ids, expert_ids, num_tokens_padded = alignment[:4]
scatter_idx = alignment[4] if use_scatter else None
else:
alignment = moe_align_block_size(
topk_ids=topk_ids,
block_size=moe_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
return_scatter_idx=use_scatter,
)
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
scatter_idx = alignment[3] if len(alignment) == 4 else None
moe_common_kwargs = {
"sorted_ids": sorted_ids,
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
from vllm.triton_utils import triton
def fused_globalize_align_block_size(
recv_topk_idx: torch.Tensor,
psum_recv_per_rank: torch.Tensor,
rank_expert_offset: int,
global_num_experts: int,
local_num_experts: int,
block_size: int,
return_scatter_idx: bool = False,
) -> tuple[torch.Tensor, ...]:
"""Fuse the DeepEP-v2 INDEXED decode metadata chain into one launch.
Replaces ``_globalize_recv_topk_idx`` + ``moe_align_block_size`` +
``count_and_sort_expert_tokens``. ``recv_topk_idx`` holds LOCAL expert ids
and is globalized in place; valid-token count is read on-device from
``psum_recv_per_rank[-1]`` (CUDA-graph safe).
Returns the globalized ``recv_topk_idx`` and the ``moe_align_block_size``
outputs ``(sorted_ids, expert_ids, num_tokens_post_pad)``. When requested,
the final output is a masked identity scatter index for valid routes.
"""
assert recv_topk_idx.dtype == torch.int64
_, topk = recv_topk_idx.shape
numel = recv_topk_idx.numel()
device = recv_topk_idx.device
max_num_tokens_padded = numel + local_num_experts * (block_size - 1)
if numel < local_num_experts:
max_num_tokens_padded = min(numel * block_size, max_num_tokens_padded)
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=device)
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
scatter_idx = (
torch.empty((numel, 1), dtype=torch.int32, device=device)
if return_scatter_idx
else None
)
ops.fused_globalize_align_block_size(
recv_topk_idx,
psum_recv_per_rank,
rank_expert_offset,
global_num_experts,
local_num_experts,
block_size,
sorted_ids,
expert_ids,
num_tokens_post_pad,
scatter_idx,
)
outputs = (recv_topk_idx, sorted_ids, expert_ids, num_tokens_post_pad)
return (*outputs, scatter_idx) if scatter_idx is not None else outputs
@@ -103,6 +103,9 @@ class ExpertTokensMetadata:
expert_num_tokens: torch.Tensor | None
expert_num_tokens_cpu: torch.Tensor | None
psum_recv_per_rank: torch.Tensor | None = None
# local->global expert-id offset, set when globalization is deferred to the
# experts' fused align kernel.
rank_expert_offset: int | None = None
@staticmethod
def make_from_list(
@@ -1048,6 +1051,18 @@ class FusedMoEKernelModularImpl:
):
self.prepare_finalize = prepare_finalize
self.fused_experts = fused_experts
# Experts that globalize recv_topk_idx themselves let a capable prepare
# stage skip its standalone launch.
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
if (
current_platform.is_cuda()
and isinstance(fused_experts, HummingIndexedExperts)
and hasattr(prepare_finalize, "defer_globalize")
):
prepare_finalize.defer_globalize = True
self.shared_experts: SharedExperts | None = None
moe_parallel_config = fused_experts.moe_config.moe_parallel_config
self.moe_parallel_config = moe_parallel_config
@@ -127,6 +127,11 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
self.use_cudagraph = use_cudagraph
self.sp_size = sp_size
# Set by FusedMoEKernelModularImpl when the experts globalize
# recv_topk_idx themselves, letting the decode path skip the standalone
# _globalize_recv_topk_idx launch.
self.defer_globalize = False
# DBO microbatching: one handle slot per micro-batch.
self.handles: list[deep_ep.EPHandle | None] = [None, None]
@@ -275,6 +280,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# during profile_run when CUDA graphs are enabled.
expert_tokens_meta = None
globalize_deferred = False
if recv_topk_idx is None:
# do_expand=True (prefill mode): build topk_ids from
# per-expert token counts.
@@ -307,12 +313,15 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# be treated as real routed tokens by experts that build routing
# over *all* rows (e.g. triton MoE backend's make_routing_data),
# polluting the per-expert token lists and corrupting real tokens.
recv_topk_idx = _globalize_recv_topk_idx(
recv_topk_idx,
psum_recv_per_rank,
self.rank_expert_offset,
self.num_experts,
)
if self.defer_globalize and self.use_cudagraph:
globalize_deferred = True
else:
recv_topk_idx = _globalize_recv_topk_idx(
recv_topk_idx,
psum_recv_per_rank,
self.rank_expert_offset,
self.num_experts,
)
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
@@ -329,6 +338,9 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
expert_num_tokens_cpu=None,
)
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
if globalize_deferred:
# recv_topk_idx still holds LOCAL ids; experts globalize it.
expert_tokens_meta.rank_expert_offset = self.rank_expert_offset
if _quantize_before_dispatch(quant_config, defer_input_quant):
if quant_config.quant_dtype == "mxfp8" and expert_x_scale is not None: