[Kernel] Fuse DeepEP-v2 decode globalize+align into one launch (#53911)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
This commit is contained in:
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
|
||||
fused_globalize_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
@@ -710,16 +713,45 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
num_sms = num_compute_units(topk_ids.get_device())
|
||||
use_scatter = topk_ids.numel() > 2 * num_sms
|
||||
|
||||
alignment = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=moe_block_size,
|
||||
num_experts=self.global_num_experts,
|
||||
expert_map=expert_map,
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=use_scatter,
|
||||
# When DeepEP-v2 deferred globalization, fuse globalize + align + sort
|
||||
# into one launch (topk_ids globalized in place); else align only.
|
||||
rank_expert_offset = (
|
||||
expert_tokens_meta.rank_expert_offset
|
||||
if expert_tokens_meta is not None
|
||||
else None
|
||||
)
|
||||
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
|
||||
scatter_idx = alignment[3] if len(alignment) == 4 else None
|
||||
psum = (
|
||||
expert_tokens_meta.psum_recv_per_rank
|
||||
if expert_tokens_meta is not None
|
||||
else None
|
||||
)
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and rank_expert_offset is not None
|
||||
and psum is not None
|
||||
):
|
||||
alignment = fused_globalize_align_block_size(
|
||||
recv_topk_idx=topk_ids,
|
||||
psum_recv_per_rank=psum,
|
||||
rank_expert_offset=rank_expert_offset,
|
||||
global_num_experts=self.global_num_experts,
|
||||
local_num_experts=self.num_experts,
|
||||
block_size=moe_block_size,
|
||||
return_scatter_idx=use_scatter,
|
||||
)
|
||||
_, sorted_ids, expert_ids, num_tokens_padded = alignment[:4]
|
||||
scatter_idx = alignment[4] if use_scatter else None
|
||||
else:
|
||||
alignment = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=moe_block_size,
|
||||
num_experts=self.global_num_experts,
|
||||
expert_map=expert_map,
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=use_scatter,
|
||||
)
|
||||
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
|
||||
scatter_idx = alignment[3] if len(alignment) == 4 else None
|
||||
|
||||
moe_common_kwargs = {
|
||||
"sorted_ids": sorted_ids,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
|
||||
def fused_globalize_align_block_size(
|
||||
recv_topk_idx: torch.Tensor,
|
||||
psum_recv_per_rank: torch.Tensor,
|
||||
rank_expert_offset: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
block_size: int,
|
||||
return_scatter_idx: bool = False,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Fuse the DeepEP-v2 INDEXED decode metadata chain into one launch.
|
||||
|
||||
Replaces ``_globalize_recv_topk_idx`` + ``moe_align_block_size`` +
|
||||
``count_and_sort_expert_tokens``. ``recv_topk_idx`` holds LOCAL expert ids
|
||||
and is globalized in place; valid-token count is read on-device from
|
||||
``psum_recv_per_rank[-1]`` (CUDA-graph safe).
|
||||
|
||||
Returns the globalized ``recv_topk_idx`` and the ``moe_align_block_size``
|
||||
outputs ``(sorted_ids, expert_ids, num_tokens_post_pad)``. When requested,
|
||||
the final output is a masked identity scatter index for valid routes.
|
||||
"""
|
||||
assert recv_topk_idx.dtype == torch.int64
|
||||
_, topk = recv_topk_idx.shape
|
||||
numel = recv_topk_idx.numel()
|
||||
device = recv_topk_idx.device
|
||||
|
||||
max_num_tokens_padded = numel + local_num_experts * (block_size - 1)
|
||||
if numel < local_num_experts:
|
||||
max_num_tokens_padded = min(numel * block_size, max_num_tokens_padded)
|
||||
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
|
||||
|
||||
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=device)
|
||||
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
|
||||
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
|
||||
scatter_idx = (
|
||||
torch.empty((numel, 1), dtype=torch.int32, device=device)
|
||||
if return_scatter_idx
|
||||
else None
|
||||
)
|
||||
|
||||
ops.fused_globalize_align_block_size(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
rank_expert_offset,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
block_size,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
scatter_idx,
|
||||
)
|
||||
|
||||
outputs = (recv_topk_idx, sorted_ids, expert_ids, num_tokens_post_pad)
|
||||
return (*outputs, scatter_idx) if scatter_idx is not None else outputs
|
||||
@@ -103,6 +103,9 @@ class ExpertTokensMetadata:
|
||||
expert_num_tokens: torch.Tensor | None
|
||||
expert_num_tokens_cpu: torch.Tensor | None
|
||||
psum_recv_per_rank: torch.Tensor | None = None
|
||||
# local->global expert-id offset, set when globalization is deferred to the
|
||||
# experts' fused align kernel.
|
||||
rank_expert_offset: int | None = None
|
||||
|
||||
@staticmethod
|
||||
def make_from_list(
|
||||
@@ -1048,6 +1051,18 @@ class FusedMoEKernelModularImpl:
|
||||
):
|
||||
self.prepare_finalize = prepare_finalize
|
||||
self.fused_experts = fused_experts
|
||||
# Experts that globalize recv_topk_idx themselves let a capable prepare
|
||||
# stage skip its standalone launch.
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and isinstance(fused_experts, HummingIndexedExperts)
|
||||
and hasattr(prepare_finalize, "defer_globalize")
|
||||
):
|
||||
prepare_finalize.defer_globalize = True
|
||||
self.shared_experts: SharedExperts | None = None
|
||||
moe_parallel_config = fused_experts.moe_config.moe_parallel_config
|
||||
self.moe_parallel_config = moe_parallel_config
|
||||
|
||||
@@ -127,6 +127,11 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
self.use_cudagraph = use_cudagraph
|
||||
self.sp_size = sp_size
|
||||
|
||||
# Set by FusedMoEKernelModularImpl when the experts globalize
|
||||
# recv_topk_idx themselves, letting the decode path skip the standalone
|
||||
# _globalize_recv_topk_idx launch.
|
||||
self.defer_globalize = False
|
||||
|
||||
# DBO microbatching: one handle slot per micro-batch.
|
||||
self.handles: list[deep_ep.EPHandle | None] = [None, None]
|
||||
|
||||
@@ -275,6 +280,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# during profile_run when CUDA graphs are enabled.
|
||||
expert_tokens_meta = None
|
||||
|
||||
globalize_deferred = False
|
||||
if recv_topk_idx is None:
|
||||
# do_expand=True (prefill mode): build topk_ids from
|
||||
# per-expert token counts.
|
||||
@@ -307,12 +313,15 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# be treated as real routed tokens by experts that build routing
|
||||
# over *all* rows (e.g. triton MoE backend's make_routing_data),
|
||||
# polluting the per-expert token lists and corrupting real tokens.
|
||||
recv_topk_idx = _globalize_recv_topk_idx(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
self.rank_expert_offset,
|
||||
self.num_experts,
|
||||
)
|
||||
if self.defer_globalize and self.use_cudagraph:
|
||||
globalize_deferred = True
|
||||
else:
|
||||
recv_topk_idx = _globalize_recv_topk_idx(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
self.rank_expert_offset,
|
||||
self.num_experts,
|
||||
)
|
||||
|
||||
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
|
||||
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
||||
@@ -329,6 +338,9 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
expert_num_tokens_cpu=None,
|
||||
)
|
||||
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
|
||||
if globalize_deferred:
|
||||
# recv_topk_idx still holds LOCAL ids; experts globalize it.
|
||||
expert_tokens_meta.rank_expert_offset = self.rank_expert_offset
|
||||
|
||||
if _quantize_before_dispatch(quant_config, defer_input_quant):
|
||||
if quant_config.quant_dtype == "mxfp8" and expert_x_scale is not None:
|
||||
|
||||
Reference in New Issue
Block a user