[Kernel] Fuse DeepEP-v2 decode globalize+align into one launch (#53911)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
This commit is contained in:
Elvir Crnčević
2026-10-13 11:10:38 -04:00
committed by GitHub
parent 669f85e686
commit b9b401b2ef
10 changed files with 686 additions and 16 deletions
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness of the fused globalize + align + sort decode metadata kernel.
The fused op replaces the DeepEP-v2 INDEXED cudagraph-decode chain
(_globalize_recv_topk_idx + moe_align_block_size + count_and_sort). It is
checked against that exact reference chain over a grid of shapes / EP ranks /
fill distributions.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
fused_globalize_align_block_size,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep_v2
if current_platform.is_cuda() and has_deep_ep_v2():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
_globalize_recv_topk_idx,
)
else:
pytest.skip("Requires CUDA and DeepEP v2", allow_module_level=True)
def _expert_map(global_num_experts, rank_expert_offset, local_num_experts, device):
expert_map = torch.full((global_num_experts,), -1, device=device, dtype=torch.int32)
rng = torch.arange(local_num_experts, device=device, dtype=torch.int32)
expert_map[rank_expert_offset : rank_expert_offset + local_num_experts] = rng
return expert_map
def _make_case(n, topk, ep_size, local_e, block_size, seed, frac_recv, rank, fill):
torch.manual_seed(seed)
device = "cuda"
global_num_experts = local_e * ep_size
rank_idx = {"first": 0, "mid": ep_size // 2, "last": ep_size - 1}[rank]
rank_expert_offset = rank_idx * local_e
if fill == "all_nonlocal":
local_idx = torch.full((n, topk), -1, device=device, dtype=torch.int64)
elif fill == "single":
local_idx = torch.zeros((n, topk), device=device, dtype=torch.int64)
elif fill == "all_local":
local_idx = torch.randint(
0, local_e, (n, topk), device=device, dtype=torch.int64
)
else:
local_idx = torch.randint(
-1, local_e, (n, topk), device=device, dtype=torch.int64
)
num_recv = max(0, min(n, int(round(n * frac_recv))))
if num_recv < n:
local_idx[num_recv:] = torch.randint(
-1, local_e, (n - num_recv, topk), device=device, dtype=torch.int64
)
psum = torch.zeros(4, device=device, dtype=torch.int32)
psum[-1] = num_recv
return local_idx, psum, rank_expert_offset, global_num_experts
def _group_by_expert(sorted_ids, expert_ids, block_size, valid_len, total):
groups: dict[int, list[int]] = {}
for b in range(valid_len // block_size):
eid = int(expert_ids[b].item())
blk = sorted_ids[b * block_size : (b + 1) * block_size]
groups.setdefault(eid, []).extend(blk[blk < total].tolist())
return groups
@pytest.mark.parametrize("n", [1, 7, 64, 257])
# topk=16 with local_num_experts=28 is the Kimi K3 decode config. topk=6 and
# block_size=16, 128 are outside the old template dispatch table: they guard
# against a regression to fixed (topk, block_size) specialization.
@pytest.mark.parametrize("topk", [1, 6, 8, 16])
@pytest.mark.parametrize("block_size", [16, 32, 64, 128])
@pytest.mark.parametrize("local_e", [28, 32])
@pytest.mark.parametrize("rank", ["first", "mid", "last"])
@pytest.mark.parametrize("fill", ["rand", "all_local", "all_nonlocal", "single"])
def test_fused_matches_reference(n, topk, block_size, local_e, rank, fill):
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n * 131 + topk,
frac_recv=0.75,
rank=rank,
fill=fill,
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_global,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
fused_global, fused_sorted, fused_expert_ids, fused_num = (
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
)
total = n * topk
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
nb = valid // block_size
assert torch.equal(fused_expert_ids[:nb], ref_expert_ids[:nb])
assert bool((fused_expert_ids[nb:] == -1).all())
assert bool((fused_sorted[valid:] == total).all())
ref_groups = _group_by_expert(
ref_sorted, ref_expert_ids, block_size, int(ref_num.item()), total
)
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, total
)
assert set(ref_groups) == set(fused_groups)
for eid in ref_groups:
assert sorted(ref_groups[eid]) == sorted(fused_groups[eid])
@pytest.mark.parametrize("n", [7, 257])
def test_fused_scatter_idx_matches_reference(n):
"""The optional scatter output masks invalid and padded routes."""
local_e = 28
local_idx, psum, reo, gne = _make_case(
n, 16, 4, local_e, 64, seed=n, frac_recv=0.75, rank="mid", fill="rand"
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
*_, ref_scatter = moe_align_block_size(
topk_ids=ref_global,
block_size=64,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
return_scatter_idx=True,
)
fused_global, *_, fused_scatter = fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, 64, return_scatter_idx=True
)
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_scatter, ref_scatter)
# These have effective_work 1504 (single block) and 6068 (cooperative).
@pytest.mark.parametrize("n", [1, 64])
def test_fused_cuda_graph_replay(n):
"""Both launch paths remain capturable and consume replay-time inputs."""
topk = 16
local_e = 28
block_size = 64
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n,
frac_recv=0.75,
rank="mid",
fill="rand",
)
graph_idx = torch.empty_like(local_idx)
graph_psum = torch.empty_like(psum)
# Initialize lazy CUDA state before capture.
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_idx.copy_(local_idx)
graph_psum.copy_(psum)
graph_outputs = fused_globalize_align_block_size(
graph_idx, graph_psum, reo, gne, local_e, block_size
)
for frac_recv in (0.75, 0.5):
psum[-1] = int(round(n * frac_recv))
ref_idx = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_idx,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
graph.replay()
torch.accelerator.synchronize()
fused_idx, fused_sorted, fused_expert_ids, fused_num = graph_outputs
assert torch.equal(fused_idx, ref_idx)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
num_blocks = valid // block_size
assert torch.equal(fused_expert_ids[:num_blocks], ref_expert_ids[:num_blocks])
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, n * topk
)
ref_groups = _group_by_expert(
ref_sorted,
ref_expert_ids,
block_size,
int(ref_num.item()),
n * topk,
)
assert set(fused_groups) == set(ref_groups)
for eid in ref_groups:
assert sorted(fused_groups[eid]) == sorted(ref_groups[eid])