[WideEP] Integrate DeepEP v2 (#41183)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tyler Michael Smith
2026-06-08 18:07:29 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 05cb606cad
commit e2f993dc41
22 changed files with 1409 additions and 17 deletions
+78
View File
@@ -19,6 +19,7 @@ from vllm.utils.flashinfer import (
has_flashinfer_nvlink_one_sided,
has_flashinfer_nvlink_two_sided,
)
from vllm.utils.import_utils import has_deep_ep_v2
from vllm.utils.network_utils import get_open_port
from ..utils import init_test_distributed_environment
@@ -194,6 +195,10 @@ requires_ptrace = pytest.mark.skipif(
not _has_sys_ptrace(),
reason="SYS_PTRACE required (docker run --cap-add=SYS_PTRACE)",
)
requires_deep_ep_v2 = pytest.mark.skipif(
not has_deep_ep_v2(),
reason="DeepEP v2 (ElasticBuffer) not available or NCCL < 2.30.4",
)
# NOTE: No module-level pytestmark here. The FlashInfer lifecycle tests have
# their own @requires_two_sided / @requires_one_sided decorators, and
@@ -856,3 +861,76 @@ def _one_sided_data_worker(rank, world_size):
def test_one_sided_dispatch_combine(world_size):
"""Test FlashInfer one-sided dispatch/combine with actual data flow."""
_spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size)
# ---------------------------------------------------------------------------
# Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle
# ---------------------------------------------------------------------------
#
# Tests DeepEPV2All2AllManager which wraps DeepEP's ElasticBuffer API using
# the NCCL GIN backend. Requires DeepEP >= 2.0 and NCCL >= 2.30.4.
#
# Uses EP group because the DeepEP v2 manager is constructed with an
# EP-scoped communicator in production. With tp=world_size the EP group
# spans all ranks.
# ---------------------------------------------------------------------------
def _deepep_v2_lifecycle_worker(rank, world_size):
from vllm.distributed.device_communicators.all2all import (
DeepEPV2All2AllManager,
)
cpu_group = get_ep_group().cpu_group
manager = DeepEPV2All2AllManager(cpu_group)
assert manager.rank == rank
assert manager.world_size == world_size
assert manager._num_sms is None
hidden_size = 7168
num_experts = world_size * 32
num_topk = 8
max_tokens = 256
handle_kwargs = dict(
num_max_tokens_per_rank=max_tokens,
hidden=hidden_size,
num_topk=num_topk,
num_experts=num_experts,
use_fp8_dispatch=False,
)
handle = manager.get_handle(handle_kwargs)
assert handle is not None
assert manager._num_sms is not None
assert manager._num_sms > 0
torch.distributed.barrier()
# get_handle again with same args should return cached handle
handle2 = manager.get_handle(dict(handle_kwargs))
assert handle2 is handle
torch.distributed.barrier()
# Destroy clears the cache
manager.destroy()
assert len(manager.handle_cache._cache) == 0
torch.distributed.barrier()
# Re-create after destroy
handle3 = manager.get_handle(dict(handle_kwargs))
assert handle3 is not None
torch.distributed.barrier()
manager.destroy()
@requires_multi_gpu
@requires_deep_ep_v2
@pytest.mark.parametrize("world_size", [2])
def test_deepep_v2_manager_lifecycle(world_size):
"""Test DeepEP v2 ElasticBuffer manager init, caching, and destroy."""
_spawn_workers(_deepep_v2_lifecycle_worker, world_size)
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import torch
@@ -43,6 +44,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
@@ -150,6 +152,10 @@ class Config:
make env data for vllm launch.
"""
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
is_moe=True,
)
vllm_config.parallel_config.data_parallel_size = self.world_size
vllm_config.parallel_config.enable_expert_parallel = True
@@ -239,6 +245,10 @@ class Config:
or info.backend == "deepep_low_latency"
)
def needs_deep_ep_v2(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.backend == "deepep_v2"
def needs_aiter(self):
info = expert_info(self.fused_experts_type)
return info.needs_aiter
@@ -315,6 +325,8 @@ class Config:
# Check dependencies (turn into asserts?)
if self.needs_deep_ep() and not has_deep_ep():
return False, "Needs DeepEP, but DeepEP not available."
if self.needs_deep_ep_v2() and not has_deep_ep_v2():
return False, "Needs DeepEP v2, but DeepEP v2 not available."
if self.needs_deep_gemm() and not has_deep_gemm():
return False, "Needs DeepGEMM, but DeepGEMM not available."
if self.needs_aiter() and not has_aiter(): # noqa: SIM103
@@ -657,6 +669,58 @@ def make_modular_kernel(
return modular_kernel
def _maybe_convert_weights_for_experts(
config: Config,
rank_weights: WeightTensors,
) -> WeightTensors:
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
)
fe_type = config.fused_experts_type
fe_name = getattr(fe_type, "__name__", "")
backend: Fp8MoeBackend | None = None
if fe_name == "TrtLlmFp8ExpertsModular":
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
elif fe_name == "FlashInferExperts":
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
if backend is None or not rank_weights.is_quantized():
return rank_weights
mock_layer = SimpleNamespace(
weight_block_size=config.quant_block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=config.N,
),
activation=SimpleNamespace(is_gated=True),
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=backend,
layer=mock_layer,
w13=rank_weights.w1,
w2=rank_weights.w2,
w13_scale=rank_weights.w1_scale,
w2_scale=rank_weights.w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
return WeightTensors(
w1=w1,
w2=w2,
w1_scale=w1_scale,
w2_scale=w2_scale,
w1_gs=rank_weights.w1_gs,
w2_gs=rank_weights.w2_gs,
)
def run_modular_kernel(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
@@ -669,6 +733,7 @@ def run_modular_kernel(
# weights for rank
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
rank_weights = _maybe_convert_weights_for_experts(config, rank_weights)
if config.quant_dtype == "nvfp4":
gscale = _make_gscale(config.num_local_experts)
@@ -714,6 +779,8 @@ def run_modular_kernel(
[num_tokens] * config.world_size, device="cuda", dtype=torch.int
)
torch.distributed.barrier()
with set_forward_context(
None,
vllm_config,
@@ -36,10 +36,12 @@ from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import (
has_flashinfer_cutlass_fused_moe,
has_flashinfer_nvlink_one_sided,
has_flashinfer_trtllm_fused_moe,
)
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
@@ -216,6 +218,19 @@ if has_deep_ep() and not current_platform.has_device_capability(100):
backend="deepep_low_latency",
)
if has_deep_ep_v2() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
DeepEPV2PrepareAndFinalize,
)
register_prepare_and_finalize(
DeepEPV2PrepareAndFinalize,
standard_format,
common_float_types,
blocked_quantization_support=True,
backend="deepep_v2",
)
if has_mori():
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
@@ -289,6 +304,18 @@ if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability
blocked_quantization_support=False,
)
if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
)
register_experts(
TrtLlmFp8ExpertsModular,
standard_format,
fp8_types,
blocked_quantization_support=True,
)
if has_aiter():
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
@@ -106,7 +106,7 @@ def _worker_parallel_launch(
if vllm_config is not None:
cpu_group = _set_vllm_config(vllm_config, world_size, rank, local_rank)
try:
def _run_worker():
worker(
ProcessGroupInfo(
world_size=world_size,
@@ -121,6 +121,13 @@ def _worker_parallel_launch(
*args,
**worker_kwargs,
)
try:
if vllm_config is not None:
with set_current_vllm_config(vllm_config):
_run_worker()
else:
_run_worker()
except Exception as ex:
print(ex)
traceback.print_exc()
+46 -3
View File
@@ -15,7 +15,7 @@ from torch.distributed import ProcessGroup
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
from typing_extensions import ParamSpec
from vllm.utils.import_utils import has_deep_ep
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
from vllm.utils.network_utils import get_open_port
if has_deep_ep():
@@ -26,6 +26,11 @@ if has_deep_ep():
DeepEPLLPrepareAndFinalize,
)
if has_deep_ep_v2():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
DeepEPV2PrepareAndFinalize,
)
## Parallel Processes Utils
P = ParamSpec("P")
@@ -55,11 +60,10 @@ def _worker_parallel_launch(
torch.accelerator.set_device_index(local_rank)
device = torch.device("cuda", local_rank)
torch.distributed.init_process_group(
backend="cpu:gloo,cuda:nccl",
backend="nccl",
init_method=init_method,
rank=rank,
world_size=world_size,
device_id=device,
)
barrier = torch.tensor([rank], device=device)
torch.distributed.all_reduce(barrier)
@@ -200,3 +204,42 @@ def make_deepep_a2a(
assert deepep_ll_args is not None
return make_deepep_ll_a2a(pg, pgi, deepep_ll_args, q_dtype, block_shape)
@dataclasses.dataclass
class DeepEPV2Args:
num_local_experts: int
num_experts: int
num_topk: int
hidden_size: int
max_tokens_per_rank: int
use_fp8_dispatch: bool
def make_deepep_v2_a2a(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
dp_size: int,
v2_args: DeepEPV2Args,
use_cudagraph: bool = False,
):
import deep_ep
buffer = deep_ep.ElasticBuffer(
group=pg,
num_max_tokens_per_rank=v2_args.max_tokens_per_rank,
hidden=v2_args.hidden_size,
num_topk=v2_args.num_topk,
use_fp8_dispatch=v2_args.use_fp8_dispatch,
explicitly_destroy=True,
)
return DeepEPV2PrepareAndFinalize(
buffer=buffer,
num_dispatchers=pgi.world_size,
dp_size=dp_size,
rank_expert_offset=pgi.rank * v2_args.num_local_experts,
num_experts=v2_args.num_experts,
num_topk=v2_args.num_topk,
use_fp8_dispatch=v2_args.use_fp8_dispatch,
use_cudagraph=use_cudagraph,
)
+542
View File
@@ -0,0 +1,542 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
Compares against a pure-PyTorch reference MoE implementation.
"""
import dataclasses
import pytest
import torch.distributed
from torch.distributed import ProcessGroup
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.utils.import_utils import has_deep_ep_v2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
from ...utils import multi_gpu_test
from .parallel_utils import ProcessGroupInfo, parallel_launch
if has_deep_ep_v2():
from .parallel_utils import DeepEPV2Args, make_deepep_v2_a2a
requires_deep_ep_v2 = pytest.mark.skipif(
not has_deep_ep_v2(),
reason="Requires DeepEP v2 (ElasticBuffer)",
)
@dataclasses.dataclass
class TestConfig:
dtype: torch.dtype
topk: int
m: int
k: int
n: int
num_experts: int
@dataclasses.dataclass
class TestTensors:
rank_tokens: torch.Tensor
rank_token_scales: torch.Tensor | None
topk: torch.Tensor
topk_weights: torch.Tensor
config: TestConfig
@staticmethod
def make(config: TestConfig) -> "TestTensors":
assert config.dtype in [torch.bfloat16, torch.float8_e4m3fn]
token_dtype = (
torch.bfloat16 if config.dtype == torch.float8_e4m3fn else config.dtype
)
rank_tokens = (
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
)
topk = torch.stack(
[
torch.randperm(config.num_experts, device="cuda")[: config.topk]
for _ in range(config.m)
]
).to(dtype=torch.int64)
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
return TestTensors(
rank_tokens=rank_tokens,
rank_token_scales=None,
topk=topk,
topk_weights=topk_weights,
config=config,
)
def make_modular_kernel(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
dp_size: int,
hidden_size: int,
num_experts: int,
num_local_experts: int,
topk: int,
q_dtype: torch.dtype | None,
use_fp8_dispatch: bool,
quant_config: FusedMoEQuantConfig,
use_cudagraph: bool = False,
) -> FusedMoEKernel:
v2_args = DeepEPV2Args(
num_local_experts=num_local_experts,
num_experts=num_experts,
num_topk=topk,
hidden_size=hidden_size,
max_tokens_per_rank=8192,
use_fp8_dispatch=use_fp8_dispatch,
)
a2a = make_deepep_v2_a2a(
pg=pg,
pgi=pgi,
dp_size=dp_size,
v2_args=v2_args,
use_cudagraph=use_cudagraph,
)
moe_config = make_dummy_moe_config(
num_experts=num_local_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
)
fused_experts = TritonExperts(
moe_config=moe_config,
quant_config=quant_config,
)
mk = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
inplace=False,
)
return mk
def deepep_v2_moe_impl(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
dp_size: int,
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
num_experts: int,
topk: int,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
) -> torch.Tensor:
num_local_experts = w1.size(0)
def build_expert_map():
expert_map = torch.full((num_experts,), fill_value=-1, dtype=torch.int32)
s = pgi.rank * num_local_experts
e = s + num_local_experts
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
device = torch.accelerator.current_device_index()
return expert_map.to(device=device, dtype=torch.int32)
is_quantized = w1.dtype == torch.float8_e4m3fn
q_dtype = torch.float8_e4m3fn if is_quantized else None
quant_config = FusedMoEQuantConfig.make(
q_dtype,
w1_scale=w1_scale,
w2_scale=w2_scale,
per_act_token_quant=per_act_token_quant,
a1_scale=test_tensors.rank_token_scales,
)
hidden_size = test_tensors.rank_tokens.size(1)
mk: FusedMoEKernel = make_modular_kernel(
pg,
pgi,
dp_size,
hidden_size,
num_experts,
num_local_experts,
topk,
q_dtype,
use_fp8_dispatch,
quant_config,
)
out = mk.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1,
w2=w2,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=build_expert_map(),
apply_router_weight_on_input=False,
)
return out
def _deep_ep_v2_moe(
pgi: ProcessGroupInfo,
dp_size: int,
config: TestConfig,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
use_fp8_dispatch: bool,
per_act_token_quant: bool,
):
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
is_quantized = w1.dtype == torch.float8_e4m3fn
device_idx = torch.accelerator.current_device_index()
w1 = w1.to(device=device_idx)
w2 = w2.to(device=device_idx)
if is_quantized:
assert w1_scale is not None and w2_scale is not None
w1_scale = w1_scale.to(device=device_idx)
w2_scale = w2_scale.to(device=device_idx)
pg = torch.distributed.new_group(list(range(pgi.world_size)))
test_tensors = TestTensors.make(config)
with set_current_vllm_config(VllmConfig()):
# Reference
q_dtype = torch.float8_e4m3fn if is_quantized else None
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1,
w2,
test_tensors.topk_weights,
test_tensors.topk,
w1_scale=w1_scale,
w2_scale=w2_scale,
quant_dtype=q_dtype,
per_act_token_quant=per_act_token_quant,
)
# Splice experts for this rank
num_local_experts = config.num_experts // pgi.world_size
e_start = num_local_experts * pgi.rank
e_end = e_start + num_local_experts
w1_ep = w1[e_start:e_end]
w2_ep = w2[e_start:e_end]
w1_scale_ep, w2_scale_ep = None, None
if is_quantized:
w1_scale_ep = w1_scale[e_start:e_end] # type: ignore
w2_scale_ep = w2_scale[e_start:e_end] # type: ignore
deepep_combined = deepep_v2_moe_impl(
pg,
pgi,
dp_size,
test_tensors,
w1_ep,
w2_ep,
w1_scale_ep,
w2_scale_ep,
config.num_experts,
config.topk,
use_fp8_dispatch,
per_act_token_quant,
)
torch.testing.assert_close(
torch_combined,
deepep_combined,
atol=6e-2,
rtol=6e-2,
)
MNKs = [
(1, 256, 256),
(2, 256, 512),
(3, 1024, 2048),
(32, 256, 1024),
(45, 512, 2048),
(64, 1024, 1024),
(222, 1024, 2048),
]
DTYPES = [torch.bfloat16, torch.float8_e4m3fn]
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m,n,k", MNKs)
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@multi_gpu_test(num_gpus=2)
@requires_deep_ep_v2
def test_deep_ep_v2_moe(
dtype: torch.dtype,
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
workspace_init,
):
per_act_token_quant = False
use_fp8_dispatch = False
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
quant_dtype = dtype if dtype == torch.float8_e4m3fn else None
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
num_experts,
n,
k,
quant_dtype=quant_dtype,
per_out_ch_quant=True,
)
parallel_launch(
world_size,
_deep_ep_v2_moe,
dp_size,
config,
w1,
w2,
w1_scale,
w2_scale,
use_fp8_dispatch,
per_act_token_quant,
)
def _deep_ep_v2_moe_cudagraph(
pgi: ProcessGroupInfo,
dp_size: int,
config: TestConfig,
w1: torch.Tensor,
w2: torch.Tensor,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
):
"""Worker function: verify DeepEP v2 + TrtLLM FP8 with do_expand=False."""
import tempfile
from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
pg = torch.distributed.new_group(list(range(pgi.world_size)))
test_tensors = TestTensors.make(config)
num_local_experts = config.num_experts // pgi.world_size
hidden_size = config.k
# Create FP8 weights directly, then dequantize for bf16 reference.
w1_fp8 = torch.randn(
(config.num_experts, 2 * config.n, config.k),
device="cuda",
dtype=torch.bfloat16,
).to(torch.float8_e4m3fn)
w2_fp8 = torch.randn(
(config.num_experts, config.k, config.n),
device="cuda",
dtype=torch.bfloat16,
).to(torch.float8_e4m3fn)
w1_ref = w1_fp8.to(torch.bfloat16)
w2_ref = w2_fp8.to(torch.bfloat16)
from vllm.config import KernelConfig
vllm_cfg = VllmConfig()
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
with set_current_vllm_config(vllm_cfg):
# Initialize vLLM parallel state (needed by FusedMoE layer)
temp_file = tempfile.mktemp()
init_distributed_environment(
world_size=pgi.world_size,
rank=pgi.rank,
distributed_init_method=f"file://{temp_file}",
local_rank=pgi.local_rank,
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
# Reference MoE using dequantized bf16 weights
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1_ref,
w2_ref,
test_tensors.topk_weights,
test_tensors.topk,
)
# Use the production pipeline: make_fused_moe_layer creates
# a FusedMoE layer, quantizes weights, runs
# process_weights_after_loading (TrtLLM W31 swap + BlockMajorK
# shuffle), and selects the kernel.
# Quantize weights using production helper, EP-slice, then
# convert to TrtLLM format.
from tests.kernels.moe.test_moe_layer import _quantize_fp8_halves
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
)
block_shape = [128, 128]
qw = _quantize_fp8_halves(w1_ref, w2_ref, block_shape)
# EP-slice before format conversion
e_start = num_local_experts * pgi.rank
e_end = e_start + num_local_experts
w1_ep = qw.w13_weight[e_start:e_end]
w2_ep = qw.w2_weight[e_start:e_end]
assert qw.w13_weight_scale is not None
assert qw.w2_weight_scale is not None
w1_scale_ep = qw.w13_weight_scale[e_start:e_end]
w2_scale_ep = qw.w2_weight_scale[e_start:e_end]
# Convert to TrtLLM format (W31 swap + BlockMajorK shuffle)
class _MockLayer:
weight_block_size = block_shape
class moe_config:
is_act_and_mul = True
intermediate_size_per_partition = config.n
class activation:
is_gated = True
w1_ep, w2_ep, w1_scale_ep, w2_scale_ep = convert_to_fp8_moe_kernel_format(
fp8_backend=Fp8MoeBackend.FLASHINFER_TRTLLM,
layer=_MockLayer(),
w13=w1_ep,
w2=w2_ep,
w13_scale=w1_scale_ep,
w2_scale=w2_scale_ep,
w13_input_scale=None,
w2_input_scale=None,
)
# Build TrtLLM expert with correct EP params
quant_config = FusedMoEQuantConfig.make(
torch.float8_e4m3fn,
block_shape=block_shape,
w1_scale=w1_scale_ep,
w2_scale=w2_scale_ep,
)
moe_config = make_dummy_moe_config(
num_experts=num_local_experts,
experts_per_token=config.topk,
hidden_dim=hidden_size,
intermediate_size=config.n,
)
fused_experts = TrtLlmFp8ExpertsModular(
moe_config=moe_config,
quant_config=quant_config,
)
v2_args = DeepEPV2Args(
num_local_experts=num_local_experts,
num_experts=config.num_experts,
num_topk=config.topk,
hidden_size=hidden_size,
max_tokens_per_rank=8192,
use_fp8_dispatch=False,
)
a2a = make_deepep_v2_a2a(
pg=pg,
pgi=pgi,
dp_size=dp_size,
v2_args=v2_args,
use_cudagraph=True,
)
mk_kernel = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
inplace=False,
)
for _ in range(3):
out = mk_kernel.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1_ep,
w2=w2_ep,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
torch.testing.assert_close(
torch_combined,
out,
atol=6e-2,
rtol=6e-2,
)
@pytest.mark.parametrize("m,n,k", [(32, 256, 1024)])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [6])
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
@multi_gpu_test(num_gpus=2)
@requires_deep_ep_v2
def test_deep_ep_v2_moe_cudagraph(
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
world_dp_size: tuple[int, int],
workspace_init,
):
set_random_seed(7)
world_size, dp_size = world_dp_size
config = TestConfig(
dtype=torch.float8_e4m3fn,
topk=topk,
m=m,
k=k,
n=n,
num_experts=num_experts,
)
parallel_launch(
world_size,
_deep_ep_v2_moe_cudagraph,
dp_size,
config,
None, # weights created inside worker
None,
None,
None,
)