[WideEP] Integrate DeepEP v2 (#41183)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tyler Michael Smith
2026-06-08 18:07:29 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 05cb606cad
commit e2f993dc41
22 changed files with 1409 additions and 17 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import torch
@@ -43,6 +44,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
@@ -150,6 +152,10 @@ class Config:
make env data for vllm launch.
"""
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
is_moe=True,
)
vllm_config.parallel_config.data_parallel_size = self.world_size
vllm_config.parallel_config.enable_expert_parallel = True
@@ -239,6 +245,10 @@ class Config:
or info.backend == "deepep_low_latency"
)
def needs_deep_ep_v2(self):
info = prepare_finalize_info(self.prepare_finalize_type)
return info.backend == "deepep_v2"
def needs_aiter(self):
info = expert_info(self.fused_experts_type)
return info.needs_aiter
@@ -315,6 +325,8 @@ class Config:
# Check dependencies (turn into asserts?)
if self.needs_deep_ep() and not has_deep_ep():
return False, "Needs DeepEP, but DeepEP not available."
if self.needs_deep_ep_v2() and not has_deep_ep_v2():
return False, "Needs DeepEP v2, but DeepEP v2 not available."
if self.needs_deep_gemm() and not has_deep_gemm():
return False, "Needs DeepGEMM, but DeepGEMM not available."
if self.needs_aiter() and not has_aiter(): # noqa: SIM103
@@ -657,6 +669,58 @@ def make_modular_kernel(
return modular_kernel
def _maybe_convert_weights_for_experts(
config: Config,
rank_weights: WeightTensors,
) -> WeightTensors:
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
)
fe_type = config.fused_experts_type
fe_name = getattr(fe_type, "__name__", "")
backend: Fp8MoeBackend | None = None
if fe_name == "TrtLlmFp8ExpertsModular":
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
elif fe_name == "FlashInferExperts":
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
if backend is None or not rank_weights.is_quantized():
return rank_weights
mock_layer = SimpleNamespace(
weight_block_size=config.quant_block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=config.N,
),
activation=SimpleNamespace(is_gated=True),
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=backend,
layer=mock_layer,
w13=rank_weights.w1,
w2=rank_weights.w2,
w13_scale=rank_weights.w1_scale,
w2_scale=rank_weights.w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
return WeightTensors(
w1=w1,
w2=w2,
w1_scale=w1_scale,
w2_scale=w2_scale,
w1_gs=rank_weights.w1_gs,
w2_gs=rank_weights.w2_gs,
)
def run_modular_kernel(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
@@ -669,6 +733,7 @@ def run_modular_kernel(
# weights for rank
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
rank_weights = _maybe_convert_weights_for_experts(config, rank_weights)
if config.quant_dtype == "nvfp4":
gscale = _make_gscale(config.num_local_experts)
@@ -714,6 +779,8 @@ def run_modular_kernel(
[num_tokens] * config.world_size, device="cuda", dtype=torch.int
)
torch.distributed.barrier()
with set_forward_context(
None,
vllm_config,
@@ -36,10 +36,12 @@ from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import (
has_flashinfer_cutlass_fused_moe,
has_flashinfer_nvlink_one_sided,
has_flashinfer_trtllm_fused_moe,
)
from vllm.utils.import_utils import (
has_aiter,
has_deep_ep,
has_deep_ep_v2,
has_deep_gemm,
has_mori,
)
@@ -216,6 +218,19 @@ if has_deep_ep() and not current_platform.has_device_capability(100):
backend="deepep_low_latency",
)
if has_deep_ep_v2() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
DeepEPV2PrepareAndFinalize,
)
register_prepare_and_finalize(
DeepEPV2PrepareAndFinalize,
standard_format,
common_float_types,
blocked_quantization_support=True,
backend="deepep_v2",
)
if has_mori():
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
@@ -289,6 +304,18 @@ if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability
blocked_quantization_support=False,
)
if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(100):
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
)
register_experts(
TrtLlmFp8ExpertsModular,
standard_format,
fp8_types,
blocked_quantization_support=True,
)
if has_aiter():
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
@@ -106,7 +106,7 @@ def _worker_parallel_launch(
if vllm_config is not None:
cpu_group = _set_vllm_config(vllm_config, world_size, rank, local_rank)
try:
def _run_worker():
worker(
ProcessGroupInfo(
world_size=world_size,
@@ -121,6 +121,13 @@ def _worker_parallel_launch(
*args,
**worker_kwargs,
)
try:
if vllm_config is not None:
with set_current_vllm_config(vllm_config):
_run_worker()
else:
_run_worker()
except Exception as ex:
print(ex)
traceback.print_exc()