[WideEP] Integrate DeepEP v2 (#41183)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
05cb606cad
commit
e2f993dc41
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -43,6 +44,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.utils.import_utils import (
|
||||
has_aiter,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_deep_gemm,
|
||||
has_mori,
|
||||
)
|
||||
@@ -150,6 +152,10 @@ class Config:
|
||||
make env data for vllm launch.
|
||||
"""
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
is_moe=True,
|
||||
)
|
||||
vllm_config.parallel_config.data_parallel_size = self.world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
|
||||
@@ -239,6 +245,10 @@ class Config:
|
||||
or info.backend == "deepep_low_latency"
|
||||
)
|
||||
|
||||
def needs_deep_ep_v2(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.backend == "deepep_v2"
|
||||
|
||||
def needs_aiter(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return info.needs_aiter
|
||||
@@ -315,6 +325,8 @@ class Config:
|
||||
# Check dependencies (turn into asserts?)
|
||||
if self.needs_deep_ep() and not has_deep_ep():
|
||||
return False, "Needs DeepEP, but DeepEP not available."
|
||||
if self.needs_deep_ep_v2() and not has_deep_ep_v2():
|
||||
return False, "Needs DeepEP v2, but DeepEP v2 not available."
|
||||
if self.needs_deep_gemm() and not has_deep_gemm():
|
||||
return False, "Needs DeepGEMM, but DeepGEMM not available."
|
||||
if self.needs_aiter() and not has_aiter(): # noqa: SIM103
|
||||
@@ -657,6 +669,58 @@ def make_modular_kernel(
|
||||
return modular_kernel
|
||||
|
||||
|
||||
def _maybe_convert_weights_for_experts(
|
||||
config: Config,
|
||||
rank_weights: WeightTensors,
|
||||
) -> WeightTensors:
|
||||
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
)
|
||||
|
||||
fe_type = config.fused_experts_type
|
||||
fe_name = getattr(fe_type, "__name__", "")
|
||||
|
||||
backend: Fp8MoeBackend | None = None
|
||||
if fe_name == "TrtLlmFp8ExpertsModular":
|
||||
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
|
||||
elif fe_name == "FlashInferExperts":
|
||||
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
|
||||
|
||||
if backend is None or not rank_weights.is_quantized():
|
||||
return rank_weights
|
||||
|
||||
mock_layer = SimpleNamespace(
|
||||
weight_block_size=config.quant_block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=config.N,
|
||||
),
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
)
|
||||
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=backend,
|
||||
layer=mock_layer,
|
||||
w13=rank_weights.w1,
|
||||
w2=rank_weights.w2,
|
||||
w13_scale=rank_weights.w1_scale,
|
||||
w2_scale=rank_weights.w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
return WeightTensors(
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
w1_gs=rank_weights.w1_gs,
|
||||
w2_gs=rank_weights.w2_gs,
|
||||
)
|
||||
|
||||
|
||||
def run_modular_kernel(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -669,6 +733,7 @@ def run_modular_kernel(
|
||||
|
||||
# weights for rank
|
||||
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
|
||||
rank_weights = _maybe_convert_weights_for_experts(config, rank_weights)
|
||||
|
||||
if config.quant_dtype == "nvfp4":
|
||||
gscale = _make_gscale(config.num_local_experts)
|
||||
@@ -714,6 +779,8 @@ def run_modular_kernel(
|
||||
[num_tokens] * config.world_size, device="cuda", dtype=torch.int
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
|
||||
@@ -36,10 +36,12 @@ from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import (
|
||||
has_flashinfer_cutlass_fused_moe,
|
||||
has_flashinfer_nvlink_one_sided,
|
||||
has_flashinfer_trtllm_fused_moe,
|
||||
)
|
||||
from vllm.utils.import_utils import (
|
||||
has_aiter,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_deep_gemm,
|
||||
has_mori,
|
||||
)
|
||||
@@ -216,6 +218,19 @@ if has_deep_ep() and not current_platform.has_device_capability(100):
|
||||
backend="deepep_low_latency",
|
||||
)
|
||||
|
||||
if has_deep_ep_v2() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
standard_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="deepep_v2",
|
||||
)
|
||||
|
||||
if has_mori():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
@@ -289,6 +304,18 @@ if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability
|
||||
blocked_quantization_support=False,
|
||||
)
|
||||
|
||||
if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
|
||||
TrtLlmFp8ExpertsModular,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
TrtLlmFp8ExpertsModular,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
)
|
||||
|
||||
if has_aiter():
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
|
||||
@@ -106,7 +106,7 @@ def _worker_parallel_launch(
|
||||
if vllm_config is not None:
|
||||
cpu_group = _set_vllm_config(vllm_config, world_size, rank, local_rank)
|
||||
|
||||
try:
|
||||
def _run_worker():
|
||||
worker(
|
||||
ProcessGroupInfo(
|
||||
world_size=world_size,
|
||||
@@ -121,6 +121,13 @@ def _worker_parallel_launch(
|
||||
*args,
|
||||
**worker_kwargs,
|
||||
)
|
||||
|
||||
try:
|
||||
if vllm_config is not None:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
_run_worker()
|
||||
else:
|
||||
_run_worker()
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
traceback.print_exc()
|
||||
|
||||
Reference in New Issue
Block a user