[feat] FlashInfer CuteDSL MegaMoE integration (#54049)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
+65
-20
@@ -3,7 +3,7 @@
|
||||
import contextlib
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import asdict, fields
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
from pydantic import Field, field_validator
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -128,10 +129,10 @@ MoEBackend = Literal[
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_moe_ep_cutedsl",
|
||||
"flashinfer_b12x",
|
||||
"b12x",
|
||||
"flashinfer_moe_ep_mega_deep_gemm",
|
||||
"flashinfer_moe_ep_mega_cutedsl",
|
||||
"marlin",
|
||||
"humming",
|
||||
"triton_unfused",
|
||||
@@ -143,21 +144,28 @@ MoEBackend = Literal[
|
||||
"rdna3",
|
||||
]
|
||||
|
||||
# Backends that run the mega-MoE model path through the flashinfer moe_ep
|
||||
# runtime. Only architectures in FLASHINFER_MOE_EP_ARCHITECTURES wire up
|
||||
# these experts.
|
||||
FLASHINFER_MOE_EP_CUTEDSL = "flashinfer_moe_ep_cutedsl"
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM = "flashinfer_moe_ep_mega_deep_gemm"
|
||||
|
||||
# Backends implemented by the shared FlashInfer MoE-EP adapter.
|
||||
FLASHINFER_MOE_EP_BACKENDS = frozenset(
|
||||
{
|
||||
"flashinfer_moe_ep_mega_deep_gemm",
|
||||
"flashinfer_moe_ep_mega_cutedsl",
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
}
|
||||
)
|
||||
# Those backends dispatch and combine themselves, so the framework must not
|
||||
# run a separate all2all around them.
|
||||
PASSTHROUGH_ALL2ALL_BACKEND: Final = "passthrough"
|
||||
|
||||
# Backends that run a mega-MoE model path (fused expert module plus
|
||||
# prepare_megamoe routing): vLLM's native deep_gemm path, which any model
|
||||
# with a mega-MoE module may use (DeepSeek-V4, Kimi K3), plus the flashinfer
|
||||
# moe_ep variants.
|
||||
MEGA_MOE_BACKENDS = frozenset({"deep_gemm_mega_moe"}) | FLASHINFER_MOE_EP_BACKENDS
|
||||
# "Native" here meaning that the backend does not use the fused MoE path.
|
||||
# Currently only relevant for the vLLM provided DeepGEMM MegaMoE.,
|
||||
# TODO: migrate to the fused MoE path ultimately.
|
||||
NATIVE_MEGA_MOE_BACKENDS = frozenset({"deep_gemm_mega_moe"})
|
||||
|
||||
# Every megakernel backend. The FlashInfer variants run through the standard
|
||||
# fused-MoE model path; the quantization method binds the kernel.
|
||||
MEGA_MOE_BACKENDS = NATIVE_MEGA_MOE_BACKENDS | FLASHINFER_MOE_EP_BACKENDS
|
||||
|
||||
SparseIndexerTopkBackend = Literal[
|
||||
"auto",
|
||||
@@ -170,8 +178,8 @@ SparseIndexerTopkBackend = Literal[
|
||||
"aiter",
|
||||
]
|
||||
|
||||
# Architectures whose model code wires up the flashinfer moe_ep experts. MTP
|
||||
# and DSpark draft variants inherit the setting from these target models.
|
||||
# DeepGEMM consumes the DeepSeek-V4 MXFP4 checkpoint recipe. CuTeDSL is
|
||||
# selected by quantization methods and is not architecture-specific.
|
||||
FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV4ForCausalLM",
|
||||
@@ -181,11 +189,48 @@ FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
|
||||
)
|
||||
|
||||
|
||||
def bind_passthrough_all2all_backend(
|
||||
moe_backend: str, parallel_config: "ParallelConfig"
|
||||
) -> None:
|
||||
"""Bind ``all2all_backend="passthrough"`` to MoE backends that own their
|
||||
expert-parallel communication (currently the FlashInfer MoE-EP megakernels).
|
||||
|
||||
Such a deployment gets ``passthrough`` when the all2all backend is left at
|
||||
its default and rejects any other explicit choice; the value is meaningless
|
||||
with a MoE backend that relies on the framework to communicate. The CLI does
|
||||
not offer the value; the early return keeps repeated
|
||||
``set_platform_defaults`` calls idempotent.
|
||||
"""
|
||||
a2a = parallel_config.all2all_backend
|
||||
if moe_backend in FLASHINFER_MOE_EP_BACKENDS:
|
||||
if a2a == PASSTHROUGH_ALL2ALL_BACKEND:
|
||||
return
|
||||
if a2a != "allgather_reducescatter":
|
||||
raise ValueError(
|
||||
f"moe_backend={moe_backend!r} dispatches and combines itself and "
|
||||
f"cannot be paired with all2all_backend={a2a!r}; leave "
|
||||
f"--all2all-backend unset."
|
||||
)
|
||||
logger.info_once(
|
||||
"moe_backend=%r: using all2all_backend=%r (dispatch and combine run "
|
||||
"inside the MoE backend).",
|
||||
moe_backend,
|
||||
PASSTHROUGH_ALL2ALL_BACKEND,
|
||||
)
|
||||
parallel_config.all2all_backend = PASSTHROUGH_ALL2ALL_BACKEND
|
||||
elif a2a == PASSTHROUGH_ALL2ALL_BACKEND:
|
||||
raise ValueError(
|
||||
f"all2all_backend={a2a!r} is only valid with a moe_backend that "
|
||||
f"dispatches and combines itself ({sorted(FLASHINFER_MOE_EP_BACKENDS)}),"
|
||||
f" got moe_backend={moe_backend!r}."
|
||||
)
|
||||
|
||||
|
||||
def validate_flashinfer_moe_ep_model(
|
||||
moe_backend: str, architectures: Iterable[str]
|
||||
) -> None:
|
||||
"""Reject flashinfer moe_ep backends for models that lack the FI path."""
|
||||
if moe_backend not in FLASHINFER_MOE_EP_BACKENDS:
|
||||
"""Reject model-specific FlashInfer MoE-EP backends."""
|
||||
if moe_backend != FLASHINFER_MOE_EP_DEEP_GEMM:
|
||||
return
|
||||
if not any(arch in FLASHINFER_MOE_EP_ARCHITECTURES for arch in architectures):
|
||||
raise ValueError(
|
||||
@@ -255,6 +300,9 @@ class KernelConfig:
|
||||
- "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels
|
||||
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
||||
- "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
|
||||
- "flashinfer_moe_ep_cutedsl": Use FlashInfer's CuTeDSL MoE-EP
|
||||
mega-kernel with NVFP4 weights (MXFP4 checkpoints are requantized at
|
||||
load); requires Blackwell, expert parallelism, and NVSHMEM
|
||||
- "flashinfer_b12x": Use FlashInfer CuteDSL fused MoE for SM12x
|
||||
(RTX Pro 6000 / DGX Spark)
|
||||
- "b12x": Use b12x FP4 MoE kernels on SM12x
|
||||
@@ -262,10 +310,6 @@ class KernelConfig:
|
||||
expert-parallel mega-MoE with the DeepGEMM megakernel, which consumes an
|
||||
MXFP4 checkpoint verbatim (Blackwell, requires expert parallel;
|
||||
DeepSeek-V4 only)
|
||||
- "flashinfer_moe_ep_mega_cutedsl": Same, with the CuteDSL megakernel
|
||||
(additionally requires NVSHMEM). The checkpoint selects the weight path:
|
||||
an NVFP4 checkpoint is consumed prequantized, MXFP4 weights are
|
||||
requantized at load
|
||||
- "marlin": Use Marlin kernels (weight-only quantization)
|
||||
- "humming": Use Humming Mixed Precision kernels
|
||||
- "triton_unfused": Use Triton unfused MoE kernels
|
||||
@@ -391,6 +435,7 @@ class KernelConfig:
|
||||
validate_flashinfer_moe_ep_model(
|
||||
self.moe_backend, vllm_config.model_config.architectures
|
||||
)
|
||||
bind_passthrough_all2all_backend(self.moe_backend, vllm_config.parallel_config)
|
||||
|
||||
platform_op_priority = current_platform.get_default_ir_op_priority(vllm_config)
|
||||
logger.debug(
|
||||
|
||||
Reference in New Issue
Block a user