[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
+65 -20
View File
@@ -3,7 +3,7 @@
import contextlib
from collections.abc import Callable, Iterable
from dataclasses import asdict, fields
from typing import TYPE_CHECKING, Any, Literal
from typing import TYPE_CHECKING, Any, Final, Literal
from pydantic import Field, field_validator
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
logger = init_logger(__name__)
@@ -128,10 +129,10 @@ MoEBackend = Literal[
"flashinfer_trtllm",
"flashinfer_cutlass",
"flashinfer_cutedsl",
"flashinfer_moe_ep_cutedsl",
"flashinfer_b12x",
"b12x",
"flashinfer_moe_ep_mega_deep_gemm",
"flashinfer_moe_ep_mega_cutedsl",
"marlin",
"humming",
"triton_unfused",
@@ -143,21 +144,28 @@ MoEBackend = Literal[
"rdna3",
]
# Backends that run the mega-MoE model path through the flashinfer moe_ep
# runtime. Only architectures in FLASHINFER_MOE_EP_ARCHITECTURES wire up
# these experts.
FLASHINFER_MOE_EP_CUTEDSL = "flashinfer_moe_ep_cutedsl"
FLASHINFER_MOE_EP_DEEP_GEMM = "flashinfer_moe_ep_mega_deep_gemm"
# Backends implemented by the shared FlashInfer MoE-EP adapter.
FLASHINFER_MOE_EP_BACKENDS = frozenset(
{
"flashinfer_moe_ep_mega_deep_gemm",
"flashinfer_moe_ep_mega_cutedsl",
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
}
)
# Those backends dispatch and combine themselves, so the framework must not
# run a separate all2all around them.
PASSTHROUGH_ALL2ALL_BACKEND: Final = "passthrough"
# Backends that run a mega-MoE model path (fused expert module plus
# prepare_megamoe routing): vLLM's native deep_gemm path, which any model
# with a mega-MoE module may use (DeepSeek-V4, Kimi K3), plus the flashinfer
# moe_ep variants.
MEGA_MOE_BACKENDS = frozenset({"deep_gemm_mega_moe"}) | FLASHINFER_MOE_EP_BACKENDS
# "Native" here meaning that the backend does not use the fused MoE path.
# Currently only relevant for the vLLM provided DeepGEMM MegaMoE.,
# TODO: migrate to the fused MoE path ultimately.
NATIVE_MEGA_MOE_BACKENDS = frozenset({"deep_gemm_mega_moe"})
# Every megakernel backend. The FlashInfer variants run through the standard
# fused-MoE model path; the quantization method binds the kernel.
MEGA_MOE_BACKENDS = NATIVE_MEGA_MOE_BACKENDS | FLASHINFER_MOE_EP_BACKENDS
SparseIndexerTopkBackend = Literal[
"auto",
@@ -170,8 +178,8 @@ SparseIndexerTopkBackend = Literal[
"aiter",
]
# Architectures whose model code wires up the flashinfer moe_ep experts. MTP
# and DSpark draft variants inherit the setting from these target models.
# DeepGEMM consumes the DeepSeek-V4 MXFP4 checkpoint recipe. CuTeDSL is
# selected by quantization methods and is not architecture-specific.
FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
{
"DeepseekV4ForCausalLM",
@@ -181,11 +189,48 @@ FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
)
def bind_passthrough_all2all_backend(
moe_backend: str, parallel_config: "ParallelConfig"
) -> None:
"""Bind ``all2all_backend="passthrough"`` to MoE backends that own their
expert-parallel communication (currently the FlashInfer MoE-EP megakernels).
Such a deployment gets ``passthrough`` when the all2all backend is left at
its default and rejects any other explicit choice; the value is meaningless
with a MoE backend that relies on the framework to communicate. The CLI does
not offer the value; the early return keeps repeated
``set_platform_defaults`` calls idempotent.
"""
a2a = parallel_config.all2all_backend
if moe_backend in FLASHINFER_MOE_EP_BACKENDS:
if a2a == PASSTHROUGH_ALL2ALL_BACKEND:
return
if a2a != "allgather_reducescatter":
raise ValueError(
f"moe_backend={moe_backend!r} dispatches and combines itself and "
f"cannot be paired with all2all_backend={a2a!r}; leave "
f"--all2all-backend unset."
)
logger.info_once(
"moe_backend=%r: using all2all_backend=%r (dispatch and combine run "
"inside the MoE backend).",
moe_backend,
PASSTHROUGH_ALL2ALL_BACKEND,
)
parallel_config.all2all_backend = PASSTHROUGH_ALL2ALL_BACKEND
elif a2a == PASSTHROUGH_ALL2ALL_BACKEND:
raise ValueError(
f"all2all_backend={a2a!r} is only valid with a moe_backend that "
f"dispatches and combines itself ({sorted(FLASHINFER_MOE_EP_BACKENDS)}),"
f" got moe_backend={moe_backend!r}."
)
def validate_flashinfer_moe_ep_model(
moe_backend: str, architectures: Iterable[str]
) -> None:
"""Reject flashinfer moe_ep backends for models that lack the FI path."""
if moe_backend not in FLASHINFER_MOE_EP_BACKENDS:
"""Reject model-specific FlashInfer MoE-EP backends."""
if moe_backend != FLASHINFER_MOE_EP_DEEP_GEMM:
return
if not any(arch in FLASHINFER_MOE_EP_ARCHITECTURES for arch in architectures):
raise ValueError(
@@ -255,6 +300,9 @@ class KernelConfig:
- "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
- "flashinfer_moe_ep_cutedsl": Use FlashInfer's CuTeDSL MoE-EP
mega-kernel with NVFP4 weights (MXFP4 checkpoints are requantized at
load); requires Blackwell, expert parallelism, and NVSHMEM
- "flashinfer_b12x": Use FlashInfer CuteDSL fused MoE for SM12x
(RTX Pro 6000 / DGX Spark)
- "b12x": Use b12x FP4 MoE kernels on SM12x
@@ -262,10 +310,6 @@ class KernelConfig:
expert-parallel mega-MoE with the DeepGEMM megakernel, which consumes an
MXFP4 checkpoint verbatim (Blackwell, requires expert parallel;
DeepSeek-V4 only)
- "flashinfer_moe_ep_mega_cutedsl": Same, with the CuteDSL megakernel
(additionally requires NVSHMEM). The checkpoint selects the weight path:
an NVFP4 checkpoint is consumed prequantized, MXFP4 weights are
requantized at load
- "marlin": Use Marlin kernels (weight-only quantization)
- "humming": Use Humming Mixed Precision kernels
- "triton_unfused": Use Triton unfused MoE kernels
@@ -391,6 +435,7 @@ class KernelConfig:
validate_flashinfer_moe_ep_model(
self.moe_backend, vllm_config.model_config.architectures
)
bind_passthrough_all2all_backend(self.moe_backend, vllm_config.parallel_config)
platform_op_priority = current_platform.get_default_ir_op_priority(vllm_config)
logger.debug(