[Model][MoE] DeepSeek-V4: add opt-in FlashInfer moe_ep expert backend (#49636)

Signed-off-by: Md Saidul Hoque Anik <mhoqueanik@nvidia.com>
Signed-off-by: Anerudhan Gopal <agopal@nvidia.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Anerudhan Gopal <agopal@nvidia.com>
This commit is contained in:
Md Saidul Hoque Anik
2026-08-25 03:35:01 +00:00
committed by GitHub
co-authored by Claude Fable 5 Anerudhan Gopal
parent d3e2888c75
commit 8fe9317f2e
5 changed files with 1083 additions and 12 deletions
+56 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
from collections.abc import Callable
from collections.abc import Callable, Iterable
from dataclasses import asdict, fields
from typing import TYPE_CHECKING, Any, Literal
@@ -131,6 +131,8 @@ MoEBackend = Literal[
"flashinfer_cutedsl",
"flashinfer_b12x",
"b12x",
"flashinfer_moe_ep_mega_deep_gemm",
"flashinfer_moe_ep_mega_cutedsl",
"marlin",
"humming",
"triton_unfused",
@@ -140,6 +142,46 @@ MoEBackend = Literal[
"emulation",
]
# Backends that run the mega-MoE model path through the flashinfer moe_ep
# runtime. Only architectures in FLASHINFER_MOE_EP_ARCHITECTURES wire up
# these experts.
FLASHINFER_MOE_EP_BACKENDS = frozenset(
{
"flashinfer_moe_ep_mega_deep_gemm",
"flashinfer_moe_ep_mega_cutedsl",
}
)
# Backends that run a mega-MoE model path (fused expert module plus
# prepare_megamoe routing): vLLM's native deep_gemm path, which any model
# with a mega-MoE module may use (DeepSeek-V4, Kimi K3), plus the flashinfer
# moe_ep variants.
MEGA_MOE_BACKENDS = frozenset({"deep_gemm_mega_moe"}) | FLASHINFER_MOE_EP_BACKENDS
# Architectures whose model code wires up the flashinfer moe_ep experts. MTP
# and DSpark draft variants inherit the setting from these target models.
FLASHINFER_MOE_EP_ARCHITECTURES = frozenset(
{
"DeepseekV4ForCausalLM",
"DeepSeekV4MTPModel",
}
)
def validate_flashinfer_moe_ep_model(
moe_backend: str, architectures: Iterable[str]
) -> None:
"""Reject flashinfer moe_ep backends for models that lack the FI path."""
if moe_backend not in FLASHINFER_MOE_EP_BACKENDS:
return
if not any(arch in FLASHINFER_MOE_EP_ARCHITECTURES for arch in architectures):
raise ValueError(
f"moe_backend={moe_backend!r} is only supported for DeepSeek-V4 "
f"models ({sorted(FLASHINFER_MOE_EP_ARCHITECTURES)}), but the "
f"model is {list(architectures)}."
)
LinearBackend = Literal[
"auto",
"cutlass",
@@ -206,6 +248,14 @@ class KernelConfig:
- "flashinfer_b12x": Use FlashInfer CuteDSL fused MoE for SM12x
(RTX Pro 6000 / DGX Spark)
- "b12x": Use b12x FP4 MoE kernels on SM12x
- "flashinfer_moe_ep_mega_deep_gemm": Use the FlashInfer moe_ep
expert-parallel mega-MoE with the DeepGEMM megakernel, which consumes an
MXFP4 checkpoint verbatim (Blackwell, requires expert parallel;
DeepSeek-V4 only)
- "flashinfer_moe_ep_mega_cutedsl": Same, with the CuteDSL megakernel
(additionally requires NVSHMEM). The checkpoint selects the weight path:
an NVFP4 checkpoint is consumed prequantized, MXFP4 weights are
requantized at load
- "marlin": Use Marlin kernels (weight-only quantization)
- "humming": Use Humming Mixed Precision kernels
- "triton_unfused": Use Triton unfused MoE kernels
@@ -288,6 +338,11 @@ class KernelConfig:
"""Set platform-specific defaults for the kernel config."""
from vllm.platforms import current_platform
if vllm_config.model_config is not None:
validate_flashinfer_moe_ep_model(
self.moe_backend, vllm_config.model_config.architectures
)
platform_op_priority = current_platform.get_default_ir_op_priority(vllm_config)
logger.debug(
"Setting platform-specific IR op priority defaults: %s, user-defined: %s",