[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
FlashInferNVLinkTwoSidedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import (
has_deep_ep,
@@ -185,6 +188,10 @@ def maybe_make_prepare_finalize(
all2all_manager: Any | None = None,
input_dtype: torch.dtype | None = None,
) -> FusedMoEPrepareAndFinalize | None:
if moe.moe_parallel_config.use_passthrough_all2all:
# The experts dispatch and combine themselves, in every EP topology.
return PassThroughPrepareAndFinalize()
if not moe.moe_parallel_config.use_all2all_kernels:
if not allow_new_interface:
return None
@@ -1068,6 +1068,12 @@ class FusedMoEParallelConfig:
and self.all2all_backend == "allgather_reducescatter"
)
@property
def use_passthrough_all2all(self):
# Not gated on use_all2all_kernels: the experts dispatch and combine
# internally in every EP topology, including TP-only.
return self.all2all_backend == "passthrough"
@property
def use_mori_kernels(self):
return self.use_all2all_kernels and self.all2all_backend in (
@@ -1530,6 +1536,10 @@ class FusedMoEConfig:
def use_deepep_ll_kernels(self):
return self.moe_parallel_config.use_deepep_ll_kernels
@property
def use_passthrough_all2all(self):
return self.moe_parallel_config.use_passthrough_all2all
@property
def use_mori_kernels(self):
return self.moe_parallel_config.use_mori_kernels
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
The megakernel routes on vLLM's top-k output and then dispatches, computes and
combines in one launch, so it is paired with a pass-through prepare/finalize.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpEpilogue,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
flashinfer_moe_ep_unsupported_reasons,
supports_current_device,
validate_flashinfer_moe_ep_layer,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
require_finite_positive_scales,
)
from vllm.model_executor.utils import replace_parameter
if TYPE_CHECKING:
pass
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
(kNvfp4Static, kMxfp4Static)
)
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> None:
super().__init__(moe_config, quant_config)
self._adapter: FlashInferMoeEp | None = None
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return False, reason
reasons = flashinfer_moe_ep_unsupported_reasons(
moe_config, weight_key, activation_key
)
if reasons:
return False, f"kernel does not support {', '.join(reasons)}"
return True, None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return supports_current_device()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key in _supported_weight_quant_schemes
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in _supported_activations
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return (
moe_parallel_config.use_ep
and not moe_parallel_config.use_batched_activation_format
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return (0,), (0,), (M, K)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Build the megakernel from the layer's canonical weights.
The oracle has already put the weights in kernel format; the per-expert
weight global scales arrive through the quant config and become the
epilogue alphas.
"""
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
adapter = FlashInferMoeEp(
self.moe_config,
FlashInferMoeEpWeights(
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=getattr(layer, "w13_weight_scale", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
),
epilogue_from_quant_config(self.quant_config),
apply_topk_in_fc1=apply_topk_in_fc1(
self.moe_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
),
)
adapter.warmup()
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
replace_parameter(layer, name, tensor)
self._adapter = adapter
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
if self._adapter is None:
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
output.copy_(
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
)
def epilogue_from_quant_config(
quant_config: FusedMoEQuantConfig,
) -> FlashInferMoeEpEpilogue:
"""Per-expert epilogue constants from the canonical quant config.
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
alphas. Activations are quantized dynamically inside the megakernel, so the
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
and keep the default epilogue. The alphas alias the layer's parameters, so
EPLB permutations reach the kernel.
"""
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
if fc1_alpha is None and fc2_alpha is None:
return FlashInferMoeEpEpilogue()
assert fc1_alpha is not None and fc2_alpha is not None
require_finite_positive_scales("g1_alphas", fc1_alpha)
require_finite_positive_scales("g2_alphas", fc2_alpha)
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
raise ValueError("the megakernel needs one weight global scale per expert")
return FlashInferMoeEpEpilogue(
fc1_alpha=fc1_alpha.float().contiguous(),
fc2_alpha=fc2_alpha.float().contiguous(),
)
@@ -0,0 +1,481 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import contextlib
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, cast
import torch
from vllm.config import get_current_vllm_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
)
from vllm.distributed import get_ep_group
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import _import_deep_gemm
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
FlashInferMoeEpKernel = Literal["cutedsl", "deep_gemm"]
FlashInferMoeEpWeightFormat = Literal["nvfp4", "mxfp4"]
_WEIGHT_FORMATS: dict[QuantKey | None, FlashInferMoeEpWeightFormat] = {
kNvfp4Static: "nvfp4",
kMxfp4Static: "mxfp4",
}
_SUPPORTED_CAPABILITIES = frozenset({(10, 0), (10, 3)})
def supports_current_device() -> bool:
capability = current_platform.get_device_capability()
return capability is not None and (
(capability.major, capability.minor) in _SUPPORTED_CAPABILITIES
)
_E2M1_LUT = (
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
)
@dataclass(frozen=True)
class FlashInferMoeEpBackendSpec:
kernel: FlashInferMoeEpKernel
weight_formats: frozenset[FlashInferMoeEpWeightFormat]
@dataclass(frozen=True)
class FlashInferMoeEpWeights:
w13: torch.Tensor
w2: torch.Tensor
w13_scale: torch.Tensor | None = None
w2_scale: torch.Tensor | None = None
@dataclass(frozen=True)
class FlashInferMoeEpEpilogue:
input_norm_const: float = 1.0
fc1_alpha: torch.Tensor | None = None
fc2_alpha: torch.Tensor | None = None
fc1_norm_const: torch.Tensor | None = None
_BACKEND_SPECS = {
FLASHINFER_MOE_EP_CUTEDSL: FlashInferMoeEpBackendSpec(
kernel="cutedsl",
weight_formats=frozenset({"nvfp4", "mxfp4"}),
),
FLASHINFER_MOE_EP_DEEP_GEMM: FlashInferMoeEpBackendSpec(
kernel="deep_gemm",
weight_formats=frozenset({"mxfp4"}),
),
}
assert frozenset(_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
def is_flashinfer_moe_ep_backend(moe_backend: str) -> bool:
return moe_backend in _BACKEND_SPECS
def flashinfer_moe_ep_backend_spec(
moe_backend: str,
) -> FlashInferMoeEpBackendSpec:
try:
return _BACKEND_SPECS[moe_backend]
except KeyError:
raise ValueError(
f"{moe_backend!r} is not a FlashInfer MoE-EP backend; expected "
f"one of {sorted(_BACKEND_SPECS)}"
) from None
def flashinfer_moe_ep_unsupported_reasons(
moe: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> tuple[str, ...]:
"""Deployment constraints of the megakernel beyond the framework's checks.
``activation_key`` is ``None`` for MXFP4 checkpoints (the kernel quantizes
bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
"""
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
unsupported: list[str] = []
weight_format = _WEIGHT_FORMATS.get(weight_key)
if weight_format is None:
unsupported.append(f"weight scheme {weight_key}")
elif weight_format not in spec.weight_formats:
unsupported.append(f"{weight_format.upper()} weights")
if weight_key == kNvfp4Static and activation_key is None:
unsupported.append("A16 activations with NVFP4 weights")
if moe.skip_final_all_reduce:
unsupported.append("skip_final_all_reduce")
if moe.in_dtype != torch.bfloat16:
unsupported.append(f"activation dtype {moe.in_dtype}")
if moe.has_bias:
unsupported.append("expert bias")
if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
unsupported.append("custom SwiGLU alpha or beta")
if (
spec.kernel == "deep_gemm"
and moe.routing_method is not RoutingMethodType.DeepseekV4
):
unsupported.append(f"routing method {moe.routing_method.name}")
vllm_config = get_current_vllm_config()
if vllm_config.weight_transfer_config is not None:
unsupported.append("runtime weight transfer")
if vllm_config.parallel_config.enable_dbo:
unsupported.append("dual batch overlap")
if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
unsupported.append("EPLB")
return tuple(unsupported)
def validate_flashinfer_moe_ep_layer(layer: RoutedExperts) -> None:
if layer.expert_map_manager.placement_strategy != "linear":
raise ValueError(
f"{layer.moe_config.moe_backend} requires contiguous linear "
"expert placement"
)
def apply_topk_in_fc1(
moe: FusedMoEConfig,
*,
apply_router_weight_on_input: bool = False,
) -> bool:
return (
apply_router_weight_on_input
or moe.routing_method is RoutingMethodType.DeepseekV4
)
def _dequant_mxfp4_ue8m0_gran32(
packed: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
raw = packed.view(torch.uint8)
lut = torch.tensor(_E2M1_LUT, dtype=torch.float32, device=raw.device)
values = torch.empty(
raw.shape[0],
raw.shape[1] * 2,
dtype=torch.float32,
device=raw.device,
)
values[:, ::2] = lut[(raw & 0x0F).to(torch.int64)]
values[:, 1::2] = lut[(raw >> 4).to(torch.int64)]
encoded_scale = (
scale
if scale.dtype == torch.float8_e8m0fnu
else scale.view(torch.float8_e8m0fnu)
)
decoded_scale = encoded_scale.float()
return (values * decoded_scale.repeat_interleave(32, dim=-1)).to(torch.bfloat16)
def _dequant_mxfp4_experts(
weight: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
num_experts, output_size, packed_input_size = weight.shape
result = torch.empty(
num_experts,
output_size,
packed_input_size * 2,
dtype=torch.bfloat16,
device=weight.device,
)
for expert_id in range(num_experts):
result[expert_id] = _dequant_mxfp4_ue8m0_gran32(
weight[expert_id], scale[expert_id]
)
return result
def mxfp4_moe_ep_weights(
moe_backend: str,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> FlashInferMoeEpWeights:
spec = flashinfer_moe_ep_backend_spec(moe_backend)
if spec.kernel == "deep_gemm":
return FlashInferMoeEpWeights(w13, w2, w13_scale, w2_scale)
return FlashInferMoeEpWeights(
w13=_dequant_mxfp4_experts(w13, w13_scale),
w2=_dequant_mxfp4_experts(w2, w2_scale),
)
class _FlashInferMoeEpApi:
def __init__(self) -> None:
try:
from flashinfer.moe_ep import (
BootstrapConfig,
DeepGemmMegaMoeConfig,
FleetParams,
MegaConfig,
MoEEpMegaLayer,
MoEEpTensors,
MoEWeightPack,
Nvfp4CutedslMegaMoeConfig,
)
except ImportError as err:
raise RuntimeError(
"FlashInfer MoE-EP requires the public API shipped with "
"FlashInfer 0.7.0 or newer"
) from err
self.BootstrapConfig = BootstrapConfig
self.DeepGemmMegaMoeConfig = DeepGemmMegaMoeConfig
self.FleetParams = FleetParams
self.MegaConfig = MegaConfig
self.MoEEpMegaLayer = MoEEpMegaLayer
self.MoEEpTensors = MoEEpTensors
self.MoEWeightPack = MoEWeightPack
self.Nvfp4CutedslMegaMoeConfig = Nvfp4CutedslMegaMoeConfig
def _load_flashinfer_moe_ep_api() -> _FlashInferMoeEpApi:
return _FlashInferMoeEpApi()
def _expose_deep_gemm_to_flashinfer() -> None:
deep_gemm = _import_deep_gemm()
if deep_gemm is None:
raise RuntimeError(
"FlashInfer MoE-EP DeepGEMM requires a usable DeepGEMM installation"
)
sys.modules.setdefault("deep_gemm", deep_gemm)
class FlashInferMoeEp:
def __init__(
self,
moe: FusedMoEConfig,
weights: FlashInferMoeEpWeights,
epilogue: FlashInferMoeEpEpilogue | None = None,
*,
apply_topk_in_fc1: bool,
) -> None:
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
if spec.kernel == "deep_gemm":
_expose_deep_gemm_to_flashinfer()
api = _load_flashinfer_moe_ep_api()
if epilogue is None:
epilogue = FlashInferMoeEpEpilogue()
ep_group = get_ep_group()
bootstrap = api.BootstrapConfig(
world_size=ep_group.world_size,
rank=ep_group.rank_in_group,
process_group=ep_group.device_group,
device=torch.accelerator.current_device_index(),
)
fleet_params = api.FleetParams(
num_experts=moe.num_experts,
max_tokens_per_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
)
weight_pack = api.MoEWeightPack(
w13=weights.w13,
w2=weights.w2,
w13_scale=weights.w13_scale,
w2_scale=weights.w2_scale,
)
if spec.kernel == "cutedsl":
megakernel = api.Nvfp4CutedslMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
gate_up_clamp=moe.swiglu_limit,
fast_math=True,
apply_topk_in_fc1=apply_topk_in_fc1,
enable_in_kernel_fc2_reduce=False,
combine_dtype="bf16",
input_norm_const=epilogue.input_norm_const,
fc1_alpha=None,
fc2_alpha=None,
fc1_norm_const=None,
knobs=None,
)
else:
megakernel = api.DeepGemmMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
activation_clamp=moe.swiglu_limit,
fast_math=True,
)
backend = api.MegaConfig(
megakernel=megakernel,
quantize_input=True,
preprocess_weights=True,
)
self._moe_ep_tensors_cls = api.MoEEpTensors
self._mega_layer: Any | None = api.MoEEpMegaLayer(
bootstrap,
fleet_params,
weight_pack,
backend,
)
self._device = weights.w13.device
self._hidden_size = moe.hidden_dim
self._max_num_tokens = moe.max_num_tokens
self._num_experts = moe.num_experts
self._top_k = moe.experts_per_token
self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
self._return_workspace_view = moe.routing_method is RoutingMethodType.DeepseekV4
@property
def can_overlap_shared_experts(self) -> bool:
return False
@property
def output_is_reduced(self) -> bool:
return True
@property
def topk_indices_dtype(self) -> torch.dtype:
return torch.int32
@property
def is_monolithic(self) -> bool:
return False
def _tensors(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> Any:
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
return self._moe_ep_tensors_cls(
hidden_states=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
fc1_alpha=epilogue.fc1_alpha,
fc2_alpha=epilogue.fc2_alpha,
fc1_norm_const=epilogue.fc1_norm_const,
)
def __call__(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
if hidden_states.shape[0] > self._max_num_tokens:
raise ValueError(
f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
f"the workspace supports at most {self._max_num_tokens}"
)
tensors = self._tensors(hidden_states, topk_ids, topk_weights)
if self._return_workspace_view and getattr(
self._mega_layer, "supports_output_view", False
):
return self._mega_layer.forward(tensors, return_workspace_view=True)
return self._mega_layer(tensors)
@torch.inference_mode()
def warmup(self) -> None:
if self._mega_layer is None:
return
hidden_states = torch.zeros(
1,
self._hidden_size,
dtype=torch.bfloat16,
device=self._device,
)
topk_ids = (
torch.arange(self._top_k, dtype=torch.int32, device=self._device)
.mul_(self._num_experts // self._top_k)
.view(1, self._top_k)
)
topk_weights = torch.full(
(1, self._top_k),
1.0 / self._top_k,
dtype=torch.float32,
device=self._device,
)
self._mega_layer.warmup(self._tensors(hidden_states, topk_ids, topk_weights))
def kernel_weights(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.
Contiguous per-expert byte tensors that alias the memory the megakernel
reads, so a caller may permute experts in place (EPLB).
"""
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
(fc1_weight, fc1_sf), (fc2_weight, fc2_sf) = self._mega_layer._transformed
return (
_expert_major_bytes(fc1_weight),
_expert_major_bytes(fc1_sf),
_expert_major_bytes(fc2_weight),
_expert_major_bytes(fc2_sf),
)
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
"""Every per-expert tensor the kernel reads: weights plus epilogue."""
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
return (*self.kernel_weights(), *(v for v in vectors if v is not None))
def destroy(self) -> None:
mega_layer = self._mega_layer
if mega_layer is None:
return
mega_layer.destroy()
self._mega_layer = None
self._epilogue = None
def __del__(self) -> None:
with contextlib.suppress(Exception):
self.destroy()
def _expert_major_bytes(tensor: torch.Tensor) -> torch.Tensor:
"""Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8.
The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
"""
if not tensor.is_contiguous():
tensor = tensor.transpose(1, 2)
if not tensor.is_contiguous():
raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
return tensor.view(torch.uint8) if tensor.element_size() == 1 else tensor
@@ -8,7 +8,11 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.kernel import (
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MoEBackend,
)
from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
@@ -145,6 +149,9 @@ class Mxfp4MoeBackend(Enum):
EMULATION = "EMULATION"
# Humming
HUMMING = "HUMMING"
# FlashInfer MoE-EP megakernels: routing outside, dispatch/GEMMs/combine fused.
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_MOE_EP_DEEP_GEMM = "FLASHINFER_MOE_EP_DEEP_GEMM"
# Backends that share the same TRTLLM weight format
@@ -165,6 +172,13 @@ B12X_BACKENDS = (
Mxfp4MoeBackend.B12X_MXFP4_BF16,
)
# Megakernels consume the checkpoint's MXFP4 tensors through the shared
# FlashInfer MoE-EP adapter; vLLM does not convert their weight layout.
FLASHINFER_MOE_EP_MXFP4_BACKENDS = (
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM,
)
def backend_to_kernel_cls(
backend: Mxfp4MoeBackend,
@@ -291,6 +305,13 @@ def backend_to_kernel_cls(
return [OCP_MXQuantizationEmulationTritonExperts]
elif backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
@@ -314,6 +335,10 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
],
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
"flashinfer_moe_ep_cutedsl": [Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL],
"flashinfer_moe_ep_mega_deep_gemm": [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
],
"triton": [Mxfp4MoeBackend.TRITON],
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
"humming": [Mxfp4MoeBackend.HUMMING],
@@ -735,6 +760,9 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
activation: MoEActivation | None = None,
) -> tuple[int, int]:
"""Round up hidden_size and intermediate_size based on backend requirements."""
if backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel takes the model's dimensions and validates them itself.
return hidden_size, intermediate_size
if backend in B12X_BACKENDS:
# b12x plans for the exact model dimensions. B12xExperts validates the
# required MXFP4 block alignment before selecting the backend.
@@ -1827,6 +1855,29 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias=w2_bias,
_cache_permute_indices=_cache_permute_indices,
)
elif mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
mxfp4_moe_ep_weights,
)
# DeepGEMM consumes MXFP4 directly; CuTeDSL gets bf16 and requantizes.
weights = mxfp4_moe_ep_weights(
FLASHINFER_MOE_EP_CUTEDSL
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
else FLASHINFER_MOE_EP_DEEP_GEMM,
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
)
return (
weights.w13,
weights.w2,
weights.w13_scale,
weights.w2_scale,
w13_bias,
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_mxfp4_moe_layer_for_cpu,
@@ -1874,6 +1925,9 @@ def make_mxfp4_moe_quant_config(
layer: "RoutedExperts | None" = None,
) -> FusedMoEQuantConfig | None:
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
if mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel quantizes activations itself and owns its scales.
return FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
if mxfp4_backend == Mxfp4MoeBackend.B12X_MXFP4_MXFP8:
return mxfp4_mxfp8_moe_quant_config(
w1_bias=w1_bias,
@@ -50,6 +50,7 @@ class NvFp4MoeBackend(Enum):
MARLIN = "MARLIN"
HUMMING = "HUMMING"
EMULATION = "EMULATION"
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_NVFP4_MOE_BACKENDS = [
@@ -148,6 +149,12 @@ def backend_to_kernel_cls(
)
return [Nvfp4QuantizationEmulationTritonExperts]
elif backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")
@@ -160,6 +167,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
"flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
"flashinfer_moe_ep_cutedsl": NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
"marlin": NvFp4MoeBackend.MARLIN,
"humming": NvFp4MoeBackend.HUMMING,
@@ -207,6 +215,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_CUTLASS,
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
NvFp4MoeBackend.EMULATION,
@@ -281,8 +290,9 @@ def select_nvfp4_moe_backend(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"explicitly requested moe_backend={runner_backend!r} does "
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
f"'b12x', 'marlin', or 'humming' instead."
f"'flashinfer_cutlass', 'flashinfer_cutedsl', "
f"'flashinfer_moe_ep_cutedsl', 'cutlass', 'b12x', 'marlin', "
f"or 'humming' instead."
)
return _return_or_raise(
requested_backend, config, weight_key, activation_key, activation_format
@@ -491,6 +501,9 @@ def convert_to_nvfp4_moe_kernel_format(
# for other experts - other selection strategies may be used.
a13_scale = 1.0 / a13_scale.max().to(torch.float32)
a2_scale = 1.0 / a2_scale.max().to(torch.float32)
elif nvfp4_backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
# The megakernel transforms the checkpoint tensors itself at init.
pass
else:
raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}")
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prepare/finalize stages that do no expert-parallel communication.
Selected by ``all2all_backend="passthrough"``: the experts implementation
dispatches, combines and reduces itself, so nothing is left for these stages.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
MoEPrepareAndFinalizeNoDPEPModular,
)
class PassThroughPrepareAndFinalize(MoEPrepareAndFinalizeNoDPEPModular):
"""Hands activations and routing straight to experts that own their EP
communication."""
def output_is_reduced(self) -> bool:
return True
def supports_deferred_moe_finalize(self) -> bool:
# The experts apply the top-k weights and combine internally; there
# is no finalize left for a consumer to take over.
return False
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
return a1, None, None, None, None
@@ -24,10 +24,15 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
is_flashinfer_moe_ep_backend,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
@@ -38,7 +43,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -49,12 +54,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
# Backend selection must match the weight preparation below: CUTLASS
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
# repacks weights and scales.
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
# repacks weights and scales, and the megakernels take the checkpoint
# tensors as they are.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.experts_cls: type[mk.FusedMoEExperts]
if moe.moe_backend == "b12x":
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
assert experts_cls is not None
self.experts_cls = experts_cls
@@ -70,6 +77,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
logger.info_once("Using MarlinExperts for MXFP4 MoE")
self.experts_cls = MarlinExperts
@property
def supports_eplb(self) -> bool:
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
def create_weights(
self,
layer: torch.nn.Module,
@@ -198,6 +209,21 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
w13, w2, w13_scale, w2_scale, _, _ = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
w13_weight_scale=layer.w13_weight_scale,
w2_weight_scale=layer.w2_weight_scale,
)
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
@@ -211,6 +237,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
@@ -38,6 +38,7 @@ _EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
}
)
@@ -61,7 +62,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
weight_key=kNvfp4Static,
activation_key=None if use_a16 else kNvfp4Dynamic,
)
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
self.nvfp4_backend
)
@@ -155,7 +155,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)
w2_weight_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_weight_global_scale", w2_weight_scale_2)
extra_weight_attrs.update(
@@ -175,7 +176,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_input_scale, extra_weight_attrs)
w2_input_scale = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_input_global_scale", w2_input_scale)
extra_weight_attrs.update(
@@ -4,6 +4,7 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -53,6 +54,11 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe) -> None:
super().__init__(moe)
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
raise ValueError(
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
"(INC) MXFP4 MoE checkpoints"
)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which use
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
@@ -783,7 +784,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer):
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
return
@@ -25,6 +25,11 @@ INT4_DTYPE = scalar_types.uint4b8
INT8_DTYPE = scalar_types.uint8b128
def require_finite_positive_scales(name: str, value: torch.Tensor) -> None:
if not torch.isfinite(value).all() or not (value > 0).all():
raise ValueError(f"{name} must contain finite positive scales")
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
"""Return a stable short name for torch and ScalarType dtypes."""
if isinstance(dtype, ScalarType):