[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
The megakernel routes on vLLM's top-k output and then dispatches, computes and
combines in one launch, so it is paired with a pass-through prepare/finalize.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpEpilogue,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
flashinfer_moe_ep_unsupported_reasons,
supports_current_device,
validate_flashinfer_moe_ep_layer,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
require_finite_positive_scales,
)
from vllm.model_executor.utils import replace_parameter
if TYPE_CHECKING:
pass
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
(kNvfp4Static, kMxfp4Static)
)
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> None:
super().__init__(moe_config, quant_config)
self._adapter: FlashInferMoeEp | None = None
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return False, reason
reasons = flashinfer_moe_ep_unsupported_reasons(
moe_config, weight_key, activation_key
)
if reasons:
return False, f"kernel does not support {', '.join(reasons)}"
return True, None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return supports_current_device()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key in _supported_weight_quant_schemes
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in _supported_activations
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return (
moe_parallel_config.use_ep
and not moe_parallel_config.use_batched_activation_format
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return (0,), (0,), (M, K)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Build the megakernel from the layer's canonical weights.
The oracle has already put the weights in kernel format; the per-expert
weight global scales arrive through the quant config and become the
epilogue alphas.
"""
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
adapter = FlashInferMoeEp(
self.moe_config,
FlashInferMoeEpWeights(
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=getattr(layer, "w13_weight_scale", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
),
epilogue_from_quant_config(self.quant_config),
apply_topk_in_fc1=apply_topk_in_fc1(
self.moe_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
),
)
adapter.warmup()
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
replace_parameter(layer, name, tensor)
self._adapter = adapter
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
if self._adapter is None:
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
output.copy_(
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
)
def epilogue_from_quant_config(
quant_config: FusedMoEQuantConfig,
) -> FlashInferMoeEpEpilogue:
"""Per-expert epilogue constants from the canonical quant config.
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
alphas. Activations are quantized dynamically inside the megakernel, so the
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
and keep the default epilogue. The alphas alias the layer's parameters, so
EPLB permutations reach the kernel.
"""
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
if fc1_alpha is None and fc2_alpha is None:
return FlashInferMoeEpEpilogue()
assert fc1_alpha is not None and fc2_alpha is not None
require_finite_positive_scales("g1_alphas", fc1_alpha)
require_finite_positive_scales("g2_alphas", fc2_alpha)
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
raise ValueError("the megakernel needs one weight global scale per expert")
return FlashInferMoeEpEpilogue(
fc1_alpha=fc1_alpha.float().contiguous(),
fc2_alpha=fc2_alpha.float().contiguous(),
)