[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -16,16 +16,14 @@ from vllm.model_executor.kernels.linear import (
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
@@ -210,7 +210,7 @@ class CompressedTensorsConfig(QuantizationConfig):
)
return None
def _add_fused_moe_to_target_scheme_map(self):
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
@@ -219,10 +219,10 @@ class CompressedTensorsConfig(QuantizationConfig):
"""
if (
"Linear" not in self.target_scheme_map
or "FusedMoE" in self.target_scheme_map
or "RoutedExperts" in self.target_scheme_map
):
return
self.target_scheme_map["FusedMoE"] = self.target_scheme_map["Linear"]
self.target_scheme_map["RoutedExperts"] = self.target_scheme_map["Linear"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "CompressedTensorsConfig":
@@ -10,14 +10,12 @@ import torch
from vllm import envs
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -44,8 +42,9 @@ from vllm.model_executor.parameter import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
if current_platform.is_cuda():
if has_humming() and current_platform.is_cuda():
from humming.dtypes import DataType
from humming.layer import HummingMethod
from humming.schema import (
@@ -155,6 +155,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -487,6 +495,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -234,7 +234,12 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
intermediate_size_per_partition = layer.intermediate_size_per_partition
# Note: The layer has not performed rounding on intermediate_size's at this
# point. Use the unpadded size which won't change.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
assert intermediate_size_per_partition is not None
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
@@ -6,6 +6,7 @@ import torch
import vllm._custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
@@ -286,7 +287,7 @@ def prepare_fp4_layer_for_marlin(
def prepare_nvfp4_moe_layer_for_marlin(
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w13_scale: torch.Tensor,
w13_scale_2: torch.Tensor,
@@ -353,7 +354,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
# WEIGHT SCALES
# Permute scales
def premute_scales(
def permute_scales(
scales: torch.Tensor, g_scales: torch.Tensor, name: str
) -> tuple[torch.Tensor, torch.Tensor]:
scales = scales.to(param_dtype)
@@ -388,8 +389,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
g_scales = g_scales / combined_scale_factor
return scales, g_scales
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
w13_scale, w13_scale_2 = permute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = permute_scales(w2_scale, w2_scale_2, "w2")
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2