[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -16,16 +16,14 @@ from vllm.model_executor.kernels.linear import (
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
|
||||
@@ -210,7 +210,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
)
|
||||
return None
|
||||
|
||||
def _add_fused_moe_to_target_scheme_map(self):
|
||||
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
|
||||
"""
|
||||
Helper function to update target_scheme_map
|
||||
since linear layers get fused into FusedMoE
|
||||
@@ -219,10 +219,10 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
"""
|
||||
if (
|
||||
"Linear" not in self.target_scheme_map
|
||||
or "FusedMoE" in self.target_scheme_map
|
||||
or "RoutedExperts" in self.target_scheme_map
|
||||
):
|
||||
return
|
||||
self.target_scheme_map["FusedMoE"] = self.target_scheme_map["Linear"]
|
||||
self.target_scheme_map["RoutedExperts"] = self.target_scheme_map["Linear"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "CompressedTensorsConfig":
|
||||
|
||||
@@ -10,14 +10,12 @@ import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -44,8 +42,9 @@ from vllm.model_executor.parameter import (
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if current_platform.is_cuda():
|
||||
if has_humming() and current_platform.is_cuda():
|
||||
from humming.dtypes import DataType
|
||||
from humming.layer import HummingMethod
|
||||
from humming.schema import (
|
||||
|
||||
@@ -155,6 +155,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
# TODO(bnell): move to MK/expert_class?
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
return self.mxfp4_backend in [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
]
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
@@ -487,6 +495,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
# TODO(bnell): move to MK/expert_class?
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
return self.mxfp4_backend in [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
]
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
|
||||
@@ -234,7 +234,12 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
intermediate_size_per_partition = layer.intermediate_size_per_partition
|
||||
# Note: The layer has not performed rounding on intermediate_size's at this
|
||||
# point. Use the unpadded size which won't change.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
assert intermediate_size_per_partition is not None
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
@@ -286,7 +287,7 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
@@ -353,7 +354,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Permute scales
|
||||
def premute_scales(
|
||||
def permute_scales(
|
||||
scales: torch.Tensor, g_scales: torch.Tensor, name: str
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
scales = scales.to(param_dtype)
|
||||
@@ -388,8 +389,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
g_scales = g_scales / combined_scale_factor
|
||||
return scales, g_scales
|
||||
|
||||
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
|
||||
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
|
||||
w13_scale, w13_scale_2 = permute_scales(w13_scale, w13_scale_2, "w13")
|
||||
w2_scale, w2_scale_2 = permute_scales(w2_scale, w2_scale_2, "w2")
|
||||
|
||||
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user