[MoE Refactor] Introduce RoutedExperts alias for FusedMoE and don't store SharedExperts in MK (#40735)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
bnellnm
2026-05-12 13:37:44 -04:00
committed by GitHub
co-authored by Robert Shaw Robert Shaw
parent 6ff7405b81
commit 4d591db470
53 changed files with 369 additions and 332 deletions
+19 -18
View File
@@ -4,7 +4,6 @@
from typing import TYPE_CHECKING, Any
import torch
from torch.nn import Module
from torch.utils._python_dispatch import TorchDispatchMode
import vllm.envs as envs
@@ -22,14 +21,15 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
FusedMoEMethodBase,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
@@ -191,7 +191,7 @@ class Fp8Config(QuantizationConfig):
offline_method = Fp8LinearMethod(self)
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
return offline_method
elif isinstance(layer, FusedMoE):
elif isinstance(layer, RoutedExperts):
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
@@ -315,7 +315,7 @@ class Fp8LinearMethod(LinearMethodBase):
def create_weights(
self,
layer: torch.nn.Module,
layer: RoutedExperts,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
@@ -389,7 +389,7 @@ class Fp8LinearMethod(LinearMethodBase):
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
def process_weights_after_loading(self, layer: Module) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.use_marlin:
# Only Marlin kernels support `marlin_input_dtype`; guard to avoid
# AttributeError if backend selection changes.
@@ -534,7 +534,7 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
)
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
def process_weights_after_loading(self, layer: Module) -> None:
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -575,7 +575,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
quant_config: The quantization config.
"""
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
super().__init__(layer.moe_config)
self.quant_config = quant_config
self.weight_block_size = self.quant_config.weight_block_size
@@ -606,7 +606,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
def create_weights(
self,
layer: Module,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -746,7 +746,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
def _setup_kernel(
self,
layer: FusedMoE,
layer: RoutedExperts,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -787,10 +787,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def process_weights_after_loading(self, layer: Module) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# Allow for accessing weights and scales in standard way.
w13 = layer.w13_weight
w2 = layer.w2_weight
@@ -844,7 +843,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"logic. This function should not be called."
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
w1_scale = getattr(layer, f"w13_{self.weight_scale_name}")
w2_scale = getattr(layer, f"w2_{self.weight_scale_name}")
a1_scale = layer.w13_input_scale
@@ -877,7 +876,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
def apply_monolithic(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
@@ -901,10 +900,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
def apply(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert not self.is_monolithic
@@ -919,6 +919,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -937,7 +938,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
uses_meta_device: bool = True
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
super().__init__(quant_config, layer)
assert not quant_config.is_checkpoint_fp8_serialized
assert quant_config.activation_scheme == "dynamic"
@@ -945,7 +946,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
def create_weights(
self,
layer: Module,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -1011,7 +1012,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
initialize_online_processing(layer)
def process_weights_after_loading(self, layer: Module) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# TODO(@ksayers): inplace fp8 quant kernel, initialize scales with ones
if getattr(layer, "_already_called_process_weights_after_loading", False):
return