[MoE Refactor] Introduce RoutedExperts alias for FusedMoE and don't store SharedExperts in MK (#40735)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
bnellnm
2026-05-12 13:37:44 -04:00
committed by GitHub
co-authored by Robert Shaw Robert Shaw
parent 6ff7405b81
commit 4d591db470
53 changed files with 369 additions and 332 deletions
@@ -11,7 +11,7 @@ from vllm.config.quantization import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
@@ -118,7 +118,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
return Mxfp8OnlineLinearMethod()
else:
return Fp8PerTensorOnlineLinearMethod()
elif isinstance(layer, FusedMoE):
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
@@ -8,7 +8,6 @@ from torch.nn import Module
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
@@ -21,6 +20,7 @@ from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -314,7 +314,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
def _setup_kernel(
self,
layer: "FusedMoE",
layer: RoutedExperts,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -355,7 +355,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def get_fused_moe_quant_config(
@@ -7,11 +7,11 @@ import torch
from torch.nn import Module
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
make_int8_moe_kernel,
make_int8_moe_quant_config,
@@ -91,7 +91,7 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
replace_parameter(layer, "w13_scale", w13_scale)
replace_parameter(layer, "w2_scale", w2_scale)
def _setup_kernel(self, layer: "FusedMoE") -> None:
def _setup_kernel(self, layer: RoutedExperts) -> None:
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
@@ -100,7 +100,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def get_fused_moe_quant_config(
@@ -6,8 +6,12 @@ from abc import abstractmethod
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
FusedMoEQuantConfig,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.model_loader.reload.layerwise import (
initialize_online_processing,
)
@@ -127,11 +131,11 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
def apply_monolithic(
self,
layer: "FusedMoE", # type: ignore[name-defined] # noqa: F821
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
@@ -151,12 +155,13 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
def apply(
self,
layer: "FusedMoE", # type: ignore[name-defined] # noqa: F821
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -169,5 +174,6 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -10,9 +10,9 @@ from torch.nn import Module
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe import (
FusedMoEQuantConfig,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
@@ -161,7 +161,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
def _setup_kernel(
self,
layer: "FusedMoE",
layer: "RoutedExperts",
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def get_fused_moe_quant_config(