[MoE] Make MoERunnerInterface a PluggableLayer for OOT support (#35178)

Signed-off-by: wxsIcey <1790571317@qq.com>
Signed-off-by: Icey <1790571317@qq.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Xiaoshuang Wang
2026-04-30 03:31:08 -07:00
committed by GitHub
co-authored by Cyrus Leung mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3179e53135
commit b55b26520c
4 changed files with 35 additions and 23 deletions
+13 -1
View File
@@ -38,8 +38,20 @@ class MockCustomOp:
return decorator
class MockPluggableLayer:
@staticmethod
def register(name):
def decorator(cls):
return cls
return decorator
mock_if_no_torch("vllm._C", MagicMock())
mock_if_no_torch("vllm.model_executor.custom_op", MagicMock(CustomOp=MockCustomOp))
mock_if_no_torch(
"vllm.model_executor.custom_op",
MagicMock(CustomOp=MockCustomOp, PluggableLayer=MockPluggableLayer),
)
mock_if_no_torch(
"vllm.utils.torch_utils", MagicMock(direct_register_custom_op=lambda *a, **k: None)
)
+11 -12
View File
@@ -1487,15 +1487,19 @@ class FusedMoE(PluggableLayer):
"w2_input_scale",
}
# Parameters of non-expert submodules that live inside runner (MoERunner).
# These must be excluded from EPLB weight rearrangement.
NON_EXPERT_PREFIXES = (
"runner._shared_experts.",
"runner.gate.",
"runner.routed_input_transform.",
"runner.routed_output_transform.",
)
assert all(
weight.is_contiguous()
for name, weight in weights
if not (
name.startswith("_shared_experts.")
or name.startswith("_gate.")
or name.startswith("_routed_input_transform.")
or name.startswith("_routed_output_transform.")
)
if not name.startswith(NON_EXPERT_PREFIXES)
and name not in NON_EXPERT_WEIGHTS
)
@@ -1504,12 +1508,7 @@ class FusedMoE(PluggableLayer):
for name, weight in weights
if name not in NON_EXPERT_WEIGHTS
and weight.shape != torch.Size([])
and not name.startswith("_shared_experts.")
# exclude parameters from non-expert submodules,
# e.g. gate/shared/transforms.
and not name.startswith("_gate.")
and not name.startswith("_routed_input_transform.")
and not name.startswith("_routed_output_transform.")
and not name.startswith(NON_EXPERT_PREFIXES)
]
def set_eplb_state(
@@ -220,7 +220,7 @@ class MoERunner(MoERunnerInterface):
self.routed_output_transform = routed_output_transform
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.quant_method = quant_method
self._quant_method = quant_method
self.enable_dbo = enable_dbo
self._shared_experts: SharedExperts | None = None
@@ -263,7 +263,7 @@ class MoERunner(MoERunnerInterface):
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
if self._shared_experts is not None:
self._shared_experts._quant_method = quant_method
self.quant_method = quant_method
self._quant_method = quant_method
def is_internal_router(self) -> bool:
return self.gate is not None
@@ -330,8 +330,8 @@ class MoERunner(MoERunnerInterface):
@property
def _fused_output_is_reduced(self) -> bool:
return (
self.quant_method.moe_kernel is not None
and self.quant_method.moe_kernel.output_is_reduced()
self._quant_method.moe_kernel is not None
and self._quant_method.moe_kernel.output_is_reduced()
)
def _maybe_reduce_shared_expert_output(
@@ -407,7 +407,7 @@ class MoERunner(MoERunnerInterface):
)
transformed_hidden_dim = hidden_states.shape[-1]
if (
not self.quant_method.skip_forward_padding
not self._quant_method.skip_forward_padding
and self.moe_config.hidden_dim != transformed_hidden_dim
):
hidden_states = F.pad(
@@ -451,8 +451,8 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
if self.quant_method.is_monolithic:
fused_out = self.quant_method.apply_monolithic(
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
x=hidden_states,
router_logits=router_logits,
@@ -467,7 +467,7 @@ class MoERunner(MoERunnerInterface):
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self.quant_method.apply(
fused_out = self._quant_method.apply(
layer=layer,
x=hidden_states,
topk_weights=topk_weights,
@@ -618,7 +618,7 @@ class MoERunner(MoERunnerInterface):
@property
def do_naive_dispatch_combine(self) -> bool:
return (
self.moe_config.dp_size > 1 and not self.quant_method.supports_internal_mk
self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk
)
def _maybe_dispatch(
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
import torch
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
@@ -12,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
)
class MoERunnerInterface(ABC):
class MoERunnerInterface(PluggableLayer, ABC):
"""
Abstract base class for Mixture of Experts (MoE) runners.