[MoE] Make MoERunnerInterface a PluggableLayer for OOT support (#35178)
Signed-off-by: wxsIcey <1790571317@qq.com> Signed-off-by: Icey <1790571317@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cyrus Leung
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3179e53135
commit
b55b26520c
@@ -38,8 +38,20 @@ class MockCustomOp:
|
||||
return decorator
|
||||
|
||||
|
||||
class MockPluggableLayer:
|
||||
@staticmethod
|
||||
def register(name):
|
||||
def decorator(cls):
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
mock_if_no_torch("vllm._C", MagicMock())
|
||||
mock_if_no_torch("vllm.model_executor.custom_op", MagicMock(CustomOp=MockCustomOp))
|
||||
mock_if_no_torch(
|
||||
"vllm.model_executor.custom_op",
|
||||
MagicMock(CustomOp=MockCustomOp, PluggableLayer=MockPluggableLayer),
|
||||
)
|
||||
mock_if_no_torch(
|
||||
"vllm.utils.torch_utils", MagicMock(direct_register_custom_op=lambda *a, **k: None)
|
||||
)
|
||||
|
||||
@@ -1487,15 +1487,19 @@ class FusedMoE(PluggableLayer):
|
||||
"w2_input_scale",
|
||||
}
|
||||
|
||||
# Parameters of non-expert submodules that live inside runner (MoERunner).
|
||||
# These must be excluded from EPLB weight rearrangement.
|
||||
NON_EXPERT_PREFIXES = (
|
||||
"runner._shared_experts.",
|
||||
"runner.gate.",
|
||||
"runner.routed_input_transform.",
|
||||
"runner.routed_output_transform.",
|
||||
)
|
||||
|
||||
assert all(
|
||||
weight.is_contiguous()
|
||||
for name, weight in weights
|
||||
if not (
|
||||
name.startswith("_shared_experts.")
|
||||
or name.startswith("_gate.")
|
||||
or name.startswith("_routed_input_transform.")
|
||||
or name.startswith("_routed_output_transform.")
|
||||
)
|
||||
if not name.startswith(NON_EXPERT_PREFIXES)
|
||||
and name not in NON_EXPERT_WEIGHTS
|
||||
)
|
||||
|
||||
@@ -1504,12 +1508,7 @@ class FusedMoE(PluggableLayer):
|
||||
for name, weight in weights
|
||||
if name not in NON_EXPERT_WEIGHTS
|
||||
and weight.shape != torch.Size([])
|
||||
and not name.startswith("_shared_experts.")
|
||||
# exclude parameters from non-expert submodules,
|
||||
# e.g. gate/shared/transforms.
|
||||
and not name.startswith("_gate.")
|
||||
and not name.startswith("_routed_input_transform.")
|
||||
and not name.startswith("_routed_output_transform.")
|
||||
and not name.startswith(NON_EXPERT_PREFIXES)
|
||||
]
|
||||
|
||||
def set_eplb_state(
|
||||
|
||||
@@ -220,7 +220,7 @@ class MoERunner(MoERunnerInterface):
|
||||
self.routed_output_transform = routed_output_transform
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.gate = gate
|
||||
self.quant_method = quant_method
|
||||
self._quant_method = quant_method
|
||||
self.enable_dbo = enable_dbo
|
||||
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
@@ -263,7 +263,7 @@ class MoERunner(MoERunnerInterface):
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
if self._shared_experts is not None:
|
||||
self._shared_experts._quant_method = quant_method
|
||||
self.quant_method = quant_method
|
||||
self._quant_method = quant_method
|
||||
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
@@ -330,8 +330,8 @@ class MoERunner(MoERunnerInterface):
|
||||
@property
|
||||
def _fused_output_is_reduced(self) -> bool:
|
||||
return (
|
||||
self.quant_method.moe_kernel is not None
|
||||
and self.quant_method.moe_kernel.output_is_reduced()
|
||||
self._quant_method.moe_kernel is not None
|
||||
and self._quant_method.moe_kernel.output_is_reduced()
|
||||
)
|
||||
|
||||
def _maybe_reduce_shared_expert_output(
|
||||
@@ -407,7 +407,7 @@ class MoERunner(MoERunnerInterface):
|
||||
)
|
||||
transformed_hidden_dim = hidden_states.shape[-1]
|
||||
if (
|
||||
not self.quant_method.skip_forward_padding
|
||||
not self._quant_method.skip_forward_padding
|
||||
and self.moe_config.hidden_dim != transformed_hidden_dim
|
||||
):
|
||||
hidden_states = F.pad(
|
||||
@@ -451,8 +451,8 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
)
|
||||
|
||||
if self.quant_method.is_monolithic:
|
||||
fused_out = self.quant_method.apply_monolithic(
|
||||
if self._quant_method.is_monolithic:
|
||||
fused_out = self._quant_method.apply_monolithic(
|
||||
layer=layer,
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -467,7 +467,7 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
# MK_INTERNAL_OVERLAPPED.
|
||||
fused_out = self.quant_method.apply(
|
||||
fused_out = self._quant_method.apply(
|
||||
layer=layer,
|
||||
x=hidden_states,
|
||||
topk_weights=topk_weights,
|
||||
@@ -618,7 +618,7 @@ class MoERunner(MoERunnerInterface):
|
||||
@property
|
||||
def do_naive_dispatch_combine(self) -> bool:
|
||||
return (
|
||||
self.moe_config.dp_size > 1 and not self.quant_method.supports_internal_mk
|
||||
self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk
|
||||
)
|
||||
|
||||
def _maybe_dispatch(
|
||||
|
||||
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
@@ -12,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
)
|
||||
|
||||
|
||||
class MoERunnerInterface(ABC):
|
||||
class MoERunnerInterface(PluggableLayer, ABC):
|
||||
"""
|
||||
Abstract base class for Mixture of Experts (MoE) runners.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user