diff --git a/docs/mkdocs/hooks/generate_argparse.py b/docs/mkdocs/hooks/generate_argparse.py index a798b8e345..3266b80e5d 100644 --- a/docs/mkdocs/hooks/generate_argparse.py +++ b/docs/mkdocs/hooks/generate_argparse.py @@ -38,8 +38,20 @@ class MockCustomOp: return decorator +class MockPluggableLayer: + @staticmethod + def register(name): + def decorator(cls): + return cls + + return decorator + + mock_if_no_torch("vllm._C", MagicMock()) -mock_if_no_torch("vllm.model_executor.custom_op", MagicMock(CustomOp=MockCustomOp)) +mock_if_no_torch( + "vllm.model_executor.custom_op", + MagicMock(CustomOp=MockCustomOp, PluggableLayer=MockPluggableLayer), +) mock_if_no_torch( "vllm.utils.torch_utils", MagicMock(direct_register_custom_op=lambda *a, **k: None) ) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 7174cdd88f..577f9a9867 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -1487,15 +1487,19 @@ class FusedMoE(PluggableLayer): "w2_input_scale", } + # Parameters of non-expert submodules that live inside runner (MoERunner). + # These must be excluded from EPLB weight rearrangement. + NON_EXPERT_PREFIXES = ( + "runner._shared_experts.", + "runner.gate.", + "runner.routed_input_transform.", + "runner.routed_output_transform.", + ) + assert all( weight.is_contiguous() for name, weight in weights - if not ( - name.startswith("_shared_experts.") - or name.startswith("_gate.") - or name.startswith("_routed_input_transform.") - or name.startswith("_routed_output_transform.") - ) + if not name.startswith(NON_EXPERT_PREFIXES) and name not in NON_EXPERT_WEIGHTS ) @@ -1504,12 +1508,7 @@ class FusedMoE(PluggableLayer): for name, weight in weights if name not in NON_EXPERT_WEIGHTS and weight.shape != torch.Size([]) - and not name.startswith("_shared_experts.") - # exclude parameters from non-expert submodules, - # e.g. gate/shared/transforms. - and not name.startswith("_gate.") - and not name.startswith("_routed_input_transform.") - and not name.startswith("_routed_output_transform.") + and not name.startswith(NON_EXPERT_PREFIXES) ] def set_eplb_state( diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index bf8641b060..2eee8acf6b 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -220,7 +220,7 @@ class MoERunner(MoERunnerInterface): self.routed_output_transform = routed_output_transform self.routed_scaling_factor = routed_scaling_factor self.gate = gate - self.quant_method = quant_method + self._quant_method = quant_method self.enable_dbo = enable_dbo self._shared_experts: SharedExperts | None = None @@ -263,7 +263,7 @@ class MoERunner(MoERunnerInterface): def _replace_quant_method(self, quant_method: FusedMoEMethodBase): if self._shared_experts is not None: self._shared_experts._quant_method = quant_method - self.quant_method = quant_method + self._quant_method = quant_method def is_internal_router(self) -> bool: return self.gate is not None @@ -330,8 +330,8 @@ class MoERunner(MoERunnerInterface): @property def _fused_output_is_reduced(self) -> bool: return ( - self.quant_method.moe_kernel is not None - and self.quant_method.moe_kernel.output_is_reduced() + self._quant_method.moe_kernel is not None + and self._quant_method.moe_kernel.output_is_reduced() ) def _maybe_reduce_shared_expert_output( @@ -407,7 +407,7 @@ class MoERunner(MoERunnerInterface): ) transformed_hidden_dim = hidden_states.shape[-1] if ( - not self.quant_method.skip_forward_padding + not self._quant_method.skip_forward_padding and self.moe_config.hidden_dim != transformed_hidden_dim ): hidden_states = F.pad( @@ -451,8 +451,8 @@ class MoERunner(MoERunnerInterface): shared_experts_input, SharedExpertsOrder.NO_OVERLAP ) - if self.quant_method.is_monolithic: - fused_out = self.quant_method.apply_monolithic( + if self._quant_method.is_monolithic: + fused_out = self._quant_method.apply_monolithic( layer=layer, x=hidden_states, router_logits=router_logits, @@ -467,7 +467,7 @@ class MoERunner(MoERunnerInterface): # Passing shared_experts_input in case SharedExpertsOrder is # MK_INTERNAL_OVERLAPPED. - fused_out = self.quant_method.apply( + fused_out = self._quant_method.apply( layer=layer, x=hidden_states, topk_weights=topk_weights, @@ -618,7 +618,7 @@ class MoERunner(MoERunnerInterface): @property def do_naive_dispatch_combine(self) -> bool: return ( - self.moe_config.dp_size > 1 and not self.quant_method.supports_internal_mk + self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk ) def _maybe_dispatch( diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py index 9a6c37aa39..e3b239ca60 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod import torch +from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) @@ -12,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( ) -class MoERunnerInterface(ABC): +class MoERunnerInterface(PluggableLayer, ABC): """ Abstract base class for Mixture of Experts (MoE) runners.