[PluggableLayer][3/N] Apply PluggableLayer to moe-related layers. (#33556)
Signed-off-by: whx-sjtu <2952154980@qq.com>
This commit is contained in:
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState, EplbState
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -213,8 +213,8 @@ def get_compressed_expert_map(expert_map: torch.Tensor) -> str:
|
||||
|
||||
|
||||
# --8<-- [start:fused_moe]
|
||||
@CustomOp.register("fused_moe")
|
||||
class FusedMoE(CustomOp):
|
||||
@PluggableLayer.register("fused_moe")
|
||||
class FusedMoE(PluggableLayer):
|
||||
"""FusedMoE layer for MoE models.
|
||||
|
||||
This layer contains both MergedColumnParallel weights (gate_up_proj /
|
||||
@@ -1532,7 +1532,7 @@ class FusedMoE(CustomOp):
|
||||
"""
|
||||
return self.runner.maybe_all_reduce_tensor_model_parallel(final_hidden_states)
|
||||
|
||||
def forward_native(
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
@@ -1548,13 +1548,6 @@ class FusedMoE(CustomOp):
|
||||
self._expert_map if not self.rocm_aiter_fmoe_enabled else self.expert_mask
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.forward_native(hidden_states, router_logits)
|
||||
|
||||
@classmethod
|
||||
def make_expert_params_mapping(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user