[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -1,28 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Iterable
from contextlib import nullcontext
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.distributed import (
get_ep_group,
get_pcp_group,
tensor_model_parallel_all_reduce,
)
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import (
ForwardContext,
get_forward_context,
is_forward_context_available,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
@@ -43,8 +54,23 @@ from vllm.utils.torch_utils import (
direct_register_custom_op,
)
logger = init_logger(__name__)
def get_layer_from_name(layer_name: str) -> torch.nn.Module:
def register_layer_for_moe_forward_op(
vllm_config: VllmConfig,
layer: "MoERunner",
):
# For smuggling this layer into the fused moe custom op
prefix = layer.layer_name
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError("Duplicate layer name: {}".format(prefix))
compilation_config.static_forward_context[prefix] = layer
compilation_config.static_all_moe_layers.append(prefix)
def get_layer_from_name(layer_name: str) -> MoERunnerInterface:
forward_context: ForwardContext = get_forward_context()
if not _USE_LAYERNAME and layer_name == "from_forward_context":
all_moe_layers = forward_context.all_moe_layers
@@ -58,7 +84,9 @@ def get_layer_from_name(layer_name: str) -> torch.nn.Module:
)
layer_name = all_moe_layers[moe_layer_index]
forward_context.moe_layer_index += 1
return forward_context.no_compile_layers[layer_name]
layer = forward_context.no_compile_layers[layer_name]
assert isinstance(layer, MoERunnerInterface)
return layer
# On torch >= 2.11, layer_name is a hoisted LayerName opaque object;
@@ -96,8 +124,7 @@ def _moe_forward(
hidden_dim_unpadded: int,
) -> torch.Tensor:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -131,8 +158,7 @@ def _moe_forward_shared(
hidden_dim_unpadded: int,
) -> tuple[torch.Tensor, torch.Tensor]:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -220,12 +246,12 @@ class MoERunner(MoERunnerInterface):
layer_name: str,
moe_config: FusedMoEConfig,
router: FusedMoERouter,
routed_input_transform: torch.nn.Module | None,
gate: torch.nn.Module | None,
shared_experts: torch.nn.Module | None,
quant_method: FusedMoEMethodBase,
enable_dbo: bool,
routed_experts: RoutedExperts,
enable_dbo: bool = False,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
shared_expert_gate: torch.nn.Module | None = None,
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
routed_scaling_factor: float = 1.0,
):
@@ -237,7 +263,7 @@ class MoERunner(MoERunnerInterface):
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.shared_expert_gate = shared_expert_gate
self._quant_method = quant_method
self.routed_experts = routed_experts
self.enable_dbo = enable_dbo
# When both gates are present and FSE is enabled, fuse their
@@ -250,23 +276,22 @@ class MoERunner(MoERunnerInterface):
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
self._shared_experts = SharedExperts(
shared_experts,
moe_config=moe_config,
# Note: For now we must pass quant_method along to SharedExperts so it
# can property determine where the shared experts are supposed to be
# called, i.e. by a MK or by the MoERunner.
# Once the MK can be created upfront, we can just pass in the proper
# flags derived from the quant_method's MK.
quant_method=quant_method,
enable_dbo=enable_dbo,
mk_can_overlap_shared_experts=can_overlap,
)
# Needed for string -> FusedMoE layer lookup in custom ops.
# Needed for string -> MoERunner layer lookup in custom ops.
self.layer_name = layer_name
self._forward_entry = self._select_forward()
# For smuggling this layer into the fused moe custom op
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we
@@ -284,11 +309,20 @@ class MoERunner(MoERunnerInterface):
def shared_experts(self) -> SharedExperts | None:
return self._shared_experts
# TODO(bnell): temporary hack, do not call this method.
@property
def is_internal_router(self) -> bool:
return self.gate is not None
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.routed_experts._replace_quant_method(quant_method)
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
self.routed_experts._set_moe_config(new_moe_config)
if self._shared_experts is not None:
self._shared_experts._quant_method = quant_method
self._quant_method = quant_method
self._shared_experts._set_moe_config(new_moe_config)
def _maybe_fuse_gate_weights(self):
"""Fuse router and shared expert gate weights on first call.
@@ -304,8 +338,9 @@ class MoERunner(MoERunnerInterface):
dim=0,
)
def is_internal_router(self) -> bool:
return self.gate is not None
@property
def _quant_method(self) -> FusedMoEMethodBase:
return self.routed_experts.quant_method
def apply_routed_input_transform(
self, hidden_states: torch.Tensor
@@ -396,9 +431,9 @@ class MoERunner(MoERunnerInterface):
def _maybe_reduce_final_output(
self,
states: torch.Tensor,
trunc_size: int,
trunc_size: int | None,
) -> torch.Tensor:
"""Truncate padded dimensions and all-reduce the combined output.
"""All-reduce the combined output if needed.
This is the "late" all-reduce path. When neither fused nor shared
output was individually reduced, the combined sum is all-reduced
@@ -415,7 +450,7 @@ class MoERunner(MoERunnerInterface):
):
states = tensor_model_parallel_all_reduce(states)
return states[..., :trunc_size]
return states[..., :trunc_size] if trunc_size is not None else states
def _encode_layer_name(self) -> str | LayerName:
if _USE_LAYERNAME:
@@ -428,34 +463,11 @@ class MoERunner(MoERunnerInterface):
return "from_forward_context"
return self.layer_name
def _trtllm_mxfp4_unpadded_dim(self) -> int:
"""Return ``hidden_dim_unpadded`` when the active backend is TRT-LLM
MXFP4 (whose kernel writes narrower than the padded
``hidden_states.shape[-1]``), else 0. Other MXFP4 backends (notably
Cutlass MXFP4 MXFP8) write the full padded width, so
``moe_config.hidden_dim_unpadded`` alone is insufficient: it encodes
the model's logical hidden, not whether the kernel narrows. Computed
caller-side and passed as an op arg; doing the isinstance check
inside the fake would specialize per ``layer_name`` and break
subgraph dedup for identical-architecture models (e.g. Phi-MoE).
"""
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsBase,
)
moe_kernel = getattr(self._quant_method, "moe_kernel", None)
fused_experts = getattr(
getattr(moe_kernel, "impl", None), "fused_experts", None
)
if isinstance(fused_experts, TrtLlmMxfp4ExpertsBase):
return self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
return 0
def _maybe_pad_hidden_states(
self,
shared_experts_input: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, int]:
) -> tuple[torch.Tensor, int | None, int | None]:
"""Pad hidden_states to moe_config.hidden_dim and compute the
original dimension for later truncation.
@@ -467,11 +479,12 @@ class MoERunner(MoERunnerInterface):
shared_experts_hidden_dim = (
shared_experts_input.shape[-1] if shared_experts_input is not None else 0
)
transformed_hidden_dim = hidden_states.shape[-1]
transformed_hidden_dim: int | None = hidden_states.shape[-1]
if (
not self._quant_method.skip_forward_padding
and self.moe_config.hidden_dim != transformed_hidden_dim
):
assert transformed_hidden_dim is not None
hidden_states = F.pad(
hidden_states,
(0, self.moe_config.hidden_dim - transformed_hidden_dim),
@@ -479,12 +492,33 @@ class MoERunner(MoERunnerInterface):
value=0.0,
)
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
orig_hidden_dims = shared_experts_hidden_dim
else:
orig_hidden_dims = transformed_hidden_dim
# Truncation sizes for stripping kernel padding from the output.
# None means no truncation needed (no padding was applied).
#
# Two truncation points exist in forward():
# pre_xform: applied to fused_output BEFORE routed_output_transform
# post_xform: applied to the final result AFTER all-reduce
#
# Latent MoE with shared experts (NemotronH):
# - pre_xform strips padding from the latent dim so
# routed_output_transform receives the correct input size
# - post_xform truncates to shared_experts_hidden_dim (full hidden)
# after shared + routed outputs are combined and all-reduced
#
# Standard MoE / MoE without transforms (GPT-OSS, Mixtral):
# - pre_xform is None (no early truncation)
# - post_xform strips padding after all-reduce (or None if unpadded)
if transformed_hidden_dim == hidden_states.shape[-1]:
transformed_hidden_dim = None
return hidden_states, orig_hidden_dims
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
pre_xform_trunc_size = transformed_hidden_dim
post_xform_trunc_size = shared_experts_hidden_dim
else:
pre_xform_trunc_size = None
post_xform_trunc_size = transformed_hidden_dim
return hidden_states, pre_xform_trunc_size, post_xform_trunc_size
def _maybe_apply_shared_experts(
self,
@@ -493,11 +527,10 @@ class MoERunner(MoERunnerInterface):
):
if self._shared_experts is not None:
assert shared_experts_input is not None
self._shared_experts.apply(shared_experts_input, order)
self._shared_experts(shared_experts_input, order)
def _apply_quant_method(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -513,24 +546,23 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
if self.routed_experts.quant_method.is_monolithic:
# Monolithic kernels: pass router_logits to routed_experts
fused_out = self.routed_experts.forward_monolithic(
x=hidden_states,
router_logits=router_logits,
input_ids=input_ids,
)
else:
# Modular kernels: select experts first, then call routed_experts
topk_weights, topk_ids = self.router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
topk_indices_dtype=self._quant_method.topk_indices_dtype,
input_ids=input_ids,
)
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(
layer=layer,
fused_out = self.routed_experts.forward_modular(
x=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
@@ -627,12 +659,13 @@ class MoERunner(MoERunnerInterface):
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
# so routed output can be trimmed before
# shared+routed add / latent up proj if needed.
routed_hidden_dim = hidden_states.shape[-1]
hidden_states, og_hidden_dim = self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
)
)
hidden_dim_was_padded = hidden_states.shape[-1] > routed_hidden_dim
result = self._forward_entry(
hidden_states,
@@ -640,7 +673,9 @@ class MoERunner(MoERunnerInterface):
shared_experts_input,
input_ids,
self._encode_layer_name(),
self._trtllm_mxfp4_unpadded_dim(),
self.moe_config.hidden_dim_unpadded
if self._quant_method.has_unpadded_output
else 0,
)
#
@@ -654,10 +689,9 @@ class MoERunner(MoERunnerInterface):
# Extract outputs from result
shared_output, fused_output = _unpack(result)
if (
shared_output is not None or self.routed_output_transform is not None
) and hidden_dim_was_padded:
fused_output = fused_output[..., :routed_hidden_dim]
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
# If combine kernel already reduced fused, reduce shared to match.
# See note above re: the two all-reduce points.
@@ -675,7 +709,7 @@ class MoERunner(MoERunnerInterface):
else:
result = fused_output
result = self._maybe_reduce_final_output(result, og_hidden_dim)
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
return self._maybe_add_zero_expert_output(result)
@@ -687,7 +721,6 @@ class MoERunner(MoERunnerInterface):
def _maybe_dispatch(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -723,7 +756,7 @@ class MoERunner(MoERunnerInterface):
self,
shared_output: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
) -> torch.Tensor | tuple[torch.Tensor | None, torch.Tensor]:
if self.do_naive_dispatch_combine:
hidden_states = get_ep_group().combine(
hidden_states, self.moe_config.is_sequence_parallel
@@ -743,7 +776,6 @@ class MoERunner(MoERunnerInterface):
def _forward_impl(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -762,7 +794,7 @@ class MoERunner(MoERunnerInterface):
Returns a single tensor of combined fused and shared output (if present).
"""
# TODO(bnell): this can be removed after MK migration is complete.
layer.ensure_moe_quant_config_init()
self.routed_experts._ensure_moe_quant_config_init()
# Sync aux and main stream for shared expert multi-stream overlap.
self._maybe_sync_shared_experts_stream(shared_experts_input)
@@ -782,13 +814,11 @@ class MoERunner(MoERunnerInterface):
# #32567 lands and the remaining kernels are made MKs. The PCP
# code will probably remain
hidden_states, router_logits = self._maybe_dispatch(
layer,
hidden_states,
router_logits,
)
shared_output, hidden_states = self._apply_quant_method(
layer=layer,
hidden_states=hidden_states,
router_logits=router_logits,
shared_experts_input=shared_experts_input,
@@ -799,3 +829,148 @@ class MoERunner(MoERunnerInterface):
shared_output,
hidden_states,
)
#########################################################
#
# Old methods from FusedMoE layer. Remove when possible.
#
#########################################################
# Note: maybe_init_modular_kernel should only be called by
# prepare_communication_buffer_for_model.
# This is called after all weight loading and post-processing, so it
# should be safe to swap out the quant_method.
def maybe_init_modular_kernel(self) -> None:
# NOTE(rob): WIP refactor. For quant methods that own the MK
# we create the MK during process_weights_after_loading.
if (
self.routed_experts.quant_method.supports_internal_mk
or self.routed_experts.quant_method.is_monolithic
):
return None
self.routed_experts._ensure_moe_quant_config_init()
# routing_tables only needed for round-robin expert placement with
# DeepEP all2all backend.
routing_tables = self._expert_routing_tables()
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
base_quant_method = self.routed_experts.quant_method.old_quant_method
else:
base_quant_method = self.routed_experts.quant_method
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
routing_tables=routing_tables
)
if prepare_finalize is not None:
logger.debug(
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
)
self._replace_quant_method(
FusedMoEModularMethod.make(
self.routed_experts,
base_quant_method,
prepare_finalize,
)
)
#
# Properties
#
@property
def layer_id(self):
# Delayed import to avoid circular dependency
from vllm.model_executor.models.utils import extract_layer_index
return extract_layer_index(self.layer_name)
#
# Attributes still needed by models
#
@property
def is_monolithic(self) -> bool:
return self.routed_experts.quant_method.is_monolithic
@property
def activation(self) -> MoEActivation:
return self.routed_experts.activation
#
# Expert maps
#
@property
def expert_map_manager(self):
"""Forward to routed_experts.expert_map_manager for backward compatibility."""
return self.routed_experts.expert_map_manager
@property
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
return self.expert_map_manager.placement_strategy
@property
def expert_global_to_physical(self) -> torch.Tensor | None:
tables = self.expert_map_manager.routing_tables
return tables[0] if tables else None
@property
def expert_physical_to_global(self) -> torch.Tensor | None:
"""Routing table: physical expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[1] if tables else None
@property
def expert_local_to_global(self) -> torch.Tensor | None:
"""Routing table: local expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[2] if tables else None
@property
def expert_map(self) -> torch.Tensor | None:
return self.routed_experts.expert_map
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
return self.routed_experts._expert_routing_tables()
def update_expert_map(self):
self.routed_experts.update_expert_map()
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
"""Map global expert ID to local expert ID."""
return self.routed_experts._map_global_expert_id_to_local_expert_id(expert_id)
def get_expert_weights(self) -> Iterable[torch.Tensor]:
return self.routed_experts.get_expert_weights()
#
# EPLB
#
@property
def eplb_state(self) -> EplbLayerState | None:
return self.router.eplb_state
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
"""
Register the EPLB state in this layer.
This is used later in forward pass, where we get the expert mapping
and record the load metrics in `expert_load_view`.
"""
if self.router.eplb_state is not None:
self.router.eplb_state.set_layer_state(
moe_layer_idx,
expert_load_view,
logical_to_physical_map,
logical_replica_count,
)
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Iterable
import torch
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
@@ -22,6 +25,11 @@ class MoERunnerInterface(PluggableLayer, ABC):
expert routing, and managing tensor parallel operations.
"""
def __init__(self):
super().__init__()
# HACK
self._already_called_process_weights_after_loading = True
@abstractmethod
def forward(
self,
@@ -31,16 +39,108 @@ class MoERunnerInterface(PluggableLayer, ABC):
) -> torch.Tensor:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
raise NotImplementedError
@property
@abstractmethod
def is_internal_router(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
def _quant_method(self) -> FusedMoEMethodBase:
raise NotImplementedError
# TODO(bnell): temporary hack, do not call this method.
# Temporary hack
@abstractmethod
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
raise NotImplementedError
########################################################################
#
# FusedMoE layer methods
#
########################################################################
@abstractmethod
def maybe_init_modular_kernel(self) -> None:
raise NotImplementedError
@property
@abstractmethod
def layer_id(self):
raise NotImplementedError
#
# Attributes still needed by models
#
@property
@abstractmethod
def is_monolithic(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def activation(self) -> MoEActivation:
raise NotImplementedError
#
# Expert maps
#
@property
@abstractmethod
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
raise NotImplementedError
@property
@abstractmethod
def expert_global_to_physical(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_physical_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_local_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_map(self) -> torch.Tensor | None:
raise NotImplementedError
@abstractmethod
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
raise NotImplementedError
@abstractmethod
def update_expert_map(self):
raise NotImplementedError
@abstractmethod
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
raise NotImplementedError
@abstractmethod
def get_expert_weights(self) -> Iterable[torch.Tensor]:
raise NotImplementedError
@abstractmethod
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
raise NotImplementedError
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from enum import IntEnum
import torch
@@ -9,9 +10,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
aux_stream,
@@ -38,21 +36,15 @@ class SharedExpertsOrder(IntEnum):
MULTI_STREAM_OVERLAPPED = (3,)
class SharedExperts:
class SharedExperts(torch.nn.Module):
def __init__(
self,
layer: torch.nn.Module,
moe_config: FusedMoEConfig,
quant_method: QuantizeMethodBase,
enable_dbo: bool,
mk_can_overlap_shared_experts: Callable[[], bool],
):
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
# quant_method must be a FusedMoEMethodBase but we can't use the type
# due to circular imports.
assert isinstance(quant_method, FusedMoEMethodBase)
super().__init__()
# The SharedExperts need to handle DBO since they can be called from
# an MK's finalize method. We keep a list of outputs indexed by current
@@ -62,7 +54,8 @@ class SharedExperts:
self._output: list[torch.Tensor | None] = [None, None]
self._layer = layer
self._moe_config = moe_config
self._quant_method = quant_method
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
# Allow disabling of the separate shared experts stream for
# debug purposes.
@@ -78,6 +71,10 @@ class SharedExperts:
if self._stream is not None:
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
@property
def _disable_shared_experts_overlap(self) -> bool:
# Disable shared expert overlap if:
@@ -96,7 +93,7 @@ class SharedExperts:
if self._disable_shared_experts_overlap:
return SharedExpertsOrder.NO_OVERLAP
if self._quant_method.mk_can_overlap_shared_experts:
if self._mk_can_overlap_shared_experts():
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
should_run_shared_in_aux_stream = (
@@ -155,7 +152,7 @@ class SharedExperts:
self._output[self._output_idx] = None
return output
def apply(
def forward(
self,
shared_experts_input: torch.Tensor,
order: SharedExpertsOrder,