[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Any, TypeAlias
from typing import Any
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
FusedMoeWeightScaleSupported,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
@@ -28,10 +27,17 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsModular,
FusedMoEPrepareAndFinalizeModular,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
FusedMoeWeightScaleSupported,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fused_moe.runner.moe_runner import (
MoERunner,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
@@ -43,10 +49,6 @@ from vllm.triton_utils import HAS_TRITON
_config: dict[str, Any] | None = None
# Temporary alias for FusedMoE, eventually we be its own class.
RoutedExperts: TypeAlias = FusedMoE
@contextmanager
def override_config(config):
global _config
@@ -74,6 +76,7 @@ __all__ = [
"FusedMoEActivationFormat",
"FusedMoEPrepareAndFinalizeModular",
"GateLinear",
"MoERunner",
"RoutingMethodType",
"RoutedExperts",
"SharedExperts",
+38 -2
View File
@@ -1236,7 +1236,7 @@ class FusedMoEConfig:
num_experts: int
experts_per_token: int
hidden_dim: int
intermediate_size_per_partition: int
intermediate_size: int
num_local_experts: int
num_logical_experts: int
activation: MoEActivation
@@ -1258,7 +1258,6 @@ class FusedMoEConfig:
moe_backend: MoEBackend = "auto"
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
has_bias: bool = False
is_act_and_mul: bool = True
is_lora_enabled: bool = False
# SwiGLU clamp limit. When set, backends that do not implement the clamp
@@ -1268,7 +1267,18 @@ class FusedMoEConfig:
max_capture_size: int = 0
# Set by __post_init__
intermediate_size_per_partition: int = -1
rocm_aiter_fmoe_enabled: bool = False
aiter_fmoe_shared_expert_enabled: bool = False
def __post_init__(self):
from vllm._aiter_ops import rocm_aiter_ops
tp_size = self.moe_parallel_config.tp_size
assert self.intermediate_size % tp_size == 0
self.intermediate_size_per_partition = self.intermediate_size // tp_size
if self.dp_size > 1:
logger.debug_once(
"Using FusedMoEConfig::max_num_tokens=%d", self.max_num_tokens
@@ -1286,6 +1296,32 @@ class FusedMoEConfig:
self.intermediate_size_per_partition
)
if self.is_act_and_mul:
self.rocm_aiter_fmoe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.aiter_fmoe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if self.use_mori_kernels:
assert self.rocm_aiter_fmoe_enabled, (
"Mori needs to be used with aiter fused_moe for now."
)
assert not self.aiter_fmoe_shared_expert_enabled, (
"Mori does not support fusion shared expert now. "
"Turn it off by setting VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0"
)
if not self.is_act_and_mul and not (
current_platform.is_cuda_alike() or current_platform.is_xpu()
):
raise NotImplementedError(
"is_act_and_mul=False is supported only for CUDA, XPU and ROCm for now"
)
@property
def is_act_and_mul(self) -> bool:
return self.activation.is_gated
@property
def tp_size(self):
return self.moe_parallel_config.tp_size
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
def _make_eep_experts(
@@ -58,10 +58,10 @@ def _make_eep_experts(
def make_eep_staged_quant_method(
module: "FusedMoE",
module: "MoERunner",
moe_config: FusedMoEConfig,
) -> FusedMoEMethodBase | None:
quant_method = module.quant_method
quant_method = module._quant_method
if not quant_method.supports_internal_mk:
return None
if getattr(quant_method, "wraps_legacy_quant_method", False):
@@ -53,7 +53,7 @@ class ActivationMethod(IntEnum):
GELU = 1
aiter_topK_meta_data = None
aiter_topK_meta_data: tuple[torch.Tensor, torch.Tensor] | None = None
@lru_cache(maxsize=1)
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
logger = init_logger(__name__)
@@ -51,7 +52,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
@abstractmethod
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -117,7 +118,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
# based on the all2all implementation, select the appropriate
# gemm implementation
@@ -128,7 +129,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
@abstractmethod
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
self, layer: "RoutedExperts"
) -> FusedMoEQuantConfig | None:
raise NotImplementedError
@@ -143,6 +144,14 @@ class FusedMoEMethodBase(QuantizeMethodBase):
"""Whether to skip the padding in the forward before applying the moe method."""
return False
@property
def has_unpadded_output(self) -> bool:
"""
Indicates that the hidden_states output might be the unpadded
hidden_states shape rather than the full padded shape.
"""
return False
@property
def supports_eplb(self) -> bool:
return False
@@ -162,20 +171,44 @@ class FusedMoEMethodBase(QuantizeMethodBase):
def apply(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None",
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
"""
Apply the MoE operation using modular kernels.
Args:
layer: RoutedExperts instance containing weight parameters
x: Input tensor
topk_weights: Expert weights from router
topk_ids: Selected expert IDs from router
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts
"""
raise NotImplementedError
def apply_monolithic(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Apply the MoE operation using monolithic kernels.
Args:
layer: RoutedExperts instance containing weight parameters
x: Input tensor
router_logits: Router logits (routing done internally)
Returns:
Output tensor from routed experts
"""
raise NotImplementedError
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
@@ -20,6 +21,11 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
logger = init_logger(__name__)
@@ -43,7 +49,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
@staticmethod
def make(
moe_layer: torch.nn.Module,
routed_experts: "RoutedExperts",
old_quant_method: FusedMoEMethodBase,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
) -> "FusedMoEModularMethod":
@@ -51,10 +57,18 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
old_quant_method,
FusedMoEKernel(
prepare_finalize,
old_quant_method.select_gemm_impl(prepare_finalize, moe_layer),
old_quant_method.select_gemm_impl(prepare_finalize, routed_experts),
),
)
@property
def skip_forward_padding(self) -> bool:
return self.old_quant_method.skip_forward_padding
@property
def has_unpadded_output(self) -> bool:
return self.old_quant_method.has_unpadded_output
@property
def supports_eplb(self) -> bool:
return self.old_quant_method.supports_eplb
@@ -65,7 +79,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -75,13 +89,13 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
raise NotImplementedError
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
self, layer: "RoutedExperts"
) -> FusedMoEQuantConfig | None:
return self.moe_quant_config
def apply(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
File diff suppressed because it is too large Load Diff
@@ -1097,8 +1097,9 @@ class FusedMoEKernelModularImpl:
shared_experts_input: torch.Tensor | None,
):
if shared_experts is not None:
assert self.prepare_finalize.supports_async()
assert shared_experts_input is not None
shared_experts.apply(
shared_experts(
shared_experts_input,
SharedExpertsOrder.MK_INTERNAL_OVERLAPPED,
)
@@ -1572,7 +1573,7 @@ class FusedMoEKernel:
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
fp8_w8a8_moe_quant_config,
fp8_w8a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
@@ -423,7 +424,7 @@ def select_fp8_moe_backend(
def convert_to_fp8_moe_kernel_format(
fp8_backend: Fp8MoeBackend,
# TODO(bnell): replace layer with weight_block_size
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -13,6 +13,7 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
@@ -1632,12 +1633,11 @@ def make_mxfp4_moe_quant_config(
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, FusedMoE)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=gemm1_alpha,
@@ -1663,7 +1663,7 @@ def make_mxfp4_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: "RoutedExperts | None" = None,
layer: RoutedExperts | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import (
nvfp4_moe_quant_config,
nvfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
@@ -335,7 +336,7 @@ def select_nvfp4_moe_backend(
def convert_to_nvfp4_moe_kernel_format(
nvfp4_backend: NvFp4MoeBackend,
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w13_scale: torch.Tensor,
w13_scale_2: torch.Tensor,
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
@@ -37,13 +36,11 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
eplb_state: EplbLayerState | None = None,
scoring_func: str = "softmax",
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.renormalize = renormalize
self.scoring_func = scoring_func
@@ -149,27 +149,16 @@ class BaseRouter(FusedMoERouter):
top_k: int,
global_num_experts: int,
eplb_state: EplbLayerState | None = None,
# TODO(bnell): Once the MK is constructed at layer init time, we
# can make this a plain value instead of a callback.
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
"""
Note: the indices dtype might not be available at router construction
time, so we need to supply a callback to get it at runtime. This is
because the indices type is supplied by modular kernels which are
created after MoE layer/router construction.
Args:
top_k: Number of experts to select per token
global_num_experts: Total number of experts
eplb_state: Optional EPLBLayerState for load balancing
indices_type_getter: Optional callback to get indices dtype
"""
super().__init__()
super().__init__(eplb_state=eplb_state)
self.top_k = top_k
self.global_num_experts = global_num_experts
self.eplb_state = eplb_state
self.indices_type_getter = indices_type_getter
self.capture_fn: Callable[[torch.Tensor], None] | None = None
def set_capture_fn(self, capture_fn: Callable[[torch.Tensor], None] | None) -> None:
@@ -189,12 +178,6 @@ class BaseRouter(FusedMoERouter):
if eplb_state.should_record_tensor is None:
raise ValueError("EPLB requires should_record_tensor != None")
def _get_indices_type(self) -> torch.dtype | None:
"""Get the desired indices dtype from the getter function."""
return (
self.indices_type_getter() if self.indices_type_getter is not None else None
)
def _apply_eplb_mapping(self, topk_ids: torch.Tensor) -> torch.Tensor:
"""Apply EPLB mapping to convert logical expert IDs to physical expert IDs."""
if self.eplb_state is not None:
@@ -247,10 +230,11 @@ class BaseRouter(FusedMoERouter):
"""
raise NotImplementedError
def select_experts(
def _select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -260,10 +244,9 @@ class BaseRouter(FusedMoERouter):
This method implements the template method pattern:
1. Validates EPLB state
2. Gets indices type
3. Calls _compute_routing() to get topk_weights and topk_ids
4. Applies EPLB mapping if enabled
5. Converts indices dtype if needed
2. Calls _compute_routing() to get topk_weights and topk_ids
3. Applies EPLB mapping if enabled
4. Converts indices dtype if needed
Returns:
(topk_weights, topk_ids)
@@ -277,22 +260,19 @@ class BaseRouter(FusedMoERouter):
# Step 1: Validate EPLB state
self._validate_eplb_state()
# Step 2: Get indices type.
indices_type = self._get_indices_type()
# Step 3: Compute routing (delegated to subclass)
# Step 2: Compute routing (delegated to subclass)
topk_weights, topk_ids = self._compute_routing(
hidden_states, router_logits, indices_type, input_ids=input_ids
hidden_states, router_logits, topk_indices_dtype, input_ids=input_ids
)
# Capture logical ids before EPLB mapping.
if self.capture_fn is not None:
self.capture_fn(topk_ids)
# Step 4: Apply EPLB mapping
# Step 3: Apply EPLB mapping
topk_ids = self._apply_eplb_mapping(topk_ids)
# Step 5: Convert indices dtype
topk_ids = self._convert_indices_dtype(topk_ids, indices_type)
# Step 4: Convert indices dtype
topk_ids = self._convert_indices_dtype(topk_ids, topk_indices_dtype)
return topk_weights, topk_ids
@@ -19,13 +19,11 @@ class CustomRoutingRouter(BaseRouter):
custom_routing_function: Callable,
eplb_state: EplbLayerState | None = None,
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.custom_routing_function = custom_routing_function
self.renormalize = renormalize
@@ -5,6 +5,7 @@ from collections.abc import Callable
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
@@ -14,6 +15,10 @@ class FusedMoERouter(ABC):
method that is used for routing hidden states based on router logits.
"""
def __init__(self, eplb_state: EplbLayerState | None = None):
self._routing_replay_out: torch.Tensor | None = None
self.eplb_state = eplb_state
@abstractmethod
def set_capture_fn(
self,
@@ -27,10 +32,21 @@ class FusedMoERouter(ABC):
raise NotImplementedError
@abstractmethod
def _select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
raise NotImplementedError
def select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -47,4 +63,19 @@ class FusedMoERouter(ABC):
equivalent to global logical ids, so should be compatible with
plain MoE implementations without redundant experts.
"""
raise NotImplementedError
topk_weights, topk_ids = self._select_experts(
hidden_states,
router_logits,
topk_indices_dtype=topk_indices_dtype,
input_ids=input_ids,
)
# Write routing data for non-monolithic path (Triton, etc.)
# (set by bind_routing_capture_to_model during capturer init)
if self._routing_replay_out is not None:
self._routing_replay_out[: topk_ids.shape[0]].copy_(
topk_ids.to(torch.int16)
)
return topk_weights, topk_ids
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Callable
import torch
import torch.nn.functional as F
@@ -300,7 +299,6 @@ class FusedTopKBiasRouter(BaseRouter):
renormalize: bool = True,
routed_scaling_factor: float = 1.0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
*,
scoring_func: str = "sigmoid",
hash_indices_table: torch.Tensor | None = None,
@@ -309,7 +307,6 @@ class FusedTopKBiasRouter(BaseRouter):
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.e_score_correction_bias = e_score_correction_bias
self.renormalize = renormalize
@@ -123,13 +123,11 @@ class FusedTopKRouter(BaseRouter):
scoring_func: str = "softmax",
renormalize: bool = True,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.renormalize = renormalize
self.scoring_func = scoring_func
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from functools import partial
import torch
@@ -259,13 +258,11 @@ class GroupedTopKRouter(BaseRouter):
e_score_correction_bias: torch.Tensor | None = None,
num_fused_shared_experts: int = 0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.num_expert_group = num_expert_group
self.topk_group = topk_group
@@ -41,7 +41,6 @@ def create_fused_moe_router(
top_k: int,
global_num_experts: int,
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
# grouped topk parameters
use_grouped_topk: bool = False,
num_expert_group: int | None = None,
@@ -77,7 +76,6 @@ def create_fused_moe_router(
top_k: Number of experts to select per token
global_num_experts: Total number of experts in the model
renormalize: Whether to renormalize the routing weights
indices_type_getter: Function to get the desired indices dtype
routing_method_type: Optional explicit routing method type
Grouped topk arguments:
@@ -116,7 +114,6 @@ def create_fused_moe_router(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
if zero_expert_type is not None:
@@ -136,7 +133,6 @@ def create_fused_moe_router(
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
indices_type_getter=indices_type_getter,
)
if use_grouped_topk:
@@ -157,7 +153,6 @@ def create_fused_moe_router(
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=num_fused_shared_experts,
indices_type_getter=indices_type_getter,
)
if (
grouped_topk_router.routing_method_type != RoutingMethodType.Unspecified
@@ -179,7 +174,6 @@ def create_fused_moe_router(
eplb_state=eplb_state,
custom_routing_function=custom_routing_function,
renormalize=renormalize,
indices_type_getter=indices_type_getter,
)
assert scoring_func in ["sigmoid", "softmax", "sqrtsoftplus"]
@@ -192,7 +186,6 @@ def create_fused_moe_router(
e_score_correction_bias=e_score_correction_bias,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
indices_type_getter=indices_type_getter,
scoring_func=scoring_func,
hash_indices_table=hash_indices_table,
)
@@ -209,7 +202,6 @@ def create_fused_moe_router(
num_fused_shared_experts=num_fused_shared_experts,
renormalize=renormalize,
scoring_func=scoring_func,
indices_type_getter=indices_type_getter,
)
return FusedTopKRouter(
@@ -218,5 +210,4 @@ def create_fused_moe_router(
eplb_state=eplb_state,
renormalize=renormalize,
scoring_func=scoring_func,
indices_type_getter=indices_type_getter,
)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Any
import torch
@@ -314,13 +313,11 @@ class RoutingSimulatorRouter(BaseRouter):
top_k: int,
global_num_experts: int,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
@property
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
@@ -39,13 +37,11 @@ class ZeroExpertRouter(BaseRouter):
renormalize: bool = False,
routed_scaling_factor: float = 1.0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.e_score_correction_bias = e_score_correction_bias
self.num_logical_experts = num_logical_experts
@@ -1,28 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Iterable
from contextlib import nullcontext
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.distributed import (
get_ep_group,
get_pcp_group,
tensor_model_parallel_all_reduce,
)
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import (
ForwardContext,
get_forward_context,
is_forward_context_available,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
@@ -43,8 +54,23 @@ from vllm.utils.torch_utils import (
direct_register_custom_op,
)
logger = init_logger(__name__)
def get_layer_from_name(layer_name: str) -> torch.nn.Module:
def register_layer_for_moe_forward_op(
vllm_config: VllmConfig,
layer: "MoERunner",
):
# For smuggling this layer into the fused moe custom op
prefix = layer.layer_name
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError("Duplicate layer name: {}".format(prefix))
compilation_config.static_forward_context[prefix] = layer
compilation_config.static_all_moe_layers.append(prefix)
def get_layer_from_name(layer_name: str) -> MoERunnerInterface:
forward_context: ForwardContext = get_forward_context()
if not _USE_LAYERNAME and layer_name == "from_forward_context":
all_moe_layers = forward_context.all_moe_layers
@@ -58,7 +84,9 @@ def get_layer_from_name(layer_name: str) -> torch.nn.Module:
)
layer_name = all_moe_layers[moe_layer_index]
forward_context.moe_layer_index += 1
return forward_context.no_compile_layers[layer_name]
layer = forward_context.no_compile_layers[layer_name]
assert isinstance(layer, MoERunnerInterface)
return layer
# On torch >= 2.11, layer_name is a hoisted LayerName opaque object;
@@ -96,8 +124,7 @@ def _moe_forward(
hidden_dim_unpadded: int,
) -> torch.Tensor:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -131,8 +158,7 @@ def _moe_forward_shared(
hidden_dim_unpadded: int,
) -> tuple[torch.Tensor, torch.Tensor]:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -220,12 +246,12 @@ class MoERunner(MoERunnerInterface):
layer_name: str,
moe_config: FusedMoEConfig,
router: FusedMoERouter,
routed_input_transform: torch.nn.Module | None,
gate: torch.nn.Module | None,
shared_experts: torch.nn.Module | None,
quant_method: FusedMoEMethodBase,
enable_dbo: bool,
routed_experts: RoutedExperts,
enable_dbo: bool = False,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
shared_expert_gate: torch.nn.Module | None = None,
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
routed_scaling_factor: float = 1.0,
):
@@ -237,7 +263,7 @@ class MoERunner(MoERunnerInterface):
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.shared_expert_gate = shared_expert_gate
self._quant_method = quant_method
self.routed_experts = routed_experts
self.enable_dbo = enable_dbo
# When both gates are present and FSE is enabled, fuse their
@@ -250,23 +276,22 @@ class MoERunner(MoERunnerInterface):
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
self._shared_experts = SharedExperts(
shared_experts,
moe_config=moe_config,
# Note: For now we must pass quant_method along to SharedExperts so it
# can property determine where the shared experts are supposed to be
# called, i.e. by a MK or by the MoERunner.
# Once the MK can be created upfront, we can just pass in the proper
# flags derived from the quant_method's MK.
quant_method=quant_method,
enable_dbo=enable_dbo,
mk_can_overlap_shared_experts=can_overlap,
)
# Needed for string -> FusedMoE layer lookup in custom ops.
# Needed for string -> MoERunner layer lookup in custom ops.
self.layer_name = layer_name
self._forward_entry = self._select_forward()
# For smuggling this layer into the fused moe custom op
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we
@@ -284,11 +309,20 @@ class MoERunner(MoERunnerInterface):
def shared_experts(self) -> SharedExperts | None:
return self._shared_experts
# TODO(bnell): temporary hack, do not call this method.
@property
def is_internal_router(self) -> bool:
return self.gate is not None
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.routed_experts._replace_quant_method(quant_method)
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
self.routed_experts._set_moe_config(new_moe_config)
if self._shared_experts is not None:
self._shared_experts._quant_method = quant_method
self._quant_method = quant_method
self._shared_experts._set_moe_config(new_moe_config)
def _maybe_fuse_gate_weights(self):
"""Fuse router and shared expert gate weights on first call.
@@ -304,8 +338,9 @@ class MoERunner(MoERunnerInterface):
dim=0,
)
def is_internal_router(self) -> bool:
return self.gate is not None
@property
def _quant_method(self) -> FusedMoEMethodBase:
return self.routed_experts.quant_method
def apply_routed_input_transform(
self, hidden_states: torch.Tensor
@@ -396,9 +431,9 @@ class MoERunner(MoERunnerInterface):
def _maybe_reduce_final_output(
self,
states: torch.Tensor,
trunc_size: int,
trunc_size: int | None,
) -> torch.Tensor:
"""Truncate padded dimensions and all-reduce the combined output.
"""All-reduce the combined output if needed.
This is the "late" all-reduce path. When neither fused nor shared
output was individually reduced, the combined sum is all-reduced
@@ -415,7 +450,7 @@ class MoERunner(MoERunnerInterface):
):
states = tensor_model_parallel_all_reduce(states)
return states[..., :trunc_size]
return states[..., :trunc_size] if trunc_size is not None else states
def _encode_layer_name(self) -> str | LayerName:
if _USE_LAYERNAME:
@@ -428,34 +463,11 @@ class MoERunner(MoERunnerInterface):
return "from_forward_context"
return self.layer_name
def _trtllm_mxfp4_unpadded_dim(self) -> int:
"""Return ``hidden_dim_unpadded`` when the active backend is TRT-LLM
MXFP4 (whose kernel writes narrower than the padded
``hidden_states.shape[-1]``), else 0. Other MXFP4 backends (notably
Cutlass MXFP4 MXFP8) write the full padded width, so
``moe_config.hidden_dim_unpadded`` alone is insufficient: it encodes
the model's logical hidden, not whether the kernel narrows. Computed
caller-side and passed as an op arg; doing the isinstance check
inside the fake would specialize per ``layer_name`` and break
subgraph dedup for identical-architecture models (e.g. Phi-MoE).
"""
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsBase,
)
moe_kernel = getattr(self._quant_method, "moe_kernel", None)
fused_experts = getattr(
getattr(moe_kernel, "impl", None), "fused_experts", None
)
if isinstance(fused_experts, TrtLlmMxfp4ExpertsBase):
return self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
return 0
def _maybe_pad_hidden_states(
self,
shared_experts_input: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, int]:
) -> tuple[torch.Tensor, int | None, int | None]:
"""Pad hidden_states to moe_config.hidden_dim and compute the
original dimension for later truncation.
@@ -467,11 +479,12 @@ class MoERunner(MoERunnerInterface):
shared_experts_hidden_dim = (
shared_experts_input.shape[-1] if shared_experts_input is not None else 0
)
transformed_hidden_dim = hidden_states.shape[-1]
transformed_hidden_dim: int | None = hidden_states.shape[-1]
if (
not self._quant_method.skip_forward_padding
and self.moe_config.hidden_dim != transformed_hidden_dim
):
assert transformed_hidden_dim is not None
hidden_states = F.pad(
hidden_states,
(0, self.moe_config.hidden_dim - transformed_hidden_dim),
@@ -479,12 +492,33 @@ class MoERunner(MoERunnerInterface):
value=0.0,
)
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
orig_hidden_dims = shared_experts_hidden_dim
else:
orig_hidden_dims = transformed_hidden_dim
# Truncation sizes for stripping kernel padding from the output.
# None means no truncation needed (no padding was applied).
#
# Two truncation points exist in forward():
# pre_xform: applied to fused_output BEFORE routed_output_transform
# post_xform: applied to the final result AFTER all-reduce
#
# Latent MoE with shared experts (NemotronH):
# - pre_xform strips padding from the latent dim so
# routed_output_transform receives the correct input size
# - post_xform truncates to shared_experts_hidden_dim (full hidden)
# after shared + routed outputs are combined and all-reduced
#
# Standard MoE / MoE without transforms (GPT-OSS, Mixtral):
# - pre_xform is None (no early truncation)
# - post_xform strips padding after all-reduce (or None if unpadded)
if transformed_hidden_dim == hidden_states.shape[-1]:
transformed_hidden_dim = None
return hidden_states, orig_hidden_dims
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
pre_xform_trunc_size = transformed_hidden_dim
post_xform_trunc_size = shared_experts_hidden_dim
else:
pre_xform_trunc_size = None
post_xform_trunc_size = transformed_hidden_dim
return hidden_states, pre_xform_trunc_size, post_xform_trunc_size
def _maybe_apply_shared_experts(
self,
@@ -493,11 +527,10 @@ class MoERunner(MoERunnerInterface):
):
if self._shared_experts is not None:
assert shared_experts_input is not None
self._shared_experts.apply(shared_experts_input, order)
self._shared_experts(shared_experts_input, order)
def _apply_quant_method(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -513,24 +546,23 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
if self.routed_experts.quant_method.is_monolithic:
# Monolithic kernels: pass router_logits to routed_experts
fused_out = self.routed_experts.forward_monolithic(
x=hidden_states,
router_logits=router_logits,
input_ids=input_ids,
)
else:
# Modular kernels: select experts first, then call routed_experts
topk_weights, topk_ids = self.router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
topk_indices_dtype=self._quant_method.topk_indices_dtype,
input_ids=input_ids,
)
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(
layer=layer,
fused_out = self.routed_experts.forward_modular(
x=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
@@ -627,12 +659,13 @@ class MoERunner(MoERunnerInterface):
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
# so routed output can be trimmed before
# shared+routed add / latent up proj if needed.
routed_hidden_dim = hidden_states.shape[-1]
hidden_states, og_hidden_dim = self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
)
)
hidden_dim_was_padded = hidden_states.shape[-1] > routed_hidden_dim
result = self._forward_entry(
hidden_states,
@@ -640,7 +673,9 @@ class MoERunner(MoERunnerInterface):
shared_experts_input,
input_ids,
self._encode_layer_name(),
self._trtllm_mxfp4_unpadded_dim(),
self.moe_config.hidden_dim_unpadded
if self._quant_method.has_unpadded_output
else 0,
)
#
@@ -654,10 +689,9 @@ class MoERunner(MoERunnerInterface):
# Extract outputs from result
shared_output, fused_output = _unpack(result)
if (
shared_output is not None or self.routed_output_transform is not None
) and hidden_dim_was_padded:
fused_output = fused_output[..., :routed_hidden_dim]
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
# If combine kernel already reduced fused, reduce shared to match.
# See note above re: the two all-reduce points.
@@ -675,7 +709,7 @@ class MoERunner(MoERunnerInterface):
else:
result = fused_output
result = self._maybe_reduce_final_output(result, og_hidden_dim)
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
return self._maybe_add_zero_expert_output(result)
@@ -687,7 +721,6 @@ class MoERunner(MoERunnerInterface):
def _maybe_dispatch(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -723,7 +756,7 @@ class MoERunner(MoERunnerInterface):
self,
shared_output: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
) -> torch.Tensor | tuple[torch.Tensor | None, torch.Tensor]:
if self.do_naive_dispatch_combine:
hidden_states = get_ep_group().combine(
hidden_states, self.moe_config.is_sequence_parallel
@@ -743,7 +776,6 @@ class MoERunner(MoERunnerInterface):
def _forward_impl(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -762,7 +794,7 @@ class MoERunner(MoERunnerInterface):
Returns a single tensor of combined fused and shared output (if present).
"""
# TODO(bnell): this can be removed after MK migration is complete.
layer.ensure_moe_quant_config_init()
self.routed_experts._ensure_moe_quant_config_init()
# Sync aux and main stream for shared expert multi-stream overlap.
self._maybe_sync_shared_experts_stream(shared_experts_input)
@@ -782,13 +814,11 @@ class MoERunner(MoERunnerInterface):
# #32567 lands and the remaining kernels are made MKs. The PCP
# code will probably remain
hidden_states, router_logits = self._maybe_dispatch(
layer,
hidden_states,
router_logits,
)
shared_output, hidden_states = self._apply_quant_method(
layer=layer,
hidden_states=hidden_states,
router_logits=router_logits,
shared_experts_input=shared_experts_input,
@@ -799,3 +829,148 @@ class MoERunner(MoERunnerInterface):
shared_output,
hidden_states,
)
#########################################################
#
# Old methods from FusedMoE layer. Remove when possible.
#
#########################################################
# Note: maybe_init_modular_kernel should only be called by
# prepare_communication_buffer_for_model.
# This is called after all weight loading and post-processing, so it
# should be safe to swap out the quant_method.
def maybe_init_modular_kernel(self) -> None:
# NOTE(rob): WIP refactor. For quant methods that own the MK
# we create the MK during process_weights_after_loading.
if (
self.routed_experts.quant_method.supports_internal_mk
or self.routed_experts.quant_method.is_monolithic
):
return None
self.routed_experts._ensure_moe_quant_config_init()
# routing_tables only needed for round-robin expert placement with
# DeepEP all2all backend.
routing_tables = self._expert_routing_tables()
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
base_quant_method = self.routed_experts.quant_method.old_quant_method
else:
base_quant_method = self.routed_experts.quant_method
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
routing_tables=routing_tables
)
if prepare_finalize is not None:
logger.debug(
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
)
self._replace_quant_method(
FusedMoEModularMethod.make(
self.routed_experts,
base_quant_method,
prepare_finalize,
)
)
#
# Properties
#
@property
def layer_id(self):
# Delayed import to avoid circular dependency
from vllm.model_executor.models.utils import extract_layer_index
return extract_layer_index(self.layer_name)
#
# Attributes still needed by models
#
@property
def is_monolithic(self) -> bool:
return self.routed_experts.quant_method.is_monolithic
@property
def activation(self) -> MoEActivation:
return self.routed_experts.activation
#
# Expert maps
#
@property
def expert_map_manager(self):
"""Forward to routed_experts.expert_map_manager for backward compatibility."""
return self.routed_experts.expert_map_manager
@property
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
return self.expert_map_manager.placement_strategy
@property
def expert_global_to_physical(self) -> torch.Tensor | None:
tables = self.expert_map_manager.routing_tables
return tables[0] if tables else None
@property
def expert_physical_to_global(self) -> torch.Tensor | None:
"""Routing table: physical expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[1] if tables else None
@property
def expert_local_to_global(self) -> torch.Tensor | None:
"""Routing table: local expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[2] if tables else None
@property
def expert_map(self) -> torch.Tensor | None:
return self.routed_experts.expert_map
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
return self.routed_experts._expert_routing_tables()
def update_expert_map(self):
self.routed_experts.update_expert_map()
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
"""Map global expert ID to local expert ID."""
return self.routed_experts._map_global_expert_id_to_local_expert_id(expert_id)
def get_expert_weights(self) -> Iterable[torch.Tensor]:
return self.routed_experts.get_expert_weights()
#
# EPLB
#
@property
def eplb_state(self) -> EplbLayerState | None:
return self.router.eplb_state
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
"""
Register the EPLB state in this layer.
This is used later in forward pass, where we get the expert mapping
and record the load metrics in `expert_load_view`.
"""
if self.router.eplb_state is not None:
self.router.eplb_state.set_layer_state(
moe_layer_idx,
expert_load_view,
logical_to_physical_map,
logical_replica_count,
)
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Iterable
import torch
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
@@ -22,6 +25,11 @@ class MoERunnerInterface(PluggableLayer, ABC):
expert routing, and managing tensor parallel operations.
"""
def __init__(self):
super().__init__()
# HACK
self._already_called_process_weights_after_loading = True
@abstractmethod
def forward(
self,
@@ -31,16 +39,108 @@ class MoERunnerInterface(PluggableLayer, ABC):
) -> torch.Tensor:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
raise NotImplementedError
@property
@abstractmethod
def is_internal_router(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
def _quant_method(self) -> FusedMoEMethodBase:
raise NotImplementedError
# TODO(bnell): temporary hack, do not call this method.
# Temporary hack
@abstractmethod
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
raise NotImplementedError
########################################################################
#
# FusedMoE layer methods
#
########################################################################
@abstractmethod
def maybe_init_modular_kernel(self) -> None:
raise NotImplementedError
@property
@abstractmethod
def layer_id(self):
raise NotImplementedError
#
# Attributes still needed by models
#
@property
@abstractmethod
def is_monolithic(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def activation(self) -> MoEActivation:
raise NotImplementedError
#
# Expert maps
#
@property
@abstractmethod
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
raise NotImplementedError
@property
@abstractmethod
def expert_global_to_physical(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_physical_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_local_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_map(self) -> torch.Tensor | None:
raise NotImplementedError
@abstractmethod
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
raise NotImplementedError
@abstractmethod
def update_expert_map(self):
raise NotImplementedError
@abstractmethod
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
raise NotImplementedError
@abstractmethod
def get_expert_weights(self) -> Iterable[torch.Tensor]:
raise NotImplementedError
@abstractmethod
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
raise NotImplementedError
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from enum import IntEnum
import torch
@@ -9,9 +10,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
aux_stream,
@@ -38,21 +36,15 @@ class SharedExpertsOrder(IntEnum):
MULTI_STREAM_OVERLAPPED = (3,)
class SharedExperts:
class SharedExperts(torch.nn.Module):
def __init__(
self,
layer: torch.nn.Module,
moe_config: FusedMoEConfig,
quant_method: QuantizeMethodBase,
enable_dbo: bool,
mk_can_overlap_shared_experts: Callable[[], bool],
):
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
# quant_method must be a FusedMoEMethodBase but we can't use the type
# due to circular imports.
assert isinstance(quant_method, FusedMoEMethodBase)
super().__init__()
# The SharedExperts need to handle DBO since they can be called from
# an MK's finalize method. We keep a list of outputs indexed by current
@@ -62,7 +54,8 @@ class SharedExperts:
self._output: list[torch.Tensor | None] = [None, None]
self._layer = layer
self._moe_config = moe_config
self._quant_method = quant_method
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
# Allow disabling of the separate shared experts stream for
# debug purposes.
@@ -78,6 +71,10 @@ class SharedExperts:
if self._stream is not None:
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
@property
def _disable_shared_experts_overlap(self) -> bool:
# Disable shared expert overlap if:
@@ -96,7 +93,7 @@ class SharedExperts:
if self._disable_shared_experts_overlap:
return SharedExpertsOrder.NO_OVERLAP
if self._quant_method.mk_can_overlap_shared_experts:
if self._mk_can_overlap_shared_experts():
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
should_run_shared_in_aux_stream = (
@@ -155,7 +152,7 @@ class SharedExperts:
self._output[self._output_idx] = None
return output
def apply(
def forward(
self,
shared_experts_input: torch.Tensor,
order: SharedExpertsOrder,
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from torch.nn import Module
import vllm.envs as envs
from vllm.logger import init_logger
@@ -38,7 +37,7 @@ from vllm.platforms import current_platform
from vllm.platforms.interface import CpuArchEnum
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
logger = init_logger(__name__)
@@ -80,7 +79,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
@@ -89,7 +88,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -156,7 +155,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def _setup_kernel(
self,
layer: Module,
layer: "RoutedExperts",
w13: torch.Tensor,
w2: torch.Tensor,
) -> None:
@@ -201,7 +200,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
super().process_weights_after_loading(layer)
# Padding the weight for better performance on ROCm.
@@ -16,16 +16,14 @@ from vllm.model_executor.kernels.linear import (
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
@@ -210,7 +210,7 @@ class CompressedTensorsConfig(QuantizationConfig):
)
return None
def _add_fused_moe_to_target_scheme_map(self):
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
@@ -219,10 +219,10 @@ class CompressedTensorsConfig(QuantizationConfig):
"""
if (
"Linear" not in self.target_scheme_map
or "FusedMoE" in self.target_scheme_map
or "RoutedExperts" in self.target_scheme_map
):
return
self.target_scheme_map["FusedMoE"] = self.target_scheme_map["Linear"]
self.target_scheme_map["RoutedExperts"] = self.target_scheme_map["Linear"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "CompressedTensorsConfig":
@@ -10,14 +10,12 @@ import torch
from vllm import envs
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -44,8 +42,9 @@ from vllm.model_executor.parameter import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
if current_platform.is_cuda():
if has_humming() and current_platform.is_cuda():
from humming.dtypes import DataType
from humming.layer import HummingMethod
from humming.schema import (
@@ -155,6 +155,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -487,6 +495,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -234,7 +234,12 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
intermediate_size_per_partition = layer.intermediate_size_per_partition
# Note: The layer has not performed rounding on intermediate_size's at this
# point. Use the unpadded size which won't change.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
assert intermediate_size_per_partition is not None
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
@@ -6,6 +6,7 @@ import torch
import vllm._custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
@@ -286,7 +287,7 @@ def prepare_fp4_layer_for_marlin(
def prepare_nvfp4_moe_layer_for_marlin(
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w13_scale: torch.Tensor,
w13_scale_2: torch.Tensor,
@@ -353,7 +354,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
# WEIGHT SCALES
# Permute scales
def premute_scales(
def permute_scales(
scales: torch.Tensor, g_scales: torch.Tensor, name: str
) -> tuple[torch.Tensor, torch.Tensor]:
scales = scales.to(param_dtype)
@@ -388,8 +389,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
g_scales = g_scales / combined_scale_factor
return scales, g_scales
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
w13_scale, w13_scale_2 = permute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = permute_scales(w2_scale, w2_scale_2, "w2")
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2
@@ -22,7 +22,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.lora.utils import is_moe_model
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import (
LinearBase,
MergedColumnParallelLinear,
@@ -464,13 +464,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
self.target_modules.append(name)
if module.disable_tp:
self.tp_disabled_modules.append(name)
elif isinstance(module, FusedMoE) and hasattr(
elif isinstance(module, RoutedExperts) and hasattr(
module.quant_method, "quant_config"
):
# TODO: support FusedMoE with prequant and 8bit.
if self.pre_quant and self.load_8bit:
raise ValueError(
"Prequant BitsAndBytes 8bit models with FusedMoE "
"Prequant BitsAndBytes 8bit models with RoutedExperts "
"is not supported yet."
)
# Get the corresponding weight name using module name and
@@ -508,7 +508,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
# dimension (dim=-1)
elif isinstance(module, (RowParallelLinear,)):
self.column_sharded_weights_modules.append(name)
elif isinstance(module, FusedMoE):
elif isinstance(module, RoutedExperts):
expert_mapping = self.expert_params_mapping
for exp in expert_mapping:
if exp[-1] == "w2":
@@ -629,7 +629,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
expert_mapping = self.expert_params_mapping
expert_qs_dict = {}
for name, module in model.named_modules():
if not isinstance(module, FusedMoE):
if not isinstance(module, RoutedExperts):
continue
w1_states_lst = []
w2_states_lst = []
@@ -13,7 +13,7 @@ import regex as re
# Matches per-expert weight names like ".experts.42.gate_proj.weight".
# Does NOT match 3D fused-expert names like ".experts.gate_proj.weight"
# (no numeric id) — those are intentionally left unfiltered so the full
# tensor is loaded and sliced later by FusedMoE.weight_loader.
# tensor is loaded and sliced later by RoutedExperts.weight_loader.
_EXPERT_ID_RE = re.compile(r"\.experts\.(\d+)\.")
@@ -13,7 +13,7 @@ import tempfile
import threading
import time
from collections import defaultdict
from collections.abc import Callable, Generator
from collections.abc import Callable, Generator, Iterable
from contextlib import contextmanager
from pathlib import Path
from typing import IO, Any
@@ -1633,3 +1633,108 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
# If there were no matches, return the untouched param name
return name
def maybe_remap_moe_expert_param_name(
name: str,
params_dict: dict[str, torch.nn.Parameter],
) -> str:
"""
Remap MoE expert parameter names to account for routed_experts hierarchy.
This handles the transition from the old FusedMoE structure where weights
were directly in the experts module, to the new MoERunner → RoutedExperts
structure.
Checkpoint weights have names like:
layers.0.mlp.experts.w13_weight
But actual parameters are now:
layers.0.mlp.experts.routed_experts.w13_weight
This function inserts 'routed_experts.' into the path when needed.
Args:
name: Parameter name from checkpoint
params_dict: Dictionary of model parameters (from named_parameters())
Returns:
Remapped parameter name if routed_experts hierarchy exists,
otherwise the original name
"""
# Only remap if this looks like an expert parameter
if ".mlp.experts." not in name:
return name
# Skip if already has routed_experts
if ".mlp.experts.routed_experts." in name:
return name
# Expert parameter patterns to check
expert_param_suffixes = [
"w13_weight",
"w2_weight",
"w13_weight_scale",
"w2_weight_scale",
"w13_input_scale",
"w2_input_scale",
"w13_bias",
"w2_bias",
"w13_scale",
"w2_scale",
"w13_g_idx",
"w2_g_idx",
"w13_qweight",
"w2_qweight",
"w13_qzeros",
"w2_qzeros",
"w13_weight_shape",
"w2_weight_shape",
]
# Check if this is an expert weight parameter
is_expert_param = any(
f".{suffix}" in name or name.endswith(suffix)
for suffix in expert_param_suffixes
)
if not is_expert_param:
return name
# Try inserting routed_experts
new_name = name.replace(".mlp.experts.", ".mlp.experts.routed_experts.")
# Only use the new name if it exists in the model
if new_name in params_dict:
return new_name
# Otherwise return original name (old checkpoint format or different structure)
return name
def remap_moe_expert_weights(
weights: Iterable[tuple[str, torch.Tensor]],
params_dict: dict[str, torch.nn.Parameter],
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""
Wrapper generator that remaps MoE expert parameter names for backward compatibility.
This allows models with custom weight loading to automatically handle both old
and new checkpoint formats without needing model-specific remapping code.
Usage:
params_dict = dict(model.named_parameters())
for name, weight in remap_moe_expert_weights(weights, params_dict):
# name is automatically remapped if needed
param = params_dict[name]
...
Args:
weights: Iterator of (name, tensor) tuples from checkpoint
params_dict: Dictionary of model parameters (from named_parameters())
Yields:
(remapped_name, tensor) tuples
"""
for name, weight in weights:
remapped_name = maybe_remap_moe_expert_param_name(name, params_dict)
yield (remapped_name, weight)
+2 -1
View File
@@ -20,6 +20,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -640,7 +641,7 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers
self.num_expert_groups = config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
+12 -9
View File
@@ -16,6 +16,7 @@ from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -216,7 +217,7 @@ class AriaProjector(nn.Module):
return out
class AriaFusedMoE(FusedMoE):
class AriaRoutedExperts(RoutedExperts):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
) -> None:
@@ -225,13 +226,14 @@ class AriaFusedMoE(FusedMoE):
# up weights for each expert.
# Note: Loading expert weights with quantization is not supported
tp_rank = get_tensor_model_parallel_rank()
tp_size = self.moe_config.tp_size
if shard_id == "w13":
# the shape of loaded_weight is
# (num_experts, hidden_size, 2 * moe_intermediate_size)
if self.tp_size > 1:
if tp_size > 1:
up, gate = loaded_weight.chunk(2, dim=-1)
up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
up_and_gate = torch.cat(
[up_current_rank, gate_current_rank], dim=-1
).transpose(1, 2)
@@ -241,8 +243,8 @@ class AriaFusedMoE(FusedMoE):
elif shard_id == "w2":
# the shape of loaded_weight is
# (num_experts, moe_intermediate_size, hidden_size)
if self.tp_size > 1:
down_current_rank = loaded_weight.chunk(self.tp_size, dim=1)[tp_rank]
if tp_size > 1:
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
param.data.copy_(down_current_rank.transpose(1, 2))
else:
param.data.copy_(loaded_weight.transpose(1, 2))
@@ -278,7 +280,7 @@ class AriaTextMoELayer(nn.Module):
bias=config.mlp_bias,
)
self.experts = AriaFusedMoE(
self.experts = FusedMoE(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_topk,
@@ -286,6 +288,7 @@ class AriaTextMoELayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.experts",
routed_experts_cls=AriaRoutedExperts,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -351,8 +354,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
("experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.w2_weight", "experts.fc2.weight", "w2"),
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -283,7 +283,7 @@ class BailingMoeV25(nn.Module):
self.hidden_size = config.hidden_size
self.quant_config = quant_config
self.num_shared_experts = config.num_shared_experts
self.score_function = getattr(config, "score_function", None)
self.score_function: str | None = getattr(config, "score_function", None)
self.n_group = getattr(config, "n_group", None)
self.topk_group = getattr(config, "topk_group", None)
self.use_grouped_topk = self.n_group is not None and self.topk_group is not None
+17 -19
View File
@@ -17,6 +17,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -73,27 +74,17 @@ class DbrxRouter(nn.Module):
return router_logits
class DbrxExperts(FusedMoE):
class DbrxExperts(RoutedExperts):
def __init__(
self,
*args,
config: DbrxConfig,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
prefix: str = "",
**kwargs,
):
super().__init__(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
intermediate_size=config.ffn_config.ffn_hidden_size,
params_dtype=params_dtype,
renormalize=True,
quant_config=quant_config,
tp_size=get_tensor_model_parallel_world_size(),
prefix=prefix,
)
super().__init__(*args, **kwargs)
self.config = config
self.d_model = config.d_model
self.tp_size = self.moe_config.tp_size
self.intermediate_size = self.config.ffn_config.ffn_hidden_size // self.tp_size
# Define custom weight loader for dbrx model
@@ -168,11 +159,18 @@ class DbrxMoE(nn.Module):
self.router = DbrxRouter(config, self.params_dtype)
self.experts = DbrxExperts(
config=config,
quant_config=quant_config,
self.experts = FusedMoE(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
intermediate_size=config.ffn_config.ffn_hidden_size,
params_dtype=self.params_dtype,
prefix=f"{prefix}.experts",
renormalize=True,
quant_config=quant_config,
tp_size=get_tensor_model_parallel_world_size(),
prefix=prefix,
routed_experts_cls=DbrxExperts,
routed_experts_args={"config": config},
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -1
View File
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -670,7 +671,7 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
self.num_moe_layers = len(moe_layers_indices)
self.num_expert_groups = 1
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
+8 -2
View File
@@ -697,12 +697,18 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
moe_offset = int(name.split(".")[-3])
vision_expert_start_idx = self.config.moe_num_experts[0]
is_text_expert = moe_offset <= vision_expert_start_idx - 1
routed_experts = (
".routed_experts" if ("w13_" in name or "w2_" in name) else ""
)
if is_text_expert:
name = name.replace(".experts.", ".text_experts.")
name = name.replace(
".experts", f".text_experts{routed_experts}"
)
else:
delta = moe_offset - vision_expert_start_idx
name = name.replace(
f".experts.{moe_offset}",
f".vision_experts.{moe_offset - vision_expert_start_idx}",
f".vision_experts{routed_experts}.{delta}",
)
for mapping in expert_params_mapping:
+4
View File
@@ -1486,6 +1486,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
continue
if is_pp_missing_parameter(name, self):
continue
# Skip if name doesn't exist in params_dict (e.g., individual
# expert weights that should have been handled above)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
@@ -33,7 +33,7 @@ from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -215,7 +215,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
self.moe_mlp_layers: list[Glm4MoeLite] = []
example_moe = None
for layer in self.model.layers.values():
+2 -2
View File
@@ -32,7 +32,7 @@ from transformers import PretrainedConfig
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -201,7 +201,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
self.moe_mlp_layers: list[Glm4MoE] = []
example_moe = None
for layer in self.model.layers.values():
+5 -3
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
remap_moe_expert_weights,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
@@ -379,7 +380,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_start = tp_rank * per_rank_intermediate_size
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
for name, weight in weights:
# Use centralized weight remapping for MoE expert parameters
for name, weight in remap_moe_expert_weights(weights, params_dict):
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
@@ -580,8 +582,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
Returns:
Weight dtype string (e.g., "mxfp4", "fp8") or None if not available
"""
if hasattr(self.layers[layer_id].mlp.experts.quant_method, "weight_dtype"):
return self.layers[layer_id].mlp.experts.quant_method.weight_dtype
if hasattr(self.layers[layer_id].mlp.experts._quant_method, "weight_dtype"):
return self.layers[layer_id].mlp.experts._quant_method.weight_dtype
return None
intermediate_size = self.config.intermediate_size
@@ -455,6 +455,8 @@ class GraniteMoeHybridModel(nn.Module):
loaded_params.add(n)
def _load_expert(n, p, name, shard_id, expert_id):
if n not in params_dict:
return
param = params_dict[n]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id)
@@ -518,14 +520,14 @@ class GraniteMoeHybridModel(nn.Module):
)
w1_param, w3_param = p[e].chunk(2, dim=0)
_load_expert(
n.replace(".input_linear.", ".experts.w13_"),
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
w1_param,
w1_name,
shard_id="w1",
expert_id=e,
)
_load_expert(
n.replace(".input_linear.", ".experts.w13_"),
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
w3_param,
w3_name,
shard_id="w3",
@@ -541,7 +543,7 @@ class GraniteMoeHybridModel(nn.Module):
)
w2_param = p[e]
_load_expert(
n.replace(".output_linear.", ".experts.w2_"),
n.replace(".output_linear.", ".experts.routed_experts.w2_"),
w2_param,
w2_name,
shard_id="w2",
@@ -214,11 +214,11 @@ class GraniteMoeSharedModel(nn.Module):
for e in range(p.size(0)):
w1_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w1.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w1.weight",
)
w3_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w3.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w3.weight",
)
w1_param, w3_param = p[e].chunk(2, dim=0)
assert w1_name not in new_weights
@@ -229,7 +229,7 @@ class GraniteMoeSharedModel(nn.Module):
for e in range(p.size(0)):
w2_name = n.replace(
".block_sparse_moe.output_linear.weight",
f".block_sparse_moe.experts.{e}.w2.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w2.weight",
)
w2_param = p[e]
assert w2_name not in new_weights
-1
View File
@@ -229,7 +229,6 @@ class NemotronHMoE(nn.Module):
scoring_func="sigmoid",
e_score_correction_bias=self.gate.e_score_correction_bias,
activation=activation_without_mul(config.mlp_hidden_act),
is_act_and_mul=False, # non-gated MoE
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
+2 -1
View File
@@ -37,6 +37,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -355,7 +356,7 @@ class SarvamMLAMoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
)
def maybe_get_fused_moe(self) -> FusedMoE:
def maybe_get_fused_moe(self) -> MoERunner:
return self.experts
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -1
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
@@ -954,7 +955,7 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
) -> None:
for layer_idx, layer in enumerate(self.moe_layers):
experts = layer.experts
assert isinstance(experts, FusedMoE)
assert isinstance(experts, MoERunner)
# Register the expert weights.
self.expert_weights.append(experts.get_expert_weights())
experts.set_eplb_state(
+63 -27
View File
@@ -16,6 +16,9 @@
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
import torch
@@ -27,6 +30,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
@@ -40,33 +44,22 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class TransformersMoEState:
topk_ids: torch.Tensor | None = None
is_sequence_parallel: bool = False
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersFusedMoE(FusedMoE):
class TransformersFusedMoE(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, **kwargs):
self._topk_ids: torch.Tensor = None
def custom_routing_function(hidden_states, gating_output, topk, renormalize):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = self._topk_ids
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = self.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs["custom_routing_function"] = custom_routing_function
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
super().__init__(*args, **kwargs)
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward(
self,
@@ -78,6 +71,8 @@ class TransformersFusedMoE(FusedMoE):
"""In Transformers `experts.forward` will have this signature.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
hidden_states,
topk_ids.to(torch.int32),
@@ -85,6 +80,18 @@ class TransformersFusedMoE(FusedMoE):
self.layer_name,
)
def _forward_super(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def transformers_moe_forward(
hidden_states: torch.Tensor,
@@ -95,11 +102,8 @@ def transformers_moe_forward(
"""Store the `topk_ids` in the layer and call the actual forward."""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._topk_ids = topk_ids
# Clone hidden_states because it will be mutated in-place in FusedMoE
# TODO(bnell): figure out a way to avoid calling runner directly.
# it is a hack that the weight are being passed via logits.
return self.runner.forward(hidden_states.clone(), topk_weights)
self._moe_state.topk_ids = topk_ids
return self._forward_super(hidden_states, topk_weights)
def transformers_moe_forward_fake(
@@ -189,6 +193,7 @@ class MoEMixin(MixtureOfExperts):
ckpt_up_proj_name=up_proj,
num_experts=num_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
)
)
return expert_mapping
@@ -286,8 +291,33 @@ class MoEMixin(MixtureOfExperts):
if "shared_expert" in mlp_param_name:
self.num_shared_experts = 1
break
# Replace experts module with FusedMoE
fused_experts = TransformersFusedMoE(
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
fused_experts = FusedMoE(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -304,6 +334,12 @@ class MoEMixin(MixtureOfExperts):
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
expert_mapping=expert_mapping,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersFusedMoE,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
@@ -11,12 +11,12 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import compute_aligned_M
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
from vllm.model_executor.layers.quantization.online.mxfp8 import Mxfp8OnlineLinearMethod
@@ -99,12 +99,13 @@ def _extract_data_from_linear_base_module(
def _extract_data_from_fused_moe_module(
m: torch.nn.Module,
m_: torch.nn.Module,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""
Extract weights, weight scales and num_topk from FusedMoE module.
"""
assert isinstance(m, FusedMoE)
assert isinstance(m_, MoERunner)
m = m_.routed_experts
w13 = m.w13_weight
w13_s = (
m.w13_weight_scale_inv
@@ -156,10 +157,11 @@ def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool:
if not (envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM):
return False
if not isinstance(module, FusedMoE):
if not isinstance(module, MoERunner):
return False
moe_quant_config = module.quant_method.get_fused_moe_quant_config(module)
quant_method = module._quant_method
moe_quant_config = quant_method.get_fused_moe_quant_config(module.routed_experts)
if (
moe_quant_config is None
@@ -168,7 +170,7 @@ def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool:
):
return False
moe_kernel = getattr(module.quant_method, "moe_kernel", None)
moe_kernel = getattr(quant_method, "moe_kernel", None)
if moe_kernel is None:
return False