[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Any, TypeAlias
from typing import Any
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
FusedMoeWeightScaleSupported,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
@@ -28,10 +27,17 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsModular,
FusedMoEPrepareAndFinalizeModular,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
FusedMoeWeightScaleSupported,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fused_moe.runner.moe_runner import (
MoERunner,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
@@ -43,10 +49,6 @@ from vllm.triton_utils import HAS_TRITON
_config: dict[str, Any] | None = None
# Temporary alias for FusedMoE, eventually we be its own class.
RoutedExperts: TypeAlias = FusedMoE
@contextmanager
def override_config(config):
global _config
@@ -74,6 +76,7 @@ __all__ = [
"FusedMoEActivationFormat",
"FusedMoEPrepareAndFinalizeModular",
"GateLinear",
"MoERunner",
"RoutingMethodType",
"RoutedExperts",
"SharedExperts",
+38 -2
View File
@@ -1236,7 +1236,7 @@ class FusedMoEConfig:
num_experts: int
experts_per_token: int
hidden_dim: int
intermediate_size_per_partition: int
intermediate_size: int
num_local_experts: int
num_logical_experts: int
activation: MoEActivation
@@ -1258,7 +1258,6 @@ class FusedMoEConfig:
moe_backend: MoEBackend = "auto"
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
has_bias: bool = False
is_act_and_mul: bool = True
is_lora_enabled: bool = False
# SwiGLU clamp limit. When set, backends that do not implement the clamp
@@ -1268,7 +1267,18 @@ class FusedMoEConfig:
max_capture_size: int = 0
# Set by __post_init__
intermediate_size_per_partition: int = -1
rocm_aiter_fmoe_enabled: bool = False
aiter_fmoe_shared_expert_enabled: bool = False
def __post_init__(self):
from vllm._aiter_ops import rocm_aiter_ops
tp_size = self.moe_parallel_config.tp_size
assert self.intermediate_size % tp_size == 0
self.intermediate_size_per_partition = self.intermediate_size // tp_size
if self.dp_size > 1:
logger.debug_once(
"Using FusedMoEConfig::max_num_tokens=%d", self.max_num_tokens
@@ -1286,6 +1296,32 @@ class FusedMoEConfig:
self.intermediate_size_per_partition
)
if self.is_act_and_mul:
self.rocm_aiter_fmoe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.aiter_fmoe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if self.use_mori_kernels:
assert self.rocm_aiter_fmoe_enabled, (
"Mori needs to be used with aiter fused_moe for now."
)
assert not self.aiter_fmoe_shared_expert_enabled, (
"Mori does not support fusion shared expert now. "
"Turn it off by setting VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0"
)
if not self.is_act_and_mul and not (
current_platform.is_cuda_alike() or current_platform.is_xpu()
):
raise NotImplementedError(
"is_act_and_mul=False is supported only for CUDA, XPU and ROCm for now"
)
@property
def is_act_and_mul(self) -> bool:
return self.activation.is_gated
@property
def tp_size(self):
return self.moe_parallel_config.tp_size
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
def _make_eep_experts(
@@ -58,10 +58,10 @@ def _make_eep_experts(
def make_eep_staged_quant_method(
module: "FusedMoE",
module: "MoERunner",
moe_config: FusedMoEConfig,
) -> FusedMoEMethodBase | None:
quant_method = module.quant_method
quant_method = module._quant_method
if not quant_method.supports_internal_mk:
return None
if getattr(quant_method, "wraps_legacy_quant_method", False):
@@ -53,7 +53,7 @@ class ActivationMethod(IntEnum):
GELU = 1
aiter_topK_meta_data = None
aiter_topK_meta_data: tuple[torch.Tensor, torch.Tensor] | None = None
@lru_cache(maxsize=1)
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
logger = init_logger(__name__)
@@ -51,7 +52,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
@abstractmethod
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -117,7 +118,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
# based on the all2all implementation, select the appropriate
# gemm implementation
@@ -128,7 +129,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
@abstractmethod
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
self, layer: "RoutedExperts"
) -> FusedMoEQuantConfig | None:
raise NotImplementedError
@@ -143,6 +144,14 @@ class FusedMoEMethodBase(QuantizeMethodBase):
"""Whether to skip the padding in the forward before applying the moe method."""
return False
@property
def has_unpadded_output(self) -> bool:
"""
Indicates that the hidden_states output might be the unpadded
hidden_states shape rather than the full padded shape.
"""
return False
@property
def supports_eplb(self) -> bool:
return False
@@ -162,20 +171,44 @@ class FusedMoEMethodBase(QuantizeMethodBase):
def apply(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None",
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
"""
Apply the MoE operation using modular kernels.
Args:
layer: RoutedExperts instance containing weight parameters
x: Input tensor
topk_weights: Expert weights from router
topk_ids: Selected expert IDs from router
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts
"""
raise NotImplementedError
def apply_monolithic(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Apply the MoE operation using monolithic kernels.
Args:
layer: RoutedExperts instance containing weight parameters
x: Input tensor
router_logits: Router logits (routing done internally)
Returns:
Output tensor from routed experts
"""
raise NotImplementedError
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
@@ -20,6 +21,11 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
logger = init_logger(__name__)
@@ -43,7 +49,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
@staticmethod
def make(
moe_layer: torch.nn.Module,
routed_experts: "RoutedExperts",
old_quant_method: FusedMoEMethodBase,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
) -> "FusedMoEModularMethod":
@@ -51,10 +57,18 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
old_quant_method,
FusedMoEKernel(
prepare_finalize,
old_quant_method.select_gemm_impl(prepare_finalize, moe_layer),
old_quant_method.select_gemm_impl(prepare_finalize, routed_experts),
),
)
@property
def skip_forward_padding(self) -> bool:
return self.old_quant_method.skip_forward_padding
@property
def has_unpadded_output(self) -> bool:
return self.old_quant_method.has_unpadded_output
@property
def supports_eplb(self) -> bool:
return self.old_quant_method.supports_eplb
@@ -65,7 +79,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -75,13 +89,13 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
raise NotImplementedError
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
self, layer: "RoutedExperts"
) -> FusedMoEQuantConfig | None:
return self.moe_quant_config
def apply(
self,
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
layer: "RoutedExperts",
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
File diff suppressed because it is too large Load Diff
@@ -1097,8 +1097,9 @@ class FusedMoEKernelModularImpl:
shared_experts_input: torch.Tensor | None,
):
if shared_experts is not None:
assert self.prepare_finalize.supports_async()
assert shared_experts_input is not None
shared_experts.apply(
shared_experts(
shared_experts_input,
SharedExpertsOrder.MK_INTERNAL_OVERLAPPED,
)
@@ -1572,7 +1573,7 @@ class FusedMoEKernel:
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
fp8_w8a8_moe_quant_config,
fp8_w8a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
@@ -423,7 +424,7 @@ def select_fp8_moe_backend(
def convert_to_fp8_moe_kernel_format(
fp8_backend: Fp8MoeBackend,
# TODO(bnell): replace layer with weight_block_size
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -13,6 +13,7 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
@@ -1632,12 +1633,11 @@ def make_mxfp4_moe_quant_config(
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, FusedMoE)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(
layer,
gemm1_alpha=gemm1_alpha,
@@ -1663,7 +1663,7 @@ def make_mxfp4_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: "RoutedExperts | None" = None,
layer: RoutedExperts | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import (
nvfp4_moe_quant_config,
nvfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
@@ -335,7 +336,7 @@ def select_nvfp4_moe_backend(
def convert_to_nvfp4_moe_kernel_format(
nvfp4_backend: NvFp4MoeBackend,
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w13_scale: torch.Tensor,
w13_scale_2: torch.Tensor,
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
@@ -37,13 +36,11 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
eplb_state: EplbLayerState | None = None,
scoring_func: str = "softmax",
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.renormalize = renormalize
self.scoring_func = scoring_func
@@ -149,27 +149,16 @@ class BaseRouter(FusedMoERouter):
top_k: int,
global_num_experts: int,
eplb_state: EplbLayerState | None = None,
# TODO(bnell): Once the MK is constructed at layer init time, we
# can make this a plain value instead of a callback.
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
"""
Note: the indices dtype might not be available at router construction
time, so we need to supply a callback to get it at runtime. This is
because the indices type is supplied by modular kernels which are
created after MoE layer/router construction.
Args:
top_k: Number of experts to select per token
global_num_experts: Total number of experts
eplb_state: Optional EPLBLayerState for load balancing
indices_type_getter: Optional callback to get indices dtype
"""
super().__init__()
super().__init__(eplb_state=eplb_state)
self.top_k = top_k
self.global_num_experts = global_num_experts
self.eplb_state = eplb_state
self.indices_type_getter = indices_type_getter
self.capture_fn: Callable[[torch.Tensor], None] | None = None
def set_capture_fn(self, capture_fn: Callable[[torch.Tensor], None] | None) -> None:
@@ -189,12 +178,6 @@ class BaseRouter(FusedMoERouter):
if eplb_state.should_record_tensor is None:
raise ValueError("EPLB requires should_record_tensor != None")
def _get_indices_type(self) -> torch.dtype | None:
"""Get the desired indices dtype from the getter function."""
return (
self.indices_type_getter() if self.indices_type_getter is not None else None
)
def _apply_eplb_mapping(self, topk_ids: torch.Tensor) -> torch.Tensor:
"""Apply EPLB mapping to convert logical expert IDs to physical expert IDs."""
if self.eplb_state is not None:
@@ -247,10 +230,11 @@ class BaseRouter(FusedMoERouter):
"""
raise NotImplementedError
def select_experts(
def _select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -260,10 +244,9 @@ class BaseRouter(FusedMoERouter):
This method implements the template method pattern:
1. Validates EPLB state
2. Gets indices type
3. Calls _compute_routing() to get topk_weights and topk_ids
4. Applies EPLB mapping if enabled
5. Converts indices dtype if needed
2. Calls _compute_routing() to get topk_weights and topk_ids
3. Applies EPLB mapping if enabled
4. Converts indices dtype if needed
Returns:
(topk_weights, topk_ids)
@@ -277,22 +260,19 @@ class BaseRouter(FusedMoERouter):
# Step 1: Validate EPLB state
self._validate_eplb_state()
# Step 2: Get indices type.
indices_type = self._get_indices_type()
# Step 3: Compute routing (delegated to subclass)
# Step 2: Compute routing (delegated to subclass)
topk_weights, topk_ids = self._compute_routing(
hidden_states, router_logits, indices_type, input_ids=input_ids
hidden_states, router_logits, topk_indices_dtype, input_ids=input_ids
)
# Capture logical ids before EPLB mapping.
if self.capture_fn is not None:
self.capture_fn(topk_ids)
# Step 4: Apply EPLB mapping
# Step 3: Apply EPLB mapping
topk_ids = self._apply_eplb_mapping(topk_ids)
# Step 5: Convert indices dtype
topk_ids = self._convert_indices_dtype(topk_ids, indices_type)
# Step 4: Convert indices dtype
topk_ids = self._convert_indices_dtype(topk_ids, topk_indices_dtype)
return topk_weights, topk_ids
@@ -19,13 +19,11 @@ class CustomRoutingRouter(BaseRouter):
custom_routing_function: Callable,
eplb_state: EplbLayerState | None = None,
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.custom_routing_function = custom_routing_function
self.renormalize = renormalize
@@ -5,6 +5,7 @@ from collections.abc import Callable
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
@@ -14,6 +15,10 @@ class FusedMoERouter(ABC):
method that is used for routing hidden states based on router logits.
"""
def __init__(self, eplb_state: EplbLayerState | None = None):
self._routing_replay_out: torch.Tensor | None = None
self.eplb_state = eplb_state
@abstractmethod
def set_capture_fn(
self,
@@ -27,10 +32,21 @@ class FusedMoERouter(ABC):
raise NotImplementedError
@abstractmethod
def _select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
raise NotImplementedError
def select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
topk_indices_dtype: torch.dtype | None = None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -47,4 +63,19 @@ class FusedMoERouter(ABC):
equivalent to global logical ids, so should be compatible with
plain MoE implementations without redundant experts.
"""
raise NotImplementedError
topk_weights, topk_ids = self._select_experts(
hidden_states,
router_logits,
topk_indices_dtype=topk_indices_dtype,
input_ids=input_ids,
)
# Write routing data for non-monolithic path (Triton, etc.)
# (set by bind_routing_capture_to_model during capturer init)
if self._routing_replay_out is not None:
self._routing_replay_out[: topk_ids.shape[0]].copy_(
topk_ids.to(torch.int16)
)
return topk_weights, topk_ids
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Callable
import torch
import torch.nn.functional as F
@@ -300,7 +299,6 @@ class FusedTopKBiasRouter(BaseRouter):
renormalize: bool = True,
routed_scaling_factor: float = 1.0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
*,
scoring_func: str = "sigmoid",
hash_indices_table: torch.Tensor | None = None,
@@ -309,7 +307,6 @@ class FusedTopKBiasRouter(BaseRouter):
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.e_score_correction_bias = e_score_correction_bias
self.renormalize = renormalize
@@ -123,13 +123,11 @@ class FusedTopKRouter(BaseRouter):
scoring_func: str = "softmax",
renormalize: bool = True,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.renormalize = renormalize
self.scoring_func = scoring_func
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from functools import partial
import torch
@@ -259,13 +258,11 @@ class GroupedTopKRouter(BaseRouter):
e_score_correction_bias: torch.Tensor | None = None,
num_fused_shared_experts: int = 0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.num_expert_group = num_expert_group
self.topk_group = topk_group
@@ -41,7 +41,6 @@ def create_fused_moe_router(
top_k: int,
global_num_experts: int,
renormalize: bool = True,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
# grouped topk parameters
use_grouped_topk: bool = False,
num_expert_group: int | None = None,
@@ -77,7 +76,6 @@ def create_fused_moe_router(
top_k: Number of experts to select per token
global_num_experts: Total number of experts in the model
renormalize: Whether to renormalize the routing weights
indices_type_getter: Function to get the desired indices dtype
routing_method_type: Optional explicit routing method type
Grouped topk arguments:
@@ -116,7 +114,6 @@ def create_fused_moe_router(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
if zero_expert_type is not None:
@@ -136,7 +133,6 @@ def create_fused_moe_router(
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
indices_type_getter=indices_type_getter,
)
if use_grouped_topk:
@@ -157,7 +153,6 @@ def create_fused_moe_router(
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=num_fused_shared_experts,
indices_type_getter=indices_type_getter,
)
if (
grouped_topk_router.routing_method_type != RoutingMethodType.Unspecified
@@ -179,7 +174,6 @@ def create_fused_moe_router(
eplb_state=eplb_state,
custom_routing_function=custom_routing_function,
renormalize=renormalize,
indices_type_getter=indices_type_getter,
)
assert scoring_func in ["sigmoid", "softmax", "sqrtsoftplus"]
@@ -192,7 +186,6 @@ def create_fused_moe_router(
e_score_correction_bias=e_score_correction_bias,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
indices_type_getter=indices_type_getter,
scoring_func=scoring_func,
hash_indices_table=hash_indices_table,
)
@@ -209,7 +202,6 @@ def create_fused_moe_router(
num_fused_shared_experts=num_fused_shared_experts,
renormalize=renormalize,
scoring_func=scoring_func,
indices_type_getter=indices_type_getter,
)
return FusedTopKRouter(
@@ -218,5 +210,4 @@ def create_fused_moe_router(
eplb_state=eplb_state,
renormalize=renormalize,
scoring_func=scoring_func,
indices_type_getter=indices_type_getter,
)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Any
import torch
@@ -314,13 +313,11 @@ class RoutingSimulatorRouter(BaseRouter):
top_k: int,
global_num_experts: int,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
@property
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
@@ -39,13 +37,11 @@ class ZeroExpertRouter(BaseRouter):
renormalize: bool = False,
routed_scaling_factor: float = 1.0,
eplb_state: EplbLayerState | None = None,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
indices_type_getter=indices_type_getter,
)
self.e_score_correction_bias = e_score_correction_bias
self.num_logical_experts = num_logical_experts
@@ -1,28 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Iterable
from contextlib import nullcontext
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.distributed import (
get_ep_group,
get_pcp_group,
tensor_model_parallel_all_reduce,
)
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import (
ForwardContext,
get_forward_context,
is_forward_context_available,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
@@ -43,8 +54,23 @@ from vllm.utils.torch_utils import (
direct_register_custom_op,
)
logger = init_logger(__name__)
def get_layer_from_name(layer_name: str) -> torch.nn.Module:
def register_layer_for_moe_forward_op(
vllm_config: VllmConfig,
layer: "MoERunner",
):
# For smuggling this layer into the fused moe custom op
prefix = layer.layer_name
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError("Duplicate layer name: {}".format(prefix))
compilation_config.static_forward_context[prefix] = layer
compilation_config.static_all_moe_layers.append(prefix)
def get_layer_from_name(layer_name: str) -> MoERunnerInterface:
forward_context: ForwardContext = get_forward_context()
if not _USE_LAYERNAME and layer_name == "from_forward_context":
all_moe_layers = forward_context.all_moe_layers
@@ -58,7 +84,9 @@ def get_layer_from_name(layer_name: str) -> torch.nn.Module:
)
layer_name = all_moe_layers[moe_layer_index]
forward_context.moe_layer_index += 1
return forward_context.no_compile_layers[layer_name]
layer = forward_context.no_compile_layers[layer_name]
assert isinstance(layer, MoERunnerInterface)
return layer
# On torch >= 2.11, layer_name is a hoisted LayerName opaque object;
@@ -96,8 +124,7 @@ def _moe_forward(
hidden_dim_unpadded: int,
) -> torch.Tensor:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -131,8 +158,7 @@ def _moe_forward_shared(
hidden_dim_unpadded: int,
) -> tuple[torch.Tensor, torch.Tensor]:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer.runner._forward_impl(
layer,
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
@@ -220,12 +246,12 @@ class MoERunner(MoERunnerInterface):
layer_name: str,
moe_config: FusedMoEConfig,
router: FusedMoERouter,
routed_input_transform: torch.nn.Module | None,
gate: torch.nn.Module | None,
shared_experts: torch.nn.Module | None,
quant_method: FusedMoEMethodBase,
enable_dbo: bool,
routed_experts: RoutedExperts,
enable_dbo: bool = False,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
shared_expert_gate: torch.nn.Module | None = None,
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
routed_scaling_factor: float = 1.0,
):
@@ -237,7 +263,7 @@ class MoERunner(MoERunnerInterface):
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.shared_expert_gate = shared_expert_gate
self._quant_method = quant_method
self.routed_experts = routed_experts
self.enable_dbo = enable_dbo
# When both gates are present and FSE is enabled, fuse their
@@ -250,23 +276,22 @@ class MoERunner(MoERunnerInterface):
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
self._shared_experts = SharedExperts(
shared_experts,
moe_config=moe_config,
# Note: For now we must pass quant_method along to SharedExperts so it
# can property determine where the shared experts are supposed to be
# called, i.e. by a MK or by the MoERunner.
# Once the MK can be created upfront, we can just pass in the proper
# flags derived from the quant_method's MK.
quant_method=quant_method,
enable_dbo=enable_dbo,
mk_can_overlap_shared_experts=can_overlap,
)
# Needed for string -> FusedMoE layer lookup in custom ops.
# Needed for string -> MoERunner layer lookup in custom ops.
self.layer_name = layer_name
self._forward_entry = self._select_forward()
# For smuggling this layer into the fused moe custom op
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we
@@ -284,11 +309,20 @@ class MoERunner(MoERunnerInterface):
def shared_experts(self) -> SharedExperts | None:
return self._shared_experts
# TODO(bnell): temporary hack, do not call this method.
@property
def is_internal_router(self) -> bool:
return self.gate is not None
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.routed_experts._replace_quant_method(quant_method)
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
self.routed_experts._set_moe_config(new_moe_config)
if self._shared_experts is not None:
self._shared_experts._quant_method = quant_method
self._quant_method = quant_method
self._shared_experts._set_moe_config(new_moe_config)
def _maybe_fuse_gate_weights(self):
"""Fuse router and shared expert gate weights on first call.
@@ -304,8 +338,9 @@ class MoERunner(MoERunnerInterface):
dim=0,
)
def is_internal_router(self) -> bool:
return self.gate is not None
@property
def _quant_method(self) -> FusedMoEMethodBase:
return self.routed_experts.quant_method
def apply_routed_input_transform(
self, hidden_states: torch.Tensor
@@ -396,9 +431,9 @@ class MoERunner(MoERunnerInterface):
def _maybe_reduce_final_output(
self,
states: torch.Tensor,
trunc_size: int,
trunc_size: int | None,
) -> torch.Tensor:
"""Truncate padded dimensions and all-reduce the combined output.
"""All-reduce the combined output if needed.
This is the "late" all-reduce path. When neither fused nor shared
output was individually reduced, the combined sum is all-reduced
@@ -415,7 +450,7 @@ class MoERunner(MoERunnerInterface):
):
states = tensor_model_parallel_all_reduce(states)
return states[..., :trunc_size]
return states[..., :trunc_size] if trunc_size is not None else states
def _encode_layer_name(self) -> str | LayerName:
if _USE_LAYERNAME:
@@ -428,34 +463,11 @@ class MoERunner(MoERunnerInterface):
return "from_forward_context"
return self.layer_name
def _trtllm_mxfp4_unpadded_dim(self) -> int:
"""Return ``hidden_dim_unpadded`` when the active backend is TRT-LLM
MXFP4 (whose kernel writes narrower than the padded
``hidden_states.shape[-1]``), else 0. Other MXFP4 backends (notably
Cutlass MXFP4 MXFP8) write the full padded width, so
``moe_config.hidden_dim_unpadded`` alone is insufficient: it encodes
the model's logical hidden, not whether the kernel narrows. Computed
caller-side and passed as an op arg; doing the isinstance check
inside the fake would specialize per ``layer_name`` and break
subgraph dedup for identical-architecture models (e.g. Phi-MoE).
"""
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsBase,
)
moe_kernel = getattr(self._quant_method, "moe_kernel", None)
fused_experts = getattr(
getattr(moe_kernel, "impl", None), "fused_experts", None
)
if isinstance(fused_experts, TrtLlmMxfp4ExpertsBase):
return self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
return 0
def _maybe_pad_hidden_states(
self,
shared_experts_input: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, int]:
) -> tuple[torch.Tensor, int | None, int | None]:
"""Pad hidden_states to moe_config.hidden_dim and compute the
original dimension for later truncation.
@@ -467,11 +479,12 @@ class MoERunner(MoERunnerInterface):
shared_experts_hidden_dim = (
shared_experts_input.shape[-1] if shared_experts_input is not None else 0
)
transformed_hidden_dim = hidden_states.shape[-1]
transformed_hidden_dim: int | None = hidden_states.shape[-1]
if (
not self._quant_method.skip_forward_padding
and self.moe_config.hidden_dim != transformed_hidden_dim
):
assert transformed_hidden_dim is not None
hidden_states = F.pad(
hidden_states,
(0, self.moe_config.hidden_dim - transformed_hidden_dim),
@@ -479,12 +492,33 @@ class MoERunner(MoERunnerInterface):
value=0.0,
)
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
orig_hidden_dims = shared_experts_hidden_dim
else:
orig_hidden_dims = transformed_hidden_dim
# Truncation sizes for stripping kernel padding from the output.
# None means no truncation needed (no padding was applied).
#
# Two truncation points exist in forward():
# pre_xform: applied to fused_output BEFORE routed_output_transform
# post_xform: applied to the final result AFTER all-reduce
#
# Latent MoE with shared experts (NemotronH):
# - pre_xform strips padding from the latent dim so
# routed_output_transform receives the correct input size
# - post_xform truncates to shared_experts_hidden_dim (full hidden)
# after shared + routed outputs are combined and all-reduced
#
# Standard MoE / MoE without transforms (GPT-OSS, Mixtral):
# - pre_xform is None (no early truncation)
# - post_xform strips padding after all-reduce (or None if unpadded)
if transformed_hidden_dim == hidden_states.shape[-1]:
transformed_hidden_dim = None
return hidden_states, orig_hidden_dims
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
pre_xform_trunc_size = transformed_hidden_dim
post_xform_trunc_size = shared_experts_hidden_dim
else:
pre_xform_trunc_size = None
post_xform_trunc_size = transformed_hidden_dim
return hidden_states, pre_xform_trunc_size, post_xform_trunc_size
def _maybe_apply_shared_experts(
self,
@@ -493,11 +527,10 @@ class MoERunner(MoERunnerInterface):
):
if self._shared_experts is not None:
assert shared_experts_input is not None
self._shared_experts.apply(shared_experts_input, order)
self._shared_experts(shared_experts_input, order)
def _apply_quant_method(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -513,24 +546,23 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
if self.routed_experts.quant_method.is_monolithic:
# Monolithic kernels: pass router_logits to routed_experts
fused_out = self.routed_experts.forward_monolithic(
x=hidden_states,
router_logits=router_logits,
input_ids=input_ids,
)
else:
# Modular kernels: select experts first, then call routed_experts
topk_weights, topk_ids = self.router.select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
topk_indices_dtype=self._quant_method.topk_indices_dtype,
input_ids=input_ids,
)
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(
layer=layer,
fused_out = self.routed_experts.forward_modular(
x=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
@@ -627,12 +659,13 @@ class MoERunner(MoERunnerInterface):
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
# so routed output can be trimmed before
# shared+routed add / latent up proj if needed.
routed_hidden_dim = hidden_states.shape[-1]
hidden_states, og_hidden_dim = self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
)
)
hidden_dim_was_padded = hidden_states.shape[-1] > routed_hidden_dim
result = self._forward_entry(
hidden_states,
@@ -640,7 +673,9 @@ class MoERunner(MoERunnerInterface):
shared_experts_input,
input_ids,
self._encode_layer_name(),
self._trtllm_mxfp4_unpadded_dim(),
self.moe_config.hidden_dim_unpadded
if self._quant_method.has_unpadded_output
else 0,
)
#
@@ -654,10 +689,9 @@ class MoERunner(MoERunnerInterface):
# Extract outputs from result
shared_output, fused_output = _unpack(result)
if (
shared_output is not None or self.routed_output_transform is not None
) and hidden_dim_was_padded:
fused_output = fused_output[..., :routed_hidden_dim]
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
# If combine kernel already reduced fused, reduce shared to match.
# See note above re: the two all-reduce points.
@@ -675,7 +709,7 @@ class MoERunner(MoERunnerInterface):
else:
result = fused_output
result = self._maybe_reduce_final_output(result, og_hidden_dim)
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
return self._maybe_add_zero_expert_output(result)
@@ -687,7 +721,6 @@ class MoERunner(MoERunnerInterface):
def _maybe_dispatch(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -723,7 +756,7 @@ class MoERunner(MoERunnerInterface):
self,
shared_output: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
) -> torch.Tensor | tuple[torch.Tensor | None, torch.Tensor]:
if self.do_naive_dispatch_combine:
hidden_states = get_ep_group().combine(
hidden_states, self.moe_config.is_sequence_parallel
@@ -743,7 +776,6 @@ class MoERunner(MoERunnerInterface):
def _forward_impl(
self,
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
@@ -762,7 +794,7 @@ class MoERunner(MoERunnerInterface):
Returns a single tensor of combined fused and shared output (if present).
"""
# TODO(bnell): this can be removed after MK migration is complete.
layer.ensure_moe_quant_config_init()
self.routed_experts._ensure_moe_quant_config_init()
# Sync aux and main stream for shared expert multi-stream overlap.
self._maybe_sync_shared_experts_stream(shared_experts_input)
@@ -782,13 +814,11 @@ class MoERunner(MoERunnerInterface):
# #32567 lands and the remaining kernels are made MKs. The PCP
# code will probably remain
hidden_states, router_logits = self._maybe_dispatch(
layer,
hidden_states,
router_logits,
)
shared_output, hidden_states = self._apply_quant_method(
layer=layer,
hidden_states=hidden_states,
router_logits=router_logits,
shared_experts_input=shared_experts_input,
@@ -799,3 +829,148 @@ class MoERunner(MoERunnerInterface):
shared_output,
hidden_states,
)
#########################################################
#
# Old methods from FusedMoE layer. Remove when possible.
#
#########################################################
# Note: maybe_init_modular_kernel should only be called by
# prepare_communication_buffer_for_model.
# This is called after all weight loading and post-processing, so it
# should be safe to swap out the quant_method.
def maybe_init_modular_kernel(self) -> None:
# NOTE(rob): WIP refactor. For quant methods that own the MK
# we create the MK during process_weights_after_loading.
if (
self.routed_experts.quant_method.supports_internal_mk
or self.routed_experts.quant_method.is_monolithic
):
return None
self.routed_experts._ensure_moe_quant_config_init()
# routing_tables only needed for round-robin expert placement with
# DeepEP all2all backend.
routing_tables = self._expert_routing_tables()
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
base_quant_method = self.routed_experts.quant_method.old_quant_method
else:
base_quant_method = self.routed_experts.quant_method
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
routing_tables=routing_tables
)
if prepare_finalize is not None:
logger.debug(
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
)
self._replace_quant_method(
FusedMoEModularMethod.make(
self.routed_experts,
base_quant_method,
prepare_finalize,
)
)
#
# Properties
#
@property
def layer_id(self):
# Delayed import to avoid circular dependency
from vllm.model_executor.models.utils import extract_layer_index
return extract_layer_index(self.layer_name)
#
# Attributes still needed by models
#
@property
def is_monolithic(self) -> bool:
return self.routed_experts.quant_method.is_monolithic
@property
def activation(self) -> MoEActivation:
return self.routed_experts.activation
#
# Expert maps
#
@property
def expert_map_manager(self):
"""Forward to routed_experts.expert_map_manager for backward compatibility."""
return self.routed_experts.expert_map_manager
@property
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
return self.expert_map_manager.placement_strategy
@property
def expert_global_to_physical(self) -> torch.Tensor | None:
tables = self.expert_map_manager.routing_tables
return tables[0] if tables else None
@property
def expert_physical_to_global(self) -> torch.Tensor | None:
"""Routing table: physical expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[1] if tables else None
@property
def expert_local_to_global(self) -> torch.Tensor | None:
"""Routing table: local expert ID to global expert ID."""
tables = self.expert_map_manager.routing_tables
return tables[2] if tables else None
@property
def expert_map(self) -> torch.Tensor | None:
return self.routed_experts.expert_map
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
return self.routed_experts._expert_routing_tables()
def update_expert_map(self):
self.routed_experts.update_expert_map()
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
"""Map global expert ID to local expert ID."""
return self.routed_experts._map_global_expert_id_to_local_expert_id(expert_id)
def get_expert_weights(self) -> Iterable[torch.Tensor]:
return self.routed_experts.get_expert_weights()
#
# EPLB
#
@property
def eplb_state(self) -> EplbLayerState | None:
return self.router.eplb_state
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
"""
Register the EPLB state in this layer.
This is used later in forward pass, where we get the expert mapping
and record the load metrics in `expert_load_view`.
"""
if self.router.eplb_state is not None:
self.router.eplb_state.set_layer_state(
moe_layer_idx,
expert_load_view,
logical_to_physical_map,
logical_replica_count,
)
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Iterable
import torch
from vllm.config.parallel import ExpertPlacementStrategy
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
@@ -22,6 +25,11 @@ class MoERunnerInterface(PluggableLayer, ABC):
expert routing, and managing tensor parallel operations.
"""
def __init__(self):
super().__init__()
# HACK
self._already_called_process_weights_after_loading = True
@abstractmethod
def forward(
self,
@@ -31,16 +39,108 @@ class MoERunnerInterface(PluggableLayer, ABC):
) -> torch.Tensor:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
raise NotImplementedError
@property
@abstractmethod
def is_internal_router(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def shared_experts(self) -> SharedExperts | None:
def _quant_method(self) -> FusedMoEMethodBase:
raise NotImplementedError
# TODO(bnell): temporary hack, do not call this method.
# Temporary hack
@abstractmethod
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
raise NotImplementedError
########################################################################
#
# FusedMoE layer methods
#
########################################################################
@abstractmethod
def maybe_init_modular_kernel(self) -> None:
raise NotImplementedError
@property
@abstractmethod
def layer_id(self):
raise NotImplementedError
#
# Attributes still needed by models
#
@property
@abstractmethod
def is_monolithic(self) -> bool:
raise NotImplementedError
@property
@abstractmethod
def activation(self) -> MoEActivation:
raise NotImplementedError
#
# Expert maps
#
@property
@abstractmethod
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
raise NotImplementedError
@property
@abstractmethod
def expert_global_to_physical(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_physical_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_local_to_global(self) -> torch.Tensor | None:
raise NotImplementedError
@property
@abstractmethod
def expert_map(self) -> torch.Tensor | None:
raise NotImplementedError
@abstractmethod
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
raise NotImplementedError
@abstractmethod
def update_expert_map(self):
raise NotImplementedError
@abstractmethod
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
raise NotImplementedError
@abstractmethod
def get_expert_weights(self) -> Iterable[torch.Tensor]:
raise NotImplementedError
@abstractmethod
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
raise NotImplementedError
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from enum import IntEnum
import torch
@@ -9,9 +10,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
aux_stream,
@@ -38,21 +36,15 @@ class SharedExpertsOrder(IntEnum):
MULTI_STREAM_OVERLAPPED = (3,)
class SharedExperts:
class SharedExperts(torch.nn.Module):
def __init__(
self,
layer: torch.nn.Module,
moe_config: FusedMoEConfig,
quant_method: QuantizeMethodBase,
enable_dbo: bool,
mk_can_overlap_shared_experts: Callable[[], bool],
):
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
# quant_method must be a FusedMoEMethodBase but we can't use the type
# due to circular imports.
assert isinstance(quant_method, FusedMoEMethodBase)
super().__init__()
# The SharedExperts need to handle DBO since they can be called from
# an MK's finalize method. We keep a list of outputs indexed by current
@@ -62,7 +54,8 @@ class SharedExperts:
self._output: list[torch.Tensor | None] = [None, None]
self._layer = layer
self._moe_config = moe_config
self._quant_method = quant_method
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
# Allow disabling of the separate shared experts stream for
# debug purposes.
@@ -78,6 +71,10 @@ class SharedExperts:
if self._stream is not None:
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
# TODO(bnell): Hack for elastic_ep. Get rid of this
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
self.moe_config = new_moe_config
@property
def _disable_shared_experts_overlap(self) -> bool:
# Disable shared expert overlap if:
@@ -96,7 +93,7 @@ class SharedExperts:
if self._disable_shared_experts_overlap:
return SharedExpertsOrder.NO_OVERLAP
if self._quant_method.mk_can_overlap_shared_experts:
if self._mk_can_overlap_shared_experts():
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
should_run_shared_in_aux_stream = (
@@ -155,7 +152,7 @@ class SharedExperts:
self._output[self._output_idx] = None
return output
def apply(
def forward(
self,
shared_experts_input: torch.Tensor,
order: SharedExpertsOrder,
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from torch.nn import Module
import vllm.envs as envs
from vllm.logger import init_logger
@@ -38,7 +37,7 @@ from vllm.platforms import current_platform
from vllm.platforms.interface import CpuArchEnum
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
logger = init_logger(__name__)
@@ -80,7 +79,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
@@ -89,7 +88,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def create_weights(
self,
layer: torch.nn.Module,
layer: "RoutedExperts",
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -156,7 +155,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def _setup_kernel(
self,
layer: Module,
layer: "RoutedExperts",
w13: torch.Tensor,
w2: torch.Tensor,
) -> None:
@@ -201,7 +200,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
super().process_weights_after_loading(layer)
# Padding the weight for better performance on ROCm.
@@ -16,16 +16,14 @@ from vllm.model_executor.kernels.linear import (
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
@@ -210,7 +210,7 @@ class CompressedTensorsConfig(QuantizationConfig):
)
return None
def _add_fused_moe_to_target_scheme_map(self):
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
since linear layers get fused into FusedMoE
@@ -219,10 +219,10 @@ class CompressedTensorsConfig(QuantizationConfig):
"""
if (
"Linear" not in self.target_scheme_map
or "FusedMoE" in self.target_scheme_map
or "RoutedExperts" in self.target_scheme_map
):
return
self.target_scheme_map["FusedMoE"] = self.target_scheme_map["Linear"]
self.target_scheme_map["RoutedExperts"] = self.target_scheme_map["Linear"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "CompressedTensorsConfig":
@@ -10,14 +10,12 @@ import torch
from vllm import envs
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
FusedMoEQuantConfig,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -44,8 +42,9 @@ from vllm.model_executor.parameter import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
if current_platform.is_cuda():
if has_humming() and current_platform.is_cuda():
from humming.dtypes import DataType
from humming.layer import HummingMethod
from humming.schema import (
@@ -155,6 +155,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -487,6 +495,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# so can skip the padding in the forward before applying the moe method
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
# TODO(bnell): move to MK/expert_class?
@property
def has_unpadded_output(self) -> bool:
return self.mxfp4_backend in [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
]
def maybe_roundup_sizes(
self,
hidden_size: int,
@@ -234,7 +234,12 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
intermediate_size_per_partition = layer.intermediate_size_per_partition
# Note: The layer has not performed rounding on intermediate_size's at this
# point. Use the unpadded size which won't change.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
assert intermediate_size_per_partition is not None
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
@@ -6,6 +6,7 @@ import torch
import vllm._custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
@@ -286,7 +287,7 @@ def prepare_fp4_layer_for_marlin(
def prepare_nvfp4_moe_layer_for_marlin(
layer: torch.nn.Module,
layer: RoutedExperts,
w13: torch.Tensor,
w13_scale: torch.Tensor,
w13_scale_2: torch.Tensor,
@@ -353,7 +354,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
# WEIGHT SCALES
# Permute scales
def premute_scales(
def permute_scales(
scales: torch.Tensor, g_scales: torch.Tensor, name: str
) -> tuple[torch.Tensor, torch.Tensor]:
scales = scales.to(param_dtype)
@@ -388,8 +389,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
g_scales = g_scales / combined_scale_factor
return scales, g_scales
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
w13_scale, w13_scale_2 = permute_scales(w13_scale, w13_scale_2, "w13")
w2_scale, w2_scale_2 = permute_scales(w2_scale, w2_scale_2, "w2")
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2