[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Test that the interaction between EPLB and FusedMoE Layer is okay
|
||||
# Test that the interaction between EPLB and FusedMoEFactory Layer is okay
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_eplb_group,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -69,8 +69,8 @@ def make_fused_moe_layer(
|
||||
rank: int,
|
||||
layer_idx: int,
|
||||
test_config: TestConfig,
|
||||
) -> FusedMoE:
|
||||
fml = FusedMoE(
|
||||
) -> MoERunner:
|
||||
fml = FusedMoEFactory(
|
||||
num_experts=test_config.num_experts,
|
||||
top_k=test_config.num_topk,
|
||||
hidden_size=test_config.hidden_size,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Test that the interaction between EPLB and FusedMoE Layer is okay for DP w/ NVFP4
|
||||
# Test that the interaction between EPLB and MoERunner Layer is okay for DP w/ NVFP4
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_eplb_group,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4FusedMoE,
|
||||
@@ -44,7 +44,7 @@ def make_fused_moe_layer(
|
||||
rank: int,
|
||||
layer_idx: int,
|
||||
test_config: TestConfig,
|
||||
) -> FusedMoE:
|
||||
) -> MoERunner:
|
||||
quant_config = None
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
@@ -55,7 +55,7 @@ def make_fused_moe_layer(
|
||||
exclude_modules=[],
|
||||
)
|
||||
|
||||
fml = FusedMoE(
|
||||
fml = FusedMoEFactory(
|
||||
num_experts=test_config.num_experts,
|
||||
top_k=test_config.num_topk,
|
||||
hidden_size=test_config.hidden_size,
|
||||
|
||||
@@ -409,7 +409,7 @@ def _deep_ep_v2_moe_cudagraph(
|
||||
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
|
||||
|
||||
with set_current_vllm_config(vllm_cfg):
|
||||
# Initialize vLLM parallel state (needed by FusedMoE layer)
|
||||
# Initialize vLLM parallel state (needed by MoERunner layer)
|
||||
temp_file = tempfile.mktemp()
|
||||
init_distributed_environment(
|
||||
world_size=pgi.world_size,
|
||||
|
||||
@@ -42,7 +42,11 @@ from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, fused_experts
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.router.router_factory import (
|
||||
@@ -80,7 +84,7 @@ NUM_EXPERTS = [8, 64]
|
||||
TOP_KS = [2, 6]
|
||||
|
||||
# dp_size, tp_size, use_ep
|
||||
# Note: DP+TP is not yet supported in the FusedMoE layer.
|
||||
# Note: DP+TP is not yet supported in the FusedMoEFactory layer.
|
||||
PARALLEL_COMBOS = [
|
||||
[1, 2, False],
|
||||
[1, 4, False],
|
||||
@@ -1023,7 +1027,7 @@ def make_fused_moe_layer(
|
||||
kwargs["routed_input_transform"] = routed_input_transform
|
||||
kwargs["routed_output_transform"] = routed_output_transform
|
||||
|
||||
layer = FusedMoE(
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=global_num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
@@ -1267,7 +1271,7 @@ def _test_body_eplb(
|
||||
):
|
||||
output_before = sp_wrapper(moe_layer)(hidden_states, router_logits)
|
||||
|
||||
# Create a fresh FusedMoE layer with enable_eplb=True
|
||||
# Create a fresh MoERunner layer with enable_eplb=True
|
||||
# Delete the original layer's registration so the constructor can
|
||||
# re-use the same "from_forward_context" prefix
|
||||
cc = vllm_config.compilation_config
|
||||
@@ -1401,7 +1405,8 @@ def _run_one_config(
|
||||
|
||||
- When is_sequence_parallel=True (EP + sequence splitting):
|
||||
* ep_size: Number of expert parallel ranks (equals dp_size * tp_size)
|
||||
* tp_size: Number of ranks to split sequence across (becomes sp_size in FusedMoE)
|
||||
* tp_size: Number of ranks to split sequence across (becomes sp_size in
|
||||
MoERunner)
|
||||
* Weights are chunked by ep_size (experts) but NOT by tp_size
|
||||
* Input sequences are chunked by tp_size (via sp_wrapper)
|
||||
"""
|
||||
@@ -1476,8 +1481,8 @@ def _run_one_config(
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
# Chunk weights for EP BEFORE creating FusedMoE
|
||||
# FusedMoE uses EP-chunked weights and handles reductions internally
|
||||
# Chunk weights for EP BEFORE creating MoERunner.
|
||||
# MoERunner uses EP-chunked weights and handles reductions internally.
|
||||
if ep_size > 1:
|
||||
# Split experts across ranks (dimension 0 is the expert dimension)
|
||||
# When EP is enabled, use EP group rank and ep_size for chunking
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FusedMoE weight loading with padded hidden dimensions.
|
||||
"""Tests for FusedMoEFactory weight loading with padded hidden dimensions.
|
||||
|
||||
When using DeepEP backends or NIXL EP with models like nemotron_h,
|
||||
hidden_size may be rounded up (e.g., 2688 -> 3072) for backend requirements.
|
||||
|
||||
@@ -6,7 +6,7 @@ Test script for the token-to-expert routing simulator.
|
||||
|
||||
This script demonstrates how to use the routing simulator to test
|
||||
different routing strategies and analyze their performance, including
|
||||
integration tests with FusedMoE layer.
|
||||
integration tests with FusedMoEFactory layer.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
@@ -77,11 +77,11 @@ def test_basic_functionality(
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoE."""
|
||||
FusedMoEFactory."""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
|
||||
# Test parameters
|
||||
num_tokens = 32
|
||||
@@ -111,7 +111,7 @@ def test_routing_strategy_integration(monkeypatch, device):
|
||||
)
|
||||
|
||||
for strategy in strategies:
|
||||
fused_moe = FusedMoE(
|
||||
fused_moe = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for FusedMoE with routed_input_transform.
|
||||
Tests for FusedMoEFactory with routed_input_transform.
|
||||
|
||||
Verifies that applying routed_input_transform inside FusedMoE
|
||||
Verifies that applying routed_input_transform inside FusedMoEFactory
|
||||
produces the same results as applying the transform manually outside.
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed
|
||||
|
||||
@@ -133,9 +133,9 @@ def test_routed_input_transform_inside_vs_outside(
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Compare FusedMoE with transform inside vs manually applying outside.
|
||||
Method A (inside): FusedMoE with routed_input_transform
|
||||
Method B (outside): Manually transform, then FusedMoE without transform
|
||||
"""Compare FusedMoEFactory with transform inside vs manually applying outside.
|
||||
Method A (inside): FusedMoEFactory with routed_input_transform
|
||||
Method B (outside): Manually transform, then FusedMoEFactory without transform
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
|
||||
@@ -157,8 +157,8 @@ def test_routed_input_transform_inside_vs_outside(
|
||||
routed_transform = SimpleLinear(hidden_size, latent_size, dtype)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
# Method A: FusedMoE WITH routed_input_transform
|
||||
moe_with_transform = FusedMoE(
|
||||
# Method A: FusedMoEFactory WITH routed_input_transform
|
||||
moe_with_transform = FusedMoEFactory(
|
||||
shared_experts=shared_experts,
|
||||
routed_input_transform=routed_transform,
|
||||
num_experts=num_experts,
|
||||
@@ -173,9 +173,9 @@ def test_routed_input_transform_inside_vs_outside(
|
||||
prefix="moe_with_transform",
|
||||
)
|
||||
|
||||
# Method B: FusedMoE WITHOUT routed_input_transform
|
||||
# Method B: FusedMoEFactory WITHOUT routed_input_transform
|
||||
# Note: shared_experts=None because when transform is done outside,
|
||||
moe_without_transform = FusedMoE(
|
||||
moe_without_transform = FusedMoEFactory(
|
||||
shared_experts=None,
|
||||
routed_input_transform=None,
|
||||
num_experts=num_experts,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FusedMoE with zero experts.
|
||||
"""Tests for FusedMoEFactory with zero experts.
|
||||
|
||||
Verifies that:
|
||||
- The ZeroExpertRouter is properly created and used as the layer router.
|
||||
- A forward pass through FusedMoE with zero experts produces correct output.
|
||||
- A forward pass through FusedMoEFactory with zero experts produces correct output.
|
||||
- The output decomposes correctly into real expert + zero expert contributions.
|
||||
|
||||
Note: tests generated with Claude.
|
||||
@@ -15,7 +15,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
|
||||
ZeroExpertRouter,
|
||||
)
|
||||
@@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@pytest.fixture
|
||||
def zero_expert_moe(dist_init, default_vllm_config):
|
||||
"""Create a FusedMoE layer with zero experts."""
|
||||
"""Create a FusedMoEFactory layer with zero experts."""
|
||||
num_experts = 4
|
||||
top_k = 2
|
||||
# hidden_size must be >= 256 for the zero expert identity kernel to
|
||||
@@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
|
||||
layer = FusedMoE(
|
||||
layer = FusedMoEFactory(
|
||||
zero_expert_type="identity",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_experts=num_experts,
|
||||
@@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config):
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens):
|
||||
"""Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter."""
|
||||
"""Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter."""
|
||||
layer, _ = zero_expert_moe
|
||||
assert isinstance(layer.router, ZeroExpertRouter), (
|
||||
f"Expected ZeroExpertRouter but got {type(layer.router).__name__}."
|
||||
@@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
"""Run a forward pass through FusedMoE with zero experts and verify output shape."""
|
||||
"""
|
||||
Run a forward pass through FusedMoEFactory with zero experts
|
||||
and verify output shape.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
|
||||
hidden_size = layer.routed_experts.hidden_size
|
||||
@@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
"""Validate that the FusedMoE output equals a plain FusedMoE
|
||||
"""Validate that the FusedMoEFactory output equals a plain FusedMoEFactory
|
||||
output (real experts only) plus the zero expert contribution.
|
||||
|
||||
The key invariant is:
|
||||
zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real)
|
||||
+ zero_expert_output
|
||||
|
||||
We create a plain FusedMoE layer with the same weights and real-expert-only
|
||||
We create a plain FusedMoEFactory layer with the same weights and real-expert-only
|
||||
router logits, compute the zero expert output via the ZeroExpertRouter, and
|
||||
verify the sum matches the FusedMoE output.
|
||||
verify the sum matches the FusedMoEFactory output.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
num_experts = 4
|
||||
@@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
get_forward_context().all_moe_layers = None
|
||||
|
||||
# Create a plain FusedMoE layer with the same config but no zero
|
||||
# Create a plain FusedMoEFactory layer with the same config but no zero
|
||||
# experts. Use a separate prefix to avoid collision.
|
||||
plain_layer = FusedMoE(
|
||||
plain_layer = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=layer.routed_experts.top_k,
|
||||
hidden_size=layer.routed_experts.hidden_size,
|
||||
@@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens):
|
||||
expected,
|
||||
atol=4e-3,
|
||||
rtol=4e-3,
|
||||
msg="FusedMoE output should equal plain FusedMoE output "
|
||||
msg="FusedMoEFactory output should equal plain FusedMoEFactory output "
|
||||
"plus zero expert contribution",
|
||||
)
|
||||
|
||||
|
||||
@@ -198,7 +198,7 @@ def test_dedup_shared_module_across_paths(default_vllm_config, dist_init, dummy_
|
||||
self.gate = gate # canonical path: "moe.gate"
|
||||
|
||||
# Inner submodule holding the SAME gate instance under another
|
||||
# path. This mirrors how FusedMoE.runner.gate references the
|
||||
# path. This mirrors how MoERunner.runner.gate references the
|
||||
# block's gate in qwen3_moe.
|
||||
class _Runner(nn.Module):
|
||||
def __init__(self, g):
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8KVCacheMethod,
|
||||
@@ -393,7 +393,7 @@ def test_fp8_reloading(
|
||||
method.use_marlin = use_marlin
|
||||
|
||||
else:
|
||||
layer = FusedMoE(
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=1,
|
||||
top_k=1,
|
||||
hidden_size=1,
|
||||
|
||||
+1
-1
@@ -1022,7 +1022,7 @@ def cutlass_fp4_moe_mm(
|
||||
An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs
|
||||
the gemms for each combination based on the specified problem sizes.
|
||||
|
||||
This is used as the MoE gemm during NVFP4 Quantized FusedMoE forward.
|
||||
This is used as the MoE gemm during NVFP4 Quantized MoERunner forward.
|
||||
- a/b_tensors: the NVFP4 a_ptrs and b_ptrs tensors which are quantized
|
||||
input and expert weights.
|
||||
- a_/b_scales: The blockscales in FP8-E4M3 precision
|
||||
|
||||
@@ -88,7 +88,7 @@ class QuantizationConfigArgs:
|
||||
"""Spec applied to ``LinearBase`` layers."""
|
||||
|
||||
moe: QuantSpec | None = None
|
||||
"""Spec applied to ``FusedMoE`` layers."""
|
||||
"""Spec applied to ``FusedMoEFactory`` layers."""
|
||||
|
||||
ignore: list[str] = Field(default_factory=list)
|
||||
"""Layers to skip quantization for."""
|
||||
|
||||
@@ -170,7 +170,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
# Only process layers that have kv_cache
|
||||
# attribute (attention layers) Skip non-attention
|
||||
# layers like FusedMoE/MLP etc.
|
||||
# layers like FusedMoEFactory/MLP etc.
|
||||
kv_cache_layer = getattr(layer, "kv_cache", None)
|
||||
if kv_cache_layer is None:
|
||||
continue
|
||||
|
||||
@@ -44,7 +44,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
"Monolithic kernels are not supported for Fused MoE LoRA."
|
||||
)
|
||||
|
||||
# Use the MoE-aware TP rank/size: when EP is active, FusedMoE collapses
|
||||
# Use the MoE-aware TP rank/size: when EP is active, MoERunner collapses
|
||||
# moe_parallel_config.tp_size to 1 (experts are sharded across the
|
||||
# TP group instead).
|
||||
moe_parallel_config = self.moe_config.moe_parallel_config
|
||||
|
||||
@@ -24,7 +24,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoEEPLoadSpec:
|
||||
"""Per-expert-parallel slicing metadata for one FusedMoE LoRA module.
|
||||
"""Per-expert-parallel slicing metadata for one FusedMoEFactory LoRA module.
|
||||
|
||||
Threaded into the LoRA loader so per-expert weights from EP ranks
|
||||
other than this one can be skipped before they ever hit CPU memory.
|
||||
@@ -193,7 +193,7 @@ class LoRAModel:
|
||||
skip_prefixes: List of module name prefixes to skip during loading.
|
||||
Models can define this to skip modules not used in inference
|
||||
(e.g., MTP layers). Format: ["mtp."]
|
||||
moe_ep_spec: When 2D FusedMoE LoRA modules are present with
|
||||
moe_ep_spec: When 2D FusedMoEFactory LoRA modules are present with
|
||||
expert parallelism enabled, the (ep_rank, local, global)
|
||||
slicing metadata shared across all MoE layers. Non-local
|
||||
expert weights are skipped at read time instead of being
|
||||
|
||||
@@ -739,7 +739,7 @@ class LoRAModelManager:
|
||||
|
||||
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
|
||||
for module_name, new_module_names in self.packed_modules.items():
|
||||
# For 2D FusedMoE modules with EP, narrow the per-expert
|
||||
# For 2D RoutedExperts modules with EP, narrow the per-expert
|
||||
# sub-module list to this rank's owned experts so pack_moe
|
||||
# produces a tensor sized to local_num_experts directly.
|
||||
packed_module_names = new_module_names
|
||||
@@ -1096,7 +1096,7 @@ class LoRAModelManager:
|
||||
|
||||
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
|
||||
"""
|
||||
Per-rank slicing metadata for 2D FusedMoE LoRA modules.
|
||||
Per-rank slicing metadata for 2D RoutedEXperts LoRA modules.
|
||||
"""
|
||||
if not self._use_ep or not self._is_moe:
|
||||
return None
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
@@ -63,7 +63,7 @@ def get_config() -> dict[str, Any] | None:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FusedMoE",
|
||||
"FusedMoEFactory",
|
||||
"FusedMoERouter",
|
||||
"FusedMoEConfig",
|
||||
"FusedMoEQuantConfig",
|
||||
|
||||
@@ -1134,9 +1134,9 @@ class FusedMoEParallelConfig:
|
||||
level's of parallelism to use in the fused moe layer.
|
||||
|
||||
Args:
|
||||
tp_size_ (int): `tp_size` passed into the FusedMoE constructor.
|
||||
pcp_size_ (int): `pcp_size` passed into the FusedMoE constructor.
|
||||
dp_size_ (int): `dp_size` passed into the FusedMoE constructor.
|
||||
tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor.
|
||||
pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor.
|
||||
dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor.
|
||||
vllm_parallel_config (ParallelConfig): vLLM's parallel config
|
||||
object which contains the `enable_expert_parallel` flag.
|
||||
|
||||
|
||||
@@ -96,8 +96,7 @@ def determine_expert_counts(
|
||||
return global_num_experts, logical_num_experts, num_fused_shared_experts
|
||||
|
||||
|
||||
# TODO: rename this
|
||||
def FusedMoE(
|
||||
def FusedMoEFactory(
|
||||
num_experts: int, # Global number of experts
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
|
||||
@@ -1016,7 +1016,7 @@ def _process_weights_xpu(
|
||||
w2: [E, K, N] int4 (uint8 storage [E, K, N // 2])
|
||||
w2_scales: [E, K, N // group_size] params_dtype
|
||||
|
||||
Input GPTQ layout from FusedMoE.weight_loader:
|
||||
Input GPTQ layout from MoERunner.weight_loader:
|
||||
w13: [E, K // 8, 2*N] int32 (8 nibbles per int32 along the input dim)
|
||||
w13_scales: [E, K // group_size, 2*N] params_dtype
|
||||
w2: [E, N // 8, K] int32
|
||||
@@ -1419,7 +1419,7 @@ def convert_to_wna16_moe_kernel_format(
|
||||
|
||||
Args:
|
||||
backend: the selected ``WNA16MoEBackend``.
|
||||
layer: the ``FusedMoE`` layer whose parameters are being prepared.
|
||||
layer: the ``MoERunner`` layer whose parameters are being prepared.
|
||||
quant_config: the ``QuantizationConfig`` for this layer.
|
||||
input_dtype: optional activation dtype, usually should be 16 bit.
|
||||
"""
|
||||
|
||||
@@ -180,7 +180,7 @@ class RoutedExpertsCapturer:
|
||||
# (trailing rows are SP ceil-div padding). The TP group
|
||||
# is always initialized on real rollout workers, and
|
||||
# every rank in the group reaches this branch in
|
||||
# lockstep (bind is per-FusedMoE layer, SP is a global
|
||||
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
|
||||
# condition), so a bare all_gather here will not
|
||||
# deadlock -- let it raise if the precondition is
|
||||
# violated rather than skip silently.
|
||||
|
||||
@@ -353,7 +353,7 @@ class MoERunner(MoERunnerInterface):
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Apply transform for routed experts (e.g., latent projection).
|
||||
|
||||
This is called by FusedMoE.forward_native. The original hidden_states
|
||||
This is called by MoERunner.forward_native. The original hidden_states
|
||||
is saved separately so shared experts get [S, hidden_size] while
|
||||
routed experts get the transformed [S, moe_latent_size].
|
||||
|
||||
@@ -646,7 +646,7 @@ class MoERunner(MoERunnerInterface):
|
||||
):
|
||||
# If router/gate provided, then apply it here.
|
||||
# (Note: This code runs only when "overlapped mode" is on to allow
|
||||
# parallel execution of shared experts with the FusedMoE via
|
||||
# parallel execution of shared experts with the RoutedExperts via
|
||||
# separate cuda stream)
|
||||
if self._shared_experts is not None:
|
||||
assert shared_experts_input is not None
|
||||
|
||||
@@ -62,7 +62,7 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
|
||||
########################################################################
|
||||
#
|
||||
# FusedMoE layer methods
|
||||
# FusedMoEFactory layer methods
|
||||
#
|
||||
########################################################################
|
||||
|
||||
|
||||
@@ -215,7 +215,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
|
||||
"""
|
||||
Helper function to update target_scheme_map
|
||||
since linear layers get fused into FusedMoE
|
||||
since linear layers get fused into RoutedExperts
|
||||
targeting 'Linear' needs to also match
|
||||
RoutedExperts modules.
|
||||
"""
|
||||
|
||||
@@ -2363,7 +2363,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
if key.startswith(prefix_dot):
|
||||
return info["quant_algo"].upper()
|
||||
|
||||
# FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's
|
||||
# RoutedExperts expert prefix is e.g. "...moe.experts", while ModelOpt's
|
||||
# quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj".
|
||||
if prefix.endswith(".experts"):
|
||||
parent_dot = prefix.rsplit(".experts", 1)[0] + "."
|
||||
|
||||
@@ -467,7 +467,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
elif isinstance(module, RoutedExperts) and hasattr(
|
||||
module.quant_method, "quant_config"
|
||||
):
|
||||
# TODO: support FusedMoE with prequant and 8bit.
|
||||
# TODO: support RoutedExperts with prequant and 8bit.
|
||||
if self.pre_quant and self.load_8bit:
|
||||
raise ValueError(
|
||||
"Prequant BitsAndBytes 8bit models with RoutedExperts "
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -168,7 +168,7 @@ class AXK1MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -122,8 +122,8 @@ class AfmoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
# Routed experts using FusedMoE
|
||||
self.experts = FusedMoE(
|
||||
# Routed experts using FusedMoEFactory
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -13,7 +13,9 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -222,7 +224,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
bias=config.mlp_bias,
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_topk,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -289,7 +289,7 @@ class BailingMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -329,8 +329,8 @@ class BailingMoeV25(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
# Routed experts using FusedMoE
|
||||
self.experts = FusedMoE(
|
||||
# Routed experts using FusedMoEFactory
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
@@ -602,7 +602,7 @@ class BailingMoeV25Model(nn.Module):
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# Expert parameter mappings from FusedMoE
|
||||
# Expert parameter mappings from FusedMoEFactory
|
||||
expert_mappings = list(self.get_expert_mapping())
|
||||
|
||||
def load_param(name: str, tensor: torch.Tensor, shard_id=None) -> bool:
|
||||
|
||||
@@ -320,7 +320,7 @@ class NomicMoE(nn.Module):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.router(hidden_states)
|
||||
# FIXME(Isotr0py): This implementation is too tricky,
|
||||
# we should use FusedMoE instead in the future
|
||||
# we should use FusedMoEFactory instead in the future
|
||||
# after supporting ungated activation for it.
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, router_logits, self.top_k, renormalize=False
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -300,7 +300,7 @@ class Cohere2Moe(nn.Module):
|
||||
self.shared_experts = None
|
||||
self.shared_expert_combination_strategy = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -318,7 +318,7 @@ class Cohere2Moe(nn.Module):
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, self.hidden_size)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
# FusedMoE handles shared expert overlap internally and returns
|
||||
# FusedMoEFactory handles shared expert overlap internally and returns
|
||||
# shared_output + routed_output when shared_experts is set.
|
||||
final_hidden_states = self.experts(hidden_states, router_logits)
|
||||
if self.shared_expert_combination_strategy == "average":
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -159,7 +159,7 @@ class DbrxMoE(nn.Module):
|
||||
|
||||
self.router = DbrxRouter(config, self.params_dtype)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.ffn_config.moe_num_experts,
|
||||
top_k=config.ffn_config.moe_top_k,
|
||||
hidden_size=config.d_model,
|
||||
|
||||
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention, RSWAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -361,7 +361,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -183,7 +183,7 @@ class Ernie4_5_MoeMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_k,
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -260,7 +260,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.text_experts_gate",
|
||||
)
|
||||
|
||||
self.text_experts = FusedMoE(
|
||||
self.text_experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts[0],
|
||||
top_k=config.moe_k,
|
||||
@@ -297,7 +297,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.vision_experts_gate",
|
||||
)
|
||||
|
||||
self.vision_experts = FusedMoE(
|
||||
self.vision_experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts[1],
|
||||
top_k=config.moe_k,
|
||||
|
||||
@@ -29,9 +29,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -126,7 +124,7 @@ class ExaoneMoe(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
@@ -73,7 +73,7 @@ class FlexOlmoMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=hf_config.num_experts,
|
||||
top_k=hf_config.num_experts_per_tok,
|
||||
hidden_size=hf_config.hidden_size,
|
||||
|
||||
@@ -38,7 +38,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -305,14 +305,14 @@ class Gemma4Router(nn.Module):
|
||||
|
||||
|
||||
class Gemma4MoE(nn.Module):
|
||||
"""Mixture of Experts for Gemma4 using vLLM's FusedMoE.
|
||||
"""Mixture of Experts for Gemma4 using vLLM's MoERunner.
|
||||
|
||||
Wraps FusedMoE with custom routing. The router projection is
|
||||
Wraps MoERunner with custom routing. The router projection is
|
||||
external (Gemma4Router) — this class only handles expert dispatch.
|
||||
|
||||
Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
per_expert_scale is folded into routing weights for mathematical
|
||||
correctness with FusedMoE's fused kernel.
|
||||
correctness with MoERunner's fused kernel.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module):
|
||||
self.num_experts = config.num_experts
|
||||
|
||||
# Per-expert output scale folded into routing weights so that
|
||||
# FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
# MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts))
|
||||
|
||||
# Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
# FusedMoE's built-in fused_topk scopes softmax differently, so
|
||||
# MoERunner's built-in fused_topk scopes softmax differently, so
|
||||
# a custom routing function is needed for numerical correctness.
|
||||
# NOTE: self.per_expert_scale is read at call time (not captured into
|
||||
# a local) so that torch.func.functional_call parameter substitution
|
||||
@@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module):
|
||||
gating_output, topk, self.per_expert_scale
|
||||
)
|
||||
|
||||
# FusedMoE experts with custom Gemma4 routing
|
||||
self.experts = FusedMoE(
|
||||
# MoERunner experts with custom Gemma4 routing
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.top_k_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
# MoE expert weight mapping: checkpoint can have either:
|
||||
# 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D)
|
||||
# 2. Already per-expert 2D weights (if quantized)
|
||||
# Map to FusedMoE parameters:
|
||||
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → FusedMoE w2
|
||||
# Map to MoERunner parameters:
|
||||
# moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → MoERunner w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → MoERunner w2
|
||||
num_experts = getattr(self.config, "num_experts", None) or 0
|
||||
# Strategy A: dot-separated suffix
|
||||
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
|
||||
@@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
param = params_dict[moe_name]
|
||||
# Expert weights are already in the correct
|
||||
# orientation for FusedMoE after _weight_iterator:
|
||||
# orientation for MoERunner after _weight_iterator:
|
||||
# gate/up: [I, H] → w1/w3 expects [I, H]
|
||||
# down: [H, I] → w2 expects [H, I]
|
||||
# Scales and other quantization params may be 1D or scalar.
|
||||
@@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM(
|
||||
|
||||
# MoE expert weights: checkpoint stores as 3D packed
|
||||
# tensors. Explode into per-expert 2D weights for
|
||||
# FusedMoE weight_loader.
|
||||
# MoERunner weight_loader.
|
||||
#
|
||||
# Checkpoint format:
|
||||
# moe.gate_up_proj: [E, 2*I, H] (fused gate + up)
|
||||
# moe.down_proj: [E, H, I]
|
||||
#
|
||||
# FusedMoE expects per-expert:
|
||||
# MoERunner expects per-expert:
|
||||
# w1 (gate): [I, H] — first half of gate_up
|
||||
# w3 (up): [I, H] — second half of gate_up
|
||||
# w2 (down): [H, I] — as-is from checkpoint
|
||||
#
|
||||
# No transpose needed: checkpoint orientation already
|
||||
# matches FusedMoE's expected layout.
|
||||
# matches MoERunner's expected layout.
|
||||
if "moe.gate_up_proj" in name and weight.dim() == 3:
|
||||
num_experts = weight.size(0)
|
||||
intermediate_size = weight.size(1) // 2
|
||||
|
||||
@@ -42,7 +42,9 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -164,7 +166,7 @@ class Glm4MoE(nn.Module):
|
||||
)
|
||||
|
||||
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
|
||||
# pattern (see Glm4MoE / FusedMoE wiring there).
|
||||
# pattern (see Glm4MoE / MoERunner wiring there).
|
||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.is_fusion_moe_shared_experts_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
@@ -183,7 +185,7 @@ class Glm4MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
@@ -211,7 +211,7 @@ class MLPBlock(torch.nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
assert config.intermediate_size % self.world_size == 0
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1010,7 +1010,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
|
||||
|
||||
# Use centralized weight remapping for MoE expert parameters.
|
||||
# The FusedMoE refactor moved expert params under
|
||||
# The MoERunner refactor moved expert params under
|
||||
# `mlp.experts.routed_experts.*`; this remaps checkpoint names so
|
||||
# MoE weight/bias keys resolve against params_dict.
|
||||
for name, weight in remap_moe_expert_weights(weights, params_dict):
|
||||
|
||||
@@ -40,7 +40,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -101,7 +101,7 @@ class GraniteMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -441,7 +441,7 @@ class HunYuanSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.shared_mlp = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_mlp,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=top_k,
|
||||
|
||||
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -184,7 +184,7 @@ class HYV3MoEFused(nn.Module):
|
||||
scoring_func = "sigmoid"
|
||||
e_score_correction_bias = self.expert_bias
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -173,7 +173,7 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
|
||||
# For custom routing function
|
||||
self.n_groups = getattr(config, "router_n_groups", -1)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -14,7 +14,9 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -81,7 +83,7 @@ class JambaMoE(nn.Module):
|
||||
prefix=f"{prefix}.router",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
self.num_total_experts,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -174,7 +174,7 @@ class LagunaMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
# Shared expert (optional) - passed to FusedMoE for overlap optimization
|
||||
# Shared expert (optional) - passed to FusedMoEFactory for overlap optimization
|
||||
self.shared_expert: LagunaMLP | None
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.shared_expert = LagunaMLP(
|
||||
@@ -191,20 +191,22 @@ class LagunaMoE(nn.Module):
|
||||
# Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The
|
||||
# checkpoint stores one [num_experts] tensor per MoE layer at
|
||||
# `mlp.experts.e_score_correction_bias`; registering it as a Parameter
|
||||
# on the FusedMoE lets the weight loader pick it up and the router
|
||||
# on the MoERunner lets the weight loader pick it up and the router
|
||||
# add it during top-k selection. The fused top-k bias router requires
|
||||
# float32 regardless of model dtype.
|
||||
e_score_correction_bias = torch.nn.Parameter(
|
||||
torch.zeros(config.num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# MoERunner with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# layer overlap the shared-expert compute with the all2all dispatch.
|
||||
# `apply_routed_scale_to_output=True` makes FusedMoE handle the
|
||||
# `apply_routed_scale_to_output=True` makes MoERunner handle the
|
||||
# FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# layer overlap the shared-expert compute with the all2all dispatch.
|
||||
# `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the
|
||||
# routed_scaling_factor, shared+routed combine, and TP all-reduce
|
||||
# internally, so forward() just returns the final hidden states.
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -15,7 +15,9 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -143,7 +145,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.gate.e_score_correction_bias = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.attention import (
|
||||
ChunkedLocalAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -131,7 +131,7 @@ class Llama4MoE(nn.Module):
|
||||
self.n_physical_experts = self.n_local_experts + self.n_redundant_experts
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -47,7 +47,7 @@ from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -297,7 +297,7 @@ class LongcatMoe(nn.Module):
|
||||
)
|
||||
|
||||
assert config.zero_expert_type is not None
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
zero_expert_type=config.zero_expert_type,
|
||||
e_score_correction_bias=self.router.e_score_correction_bias,
|
||||
num_experts=num_experts,
|
||||
@@ -317,7 +317,7 @@ class LongcatMoe(nn.Module):
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# Align to FusedMoE padded hidden size to avoid dim mismatch
|
||||
# Align to MoERunner padded hidden size to avoid dim mismatch
|
||||
padded_hidden = self.experts.moe_config.hidden_dim
|
||||
if hidden_dim < padded_hidden:
|
||||
hidden_states_padded = torch.nn.functional.pad(
|
||||
@@ -333,7 +333,7 @@ class LongcatMoe(nn.Module):
|
||||
hidden_states_padded.to(self.router_params_dtype)
|
||||
)
|
||||
|
||||
# FusedMoE handles routing memoization and zero expert computation
|
||||
# MoERunner handles routing memoization and zero expert computation
|
||||
# internally. Pass full router_logits (including zero experts) so that
|
||||
# zero experts can be properly identified in routing.
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -171,7 +171,7 @@ class MiMoV2MoE(nn.Module):
|
||||
torch.empty(config.n_routed_experts, dtype=self.gate_dtype)
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -39,9 +39,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -95,7 +93,7 @@ class MiniMaxM2MoE(nn.Module):
|
||||
else:
|
||||
self.e_score_correction_bias = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
scoring_func=config.scoring_func,
|
||||
|
||||
@@ -39,9 +39,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -123,7 +121,7 @@ class MixtralMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
activation_without_mul,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -209,7 +209,7 @@ class NemotronHMoE(nn.Module):
|
||||
self.fc1_latent_proj = None
|
||||
self.fc2_latent_proj = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -32,9 +32,7 @@ from vllm.distributed import (
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -94,7 +92,7 @@ class OlmoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
StaticSinkAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -195,7 +195,7 @@ class OpenPanguMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -32,7 +32,9 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -325,7 +327,7 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None # type: ignore[assignment]
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
@@ -342,7 +344,7 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def maybe_get_fused_moe(self) -> FusedMoE:
|
||||
def maybe_get_fused_moe(self) -> FusedMoEFactory:
|
||||
return self.experts
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -35,9 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -273,7 +271,7 @@ class PhiMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -40,9 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -165,7 +163,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -201,7 +201,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
self.shared_expert_gate = None
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
@@ -228,13 +228,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
else:
|
||||
# Actually this will be dead code, since we always pass gate into
|
||||
# FusedMoE in the current implementation. But we keep this code
|
||||
# MoERunner in the current implementation. But we keep this code
|
||||
# here for clarity and future flexibility.
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -20,9 +20,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm as Qwen3NextRMSNorm,
|
||||
@@ -175,7 +173,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
@@ -208,7 +206,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -334,7 +334,7 @@ class SarvamMLAMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -68,7 +68,7 @@ class FusedMoEBlock(nn.Module):
|
||||
f"the number of experts {config.moe_num_experts}."
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_top_k,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -376,7 +376,7 @@ class FusedMoEBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.share_expert",
|
||||
)
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.share_expert,
|
||||
gate=self.gate,
|
||||
num_experts=config.moe_num_experts,
|
||||
@@ -404,7 +404,7 @@ class FusedMoEBlock(nn.Module):
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
else:
|
||||
# TODO(bnell): this gate could be moved into the FusedMoE?
|
||||
# TODO(bnell): this gate could be moved into the MoERunner?
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
|
||||
@@ -29,7 +29,11 @@ from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
@@ -53,7 +57,7 @@ class TransformersMoEState:
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersMoERunner(MoERunner):
|
||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||
"""Custom MoERunner for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
@@ -297,7 +301,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in FusedMoE.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
@@ -331,7 +335,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
if fuser.shared_name:
|
||||
routed += " + shared experts"
|
||||
logger.info_once(
|
||||
"Fused: %s (%s) -> FusedMoE (internal routing)",
|
||||
"Fused: %s (%s) -> MoERunner (internal routing)",
|
||||
routed,
|
||||
moe_block_cls,
|
||||
)
|
||||
@@ -373,10 +377,10 @@ class MoEMixin(MixtureOfExperts):
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
logger.info_once(
|
||||
"Fused: experts (%s) -> FusedMoE (external routing)",
|
||||
"Fused: experts (%s) -> MoERunner (external routing)",
|
||||
experts_cls,
|
||||
)
|
||||
fused_experts = FusedMoE(**kwargs)
|
||||
fused_experts = FusedMoEFactory(**kwargs)
|
||||
moe_block.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
|
||||
@@ -108,7 +108,7 @@ def _extract_data_from_fused_moe_module(
|
||||
m_: torch.nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
"""
|
||||
Extract weights, weight scales and num_topk from FusedMoE module.
|
||||
Extract weights, weight scales and num_topk from MoERunner module.
|
||||
"""
|
||||
assert isinstance(m_, MoERunner)
|
||||
m = m_.routed_experts
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -279,7 +279,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.fuse_shared_experts else None
|
||||
@@ -308,7 +308,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
|
||||
@@ -51,7 +51,7 @@ from .model import DeepseekV4DecoderLayer
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which MoERunner method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
@@ -670,7 +670,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.physical_expert_start = self.experts_start_idx
|
||||
self.physical_expert_end = self.experts_end_idx
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
@@ -736,7 +736,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
org_shape = hidden_states.shape
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
|
||||
@@ -59,7 +59,7 @@ from .model import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -729,7 +729,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.n_local_experts = config.n_routed_experts // self.tp_size
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -54,7 +54,7 @@ from .model import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -646,7 +646,7 @@ def _load_inkling_weights(
|
||||
weight.shard_id = shard_id
|
||||
|
||||
# MoE expert tensors (fused stacked, routed + shared sink): translate
|
||||
# the checkpoint layout to per-expert FusedMoE loads.
|
||||
# the checkpoint layout to per-expert MoERunner loads.
|
||||
moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
|
||||
if moe_match is not None and moe_match.group("mlp") in moe_modules:
|
||||
moe = moe_modules[moe_match.group("mlp")]
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
|
||||
"""Inkling mixture-of-experts on vLLM's MoERunner abstraction.
|
||||
|
||||
Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
|
||||
the routed experts, log-sigmoid renormalization over the k routed + S shared
|
||||
"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
|
||||
through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
|
||||
through vLLM's MoERunner (which handles TP/EP); the sink experts run in
|
||||
:class:`InklingSinkExperts` -- replicated across EP ranks (every token
|
||||
activates every sink) and always bf16 (the checkpoint excludes every
|
||||
``shared_experts`` from quantization).
|
||||
@@ -35,7 +35,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -265,7 +265,7 @@ class InklingGate(nn.Module):
|
||||
|
||||
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the FusedMoE layer will run with (mirrors
|
||||
"""EP size the MoERunner layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled)."""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
@@ -444,7 +444,7 @@ class InklingMoE(nn.Module):
|
||||
|
||||
# The released MXFP4 checkpoint keeps the first routed-expert layer
|
||||
# in bf16 and lists its two expert weights explicitly in Quark's
|
||||
# exclusion list. FusedMoE asks for a quant method at the module
|
||||
# exclusion list. FusedMoEFactory asks for a quant method at the module
|
||||
# prefix, so an exact weight exclusion would otherwise be missed and
|
||||
# the bf16 tensors would be loaded into MXFP4 parameters.
|
||||
routed_quant_config = quant_config
|
||||
@@ -458,7 +458,7 @@ class InklingMoE(nn.Module):
|
||||
if routed_weights <= excluded:
|
||||
routed_quant_config = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -503,7 +503,7 @@ class InklingMoE(nn.Module):
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""FusedMoE ``custom_routing_function``: the routed top-k slice of the
|
||||
"""MoERunner ``custom_routing_function``: the routed top-k slice of the
|
||||
full (routed + sink) selection.
|
||||
|
||||
forward() stashes its selection (keyed by logits identity) so the
|
||||
@@ -523,7 +523,7 @@ class InklingMoE(nn.Module):
|
||||
router_logits = self.gate.compute_logits(x)
|
||||
num_tokens = x.shape[0]
|
||||
# One gate select per layer: the routed slice is stashed for the
|
||||
# routing function inside the FusedMoE op; the sink gammas are the
|
||||
# routing function inside the MoERunner op; the sink gammas are the
|
||||
# trailing columns.
|
||||
k = self.gate.topk
|
||||
weights, ids = self.gate.select_experts(router_logits)
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
@@ -439,7 +439,7 @@ class InklingMoE(nn.Module):
|
||||
# power-of-two EP sizes (n_routed is a power of two).
|
||||
num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -15,7 +15,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
@@ -260,7 +260,7 @@ class KimiMoE(nn.Module):
|
||||
self.routed_expert_up_proj = None
|
||||
self.routed_output_transform = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=num_experts_per_token,
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.forward_context import get_forward_context, is_forward_context_availab
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
@@ -560,7 +560,7 @@ class KimiMoE(nn.Module):
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=num_experts_per_token,
|
||||
@@ -578,7 +578,7 @@ class KimiMoE(nn.Module):
|
||||
scoring_func=config.moe_router_activation_func,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
# Down projection runs outside FusedMoE so it can overlap the
|
||||
# Down projection runs outside MoERunner so it can overlap the
|
||||
# router gate on the aux stream (see forward()); the original
|
||||
# hidden states are passed to forward() as shared_experts_input
|
||||
# so shared experts still see the untransformed input.
|
||||
@@ -689,7 +689,7 @@ class KimiMoE(nn.Module):
|
||||
)
|
||||
else:
|
||||
# Routed experts consume the down-projected latent; shared experts
|
||||
# (inside FusedMoE) get the original hidden states via
|
||||
# (inside MoERunner) get the original hidden states via
|
||||
# shared_experts_input.
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=routed_hidden_states,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -391,7 +391,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
# always-on shared expert; aiter applies the routed scaling internally.
|
||||
# Every other path (vLLM top-k bias router, or no fusion) applies the
|
||||
# routed scaling to the MoE output here.
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -31,7 +31,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -244,7 +244,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
Reference in New Issue
Block a user