From 6e311c6e2014a54868e2cbedf944a6873d06228b Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Fri, 31 Jul 2026 06:28:18 -0400 Subject: [PATCH] [MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941) Signed-off-by: Bill Nell Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../distributed/test_eplb_fused_moe_layer.py | 8 ++--- .../test_eplb_fused_moe_layer_dep_nvfp4.py | 8 ++--- tests/kernels/moe/test_deepep_v2_moe.py | 2 +- tests/kernels/moe/test_moe_layer.py | 19 +++++++---- .../moe/test_moe_weight_loading_padded.py | 2 +- tests/kernels/moe/test_routing_simulator.py | 8 ++--- .../test_shared_fused_moe_routed_transform.py | 20 ++++++------ tests/kernels/moe/test_zero_expert_moe.py | 29 +++++++++-------- tests/lora/test_lora_manager.py | 2 +- tests/quantization/test_fp8.py | 4 +-- vllm/_custom_ops.py | 2 +- vllm/config/quantization.py | 2 +- .../kv_connector/v1/example_connector.py | 2 +- vllm/lora/layers/fused_moe.py | 2 +- vllm/lora/lora_model.py | 4 +-- vllm/lora/model_manager.py | 4 +-- .../layers/fused_moe/__init__.py | 4 +-- .../model_executor/layers/fused_moe/config.py | 6 ++-- vllm/model_executor/layers/fused_moe/layer.py | 3 +- .../layers/fused_moe/oracle/int_wna16.py | 4 +-- .../fused_moe/routed_experts_capturer.py | 2 +- .../layers/fused_moe/runner/moe_runner.py | 4 +-- .../fused_moe/runner/moe_runner_interface.py | 2 +- .../compressed_tensors/compressed_tensors.py | 2 +- .../layers/quantization/modelopt.py | 2 +- .../model_loader/bitsandbytes_loader.py | 2 +- vllm/model_executor/models/AXK1.py | 4 +-- vllm/model_executor/models/afmoe.py | 6 ++-- vllm/model_executor/models/aria.py | 6 ++-- vllm/model_executor/models/bailing_moe.py | 4 +-- .../models/bailing_moe_linear.py | 8 ++--- vllm/model_executor/models/bert_with_rope.py | 2 +- vllm/model_executor/models/cohere2_moe.py | 6 ++-- vllm/model_executor/models/dbrx.py | 4 +-- vllm/model_executor/models/deepseek_v2.py | 4 +-- vllm/model_executor/models/ernie45_moe.py | 4 +-- vllm/model_executor/models/ernie45_vl_moe.py | 6 ++-- vllm/model_executor/models/exaone_moe.py | 6 ++-- vllm/model_executor/models/flex_olmo.py | 4 +-- vllm/model_executor/models/gemma4.py | 32 +++++++++---------- vllm/model_executor/models/glm4_moe.py | 8 +++-- vllm/model_executor/models/gpt_oss.py | 6 ++-- vllm/model_executor/models/granitemoe.py | 4 +-- vllm/model_executor/models/hunyuan_v1.py | 4 +-- vllm/model_executor/models/hy_v3.py | 4 +-- vllm/model_executor/models/interns1_pro.py | 4 +-- vllm/model_executor/models/jamba.py | 6 ++-- vllm/model_executor/models/laguna.py | 16 ++++++---- vllm/model_executor/models/lfm2_moe.py | 6 ++-- vllm/model_executor/models/llama4.py | 4 +-- vllm/model_executor/models/longcat_flash.py | 8 ++--- vllm/model_executor/models/mimo_v2.py | 4 +-- vllm/model_executor/models/minimax_m2.py | 6 ++-- vllm/model_executor/models/mixtral.py | 6 ++-- vllm/model_executor/models/nemotron_h.py | 4 +-- vllm/model_executor/models/olmoe.py | 6 ++-- vllm/model_executor/models/openpangu.py | 4 +-- vllm/model_executor/models/param2moe.py | 8 +++-- vllm/model_executor/models/phimoe.py | 6 ++-- vllm/model_executor/models/qwen2_moe.py | 6 ++-- vllm/model_executor/models/qwen3_moe.py | 8 ++--- vllm/model_executor/models/qwen3_next.py | 8 ++--- vllm/model_executor/models/sarvam.py | 4 +-- vllm/model_executor/models/step3_text.py | 4 +-- vllm/model_executor/models/step3p5.py | 6 ++-- .../model_executor/models/transformers/moe.py | 16 ++++++---- .../model_executor/warmup/deep_gemm_warmup.py | 2 +- vllm/models/deepseek_v4/amd/model.py | 6 ++-- vllm/models/deepseek_v4/amd/mtp.py | 2 +- vllm/models/deepseek_v4/nvidia/model.py | 6 ++-- vllm/models/deepseek_v4/nvidia/mtp.py | 2 +- vllm/models/deepseek_v4/xpu/model.py | 4 +-- vllm/models/deepseek_v4/xpu/mtp.py | 2 +- vllm/models/inkling/amd/model.py | 2 +- vllm/models/inkling/amd/moe.py | 16 +++++----- vllm/models/inkling/nvidia/moe.py | 4 +-- vllm/models/kimi_k3/amd/linear.py | 4 +-- vllm/models/kimi_k3/nvidia/model.py | 8 ++--- vllm/models/minimax_m3/amd/model.py | 4 +-- vllm/models/minimax_m3/nvidia/model.py | 4 +-- 80 files changed, 243 insertions(+), 234 deletions(-) diff --git a/tests/distributed/test_eplb_fused_moe_layer.py b/tests/distributed/test_eplb_fused_moe_layer.py index 7d5e58b26e..d8d870bc98 100644 --- a/tests/distributed/test_eplb_fused_moe_layer.py +++ b/tests/distributed/test_eplb_fused_moe_layer.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Test that the interaction between EPLB and FusedMoE Layer is okay +# Test that the interaction between EPLB and FusedMoEFactory Layer is okay from dataclasses import dataclass @@ -16,7 +16,7 @@ from vllm.distributed.parallel_state import ( get_eplb_group, get_tp_group, ) -from vllm.model_executor.layers.fused_moe.layer import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner from .eplb_utils import distributed_run, set_env_vars_and_device @@ -69,8 +69,8 @@ def make_fused_moe_layer( rank: int, layer_idx: int, test_config: TestConfig, -) -> FusedMoE: - fml = FusedMoE( +) -> MoERunner: + fml = FusedMoEFactory( num_experts=test_config.num_experts, top_k=test_config.num_topk, hidden_size=test_config.hidden_size, diff --git a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py index e2d54821ce..fe11e13cd6 100644 --- a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py +++ b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Test that the interaction between EPLB and FusedMoE Layer is okay for DP w/ NVFP4 +# Test that the interaction between EPLB and MoERunner Layer is okay for DP w/ NVFP4 from dataclasses import dataclass @@ -19,7 +19,7 @@ from vllm.distributed.parallel_state import ( get_eplb_group, ) from vllm.forward_context import set_forward_context -from vllm.model_executor.layers.fused_moe.layer import FusedMoE +from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory, MoERunner from vllm.model_executor.layers.quantization.modelopt import ( ModelOptNvFp4Config, ModelOptNvFp4FusedMoE, @@ -44,7 +44,7 @@ def make_fused_moe_layer( rank: int, layer_idx: int, test_config: TestConfig, -) -> FusedMoE: +) -> MoERunner: quant_config = None device = torch.device(f"cuda:{rank}") @@ -55,7 +55,7 @@ def make_fused_moe_layer( exclude_modules=[], ) - fml = FusedMoE( + fml = FusedMoEFactory( num_experts=test_config.num_experts, top_k=test_config.num_topk, hidden_size=test_config.hidden_size, diff --git a/tests/kernels/moe/test_deepep_v2_moe.py b/tests/kernels/moe/test_deepep_v2_moe.py index e4cb5bff09..a2ff56be51 100644 --- a/tests/kernels/moe/test_deepep_v2_moe.py +++ b/tests/kernels/moe/test_deepep_v2_moe.py @@ -409,7 +409,7 @@ def _deep_ep_v2_moe_cudagraph( vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm") with set_current_vllm_config(vllm_cfg): - # Initialize vLLM parallel state (needed by FusedMoE layer) + # Initialize vLLM parallel state (needed by MoERunner layer) temp_file = tempfile.mktemp() init_distributed_environment( world_size=pgi.world_size, diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index cc8d9c36dc..1942ce96bc 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -42,7 +42,11 @@ from vllm.distributed.device_communicators.all_reduce_utils import ( from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace from vllm.forward_context import set_forward_context -from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, fused_experts +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + MoERunner, + fused_experts, +) from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.router.router_factory import ( @@ -80,7 +84,7 @@ NUM_EXPERTS = [8, 64] TOP_KS = [2, 6] # dp_size, tp_size, use_ep -# Note: DP+TP is not yet supported in the FusedMoE layer. +# Note: DP+TP is not yet supported in the FusedMoEFactory layer. PARALLEL_COMBOS = [ [1, 2, False], [1, 4, False], @@ -1023,7 +1027,7 @@ def make_fused_moe_layer( kwargs["routed_input_transform"] = routed_input_transform kwargs["routed_output_transform"] = routed_output_transform - layer = FusedMoE( + layer = FusedMoEFactory( num_experts=global_num_experts, top_k=top_k, hidden_size=hidden_size, @@ -1267,7 +1271,7 @@ def _test_body_eplb( ): output_before = sp_wrapper(moe_layer)(hidden_states, router_logits) - # Create a fresh FusedMoE layer with enable_eplb=True + # Create a fresh MoERunner layer with enable_eplb=True # Delete the original layer's registration so the constructor can # re-use the same "from_forward_context" prefix cc = vllm_config.compilation_config @@ -1401,7 +1405,8 @@ def _run_one_config( - When is_sequence_parallel=True (EP + sequence splitting): * ep_size: Number of expert parallel ranks (equals dp_size * tp_size) - * tp_size: Number of ranks to split sequence across (becomes sp_size in FusedMoE) + * tp_size: Number of ranks to split sequence across (becomes sp_size in + MoERunner) * Weights are chunked by ep_size (experts) but NOT by tp_size * Input sequences are chunked by tp_size (via sp_wrapper) """ @@ -1476,8 +1481,8 @@ def _run_one_config( torch.accelerator.empty_cache() with set_current_vllm_config(vllm_config): - # Chunk weights for EP BEFORE creating FusedMoE - # FusedMoE uses EP-chunked weights and handles reductions internally + # Chunk weights for EP BEFORE creating MoERunner. + # MoERunner uses EP-chunked weights and handles reductions internally. if ep_size > 1: # Split experts across ranks (dimension 0 is the expert dimension) # When EP is enabled, use EP group rank and ep_size for chunking diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index 2fd4e0fed5..b7dad4da5a 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for FusedMoE weight loading with padded hidden dimensions. +"""Tests for FusedMoEFactory weight loading with padded hidden dimensions. When using DeepEP backends or NIXL EP with models like nemotron_h, hidden_size may be rounded up (e.g., 2688 -> 3072) for backend requirements. diff --git a/tests/kernels/moe/test_routing_simulator.py b/tests/kernels/moe/test_routing_simulator.py index 4ef984a329..35b1af618c 100644 --- a/tests/kernels/moe/test_routing_simulator.py +++ b/tests/kernels/moe/test_routing_simulator.py @@ -6,7 +6,7 @@ Test script for the token-to-expert routing simulator. This script demonstrates how to use the routing simulator to test different routing strategies and analyze their performance, including -integration tests with FusedMoE layer. +integration tests with FusedMoEFactory layer. """ import tempfile @@ -77,11 +77,11 @@ def test_basic_functionality( def test_routing_strategy_integration(monkeypatch, device): """Test that the routing strategy environment variable works with - FusedMoE.""" + FusedMoEFactory.""" pytest.importorskip("vllm.model_executor.layers.fused_moe.layer") import vllm.envs as envs - from vllm.model_executor.layers.fused_moe.layer import FusedMoE + from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory # Test parameters num_tokens = 32 @@ -111,7 +111,7 @@ def test_routing_strategy_integration(monkeypatch, device): ) for strategy in strategies: - fused_moe = FusedMoE( + fused_moe = FusedMoEFactory( num_experts=num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py index 4515021a4e..af9455b960 100644 --- a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py +++ b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Tests for FusedMoE with routed_input_transform. +Tests for FusedMoEFactory with routed_input_transform. -Verifies that applying routed_input_transform inside FusedMoE +Verifies that applying routed_input_transform inside FusedMoEFactory produces the same results as applying the transform manually outside. """ @@ -13,7 +13,7 @@ import torch.nn as nn from vllm.config import VllmConfig, set_current_vllm_config from vllm.forward_context import set_forward_context -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.platforms import current_platform from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed @@ -133,9 +133,9 @@ def test_routed_input_transform_inside_vs_outside( workspace_init, monkeypatch, ): - """Compare FusedMoE with transform inside vs manually applying outside. - Method A (inside): FusedMoE with routed_input_transform - Method B (outside): Manually transform, then FusedMoE without transform + """Compare FusedMoEFactory with transform inside vs manually applying outside. + Method A (inside): FusedMoEFactory with routed_input_transform + Method B (outside): Manually transform, then FusedMoEFactory without transform """ if current_platform.is_rocm(): monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0") @@ -157,8 +157,8 @@ def test_routed_input_transform_inside_vs_outside( routed_transform = SimpleLinear(hidden_size, latent_size, dtype) with set_current_vllm_config(vllm_config): - # Method A: FusedMoE WITH routed_input_transform - moe_with_transform = FusedMoE( + # Method A: FusedMoEFactory WITH routed_input_transform + moe_with_transform = FusedMoEFactory( shared_experts=shared_experts, routed_input_transform=routed_transform, num_experts=num_experts, @@ -173,9 +173,9 @@ def test_routed_input_transform_inside_vs_outside( prefix="moe_with_transform", ) - # Method B: FusedMoE WITHOUT routed_input_transform + # Method B: FusedMoEFactory WITHOUT routed_input_transform # Note: shared_experts=None because when transform is done outside, - moe_without_transform = FusedMoE( + moe_without_transform = FusedMoEFactory( shared_experts=None, routed_input_transform=None, num_experts=num_experts, diff --git a/tests/kernels/moe/test_zero_expert_moe.py b/tests/kernels/moe/test_zero_expert_moe.py index 71e33b7dfa..c2084800f9 100644 --- a/tests/kernels/moe/test_zero_expert_moe.py +++ b/tests/kernels/moe/test_zero_expert_moe.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for FusedMoE with zero experts. +"""Tests for FusedMoEFactory with zero experts. Verifies that: - The ZeroExpertRouter is properly created and used as the layer router. -- A forward pass through FusedMoE with zero experts produces correct output. +- A forward pass through FusedMoEFactory with zero experts produces correct output. - The output decomposes correctly into real expert + zero expert contributions. Note: tests generated with Claude. @@ -15,7 +15,7 @@ import torch from vllm.config import VllmConfig, set_current_vllm_config from vllm.forward_context import get_forward_context, set_forward_context -from vllm.model_executor.layers.fused_moe.layer import FusedMoE +from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory from vllm.model_executor.layers.fused_moe.router.zero_expert_router import ( ZeroExpertRouter, ) @@ -24,7 +24,7 @@ from vllm.v1.worker.workspace import init_workspace_manager @pytest.fixture def zero_expert_moe(dist_init, default_vllm_config): - """Create a FusedMoE layer with zero experts.""" + """Create a FusedMoEFactory layer with zero experts.""" num_experts = 4 top_k = 2 # hidden_size must be >= 256 for the zero expert identity kernel to @@ -45,7 +45,7 @@ def zero_expert_moe(dist_init, default_vllm_config): with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config): init_workspace_manager(torch.accelerator.current_device_index()) - layer = FusedMoE( + layer = FusedMoEFactory( zero_expert_type="identity", e_score_correction_bias=e_score_correction_bias, num_experts=num_experts, @@ -66,7 +66,7 @@ def zero_expert_moe(dist_init, default_vllm_config): @pytest.mark.parametrize("num_tokens", [1, 32]) def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_tokens): - """Verify that FusedMoE with zero_expert_type creates a ZeroExpertRouter.""" + """Verify that FusedMoEFactory with zero_expert_type creates a ZeroExpertRouter.""" layer, _ = zero_expert_moe assert isinstance(layer.router, ZeroExpertRouter), ( f"Expected ZeroExpertRouter but got {type(layer.router).__name__}." @@ -83,7 +83,10 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token @pytest.mark.parametrize("num_tokens", [1, 32]) def test_zero_expert_moe_forward(zero_expert_moe, num_tokens): - """Run a forward pass through FusedMoE with zero experts and verify output shape.""" + """ + Run a forward pass through FusedMoEFactory with zero experts + and verify output shape. + """ layer, vllm_config = zero_expert_moe hidden_size = layer.routed_experts.hidden_size @@ -118,16 +121,16 @@ def test_zero_expert_moe_forward(zero_expert_moe, num_tokens): @pytest.mark.parametrize("num_tokens", [1, 32]) def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens): - """Validate that the FusedMoE output equals a plain FusedMoE + """Validate that the FusedMoEFactory output equals a plain FusedMoEFactory output (real experts only) plus the zero expert contribution. The key invariant is: zero_layer.forward(h, r_full) == plain_layer.forward(h, r_real) + zero_expert_output - We create a plain FusedMoE layer with the same weights and real-expert-only + We create a plain FusedMoEFactory layer with the same weights and real-expert-only router logits, compute the zero expert output via the ZeroExpertRouter, and - verify the sum matches the FusedMoE output. + verify the sum matches the FusedMoEFactory output. """ layer, vllm_config = zero_expert_moe num_experts = 4 @@ -152,9 +155,9 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens): with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config): get_forward_context().all_moe_layers = None - # Create a plain FusedMoE layer with the same config but no zero + # Create a plain FusedMoEFactory layer with the same config but no zero # experts. Use a separate prefix to avoid collision. - plain_layer = FusedMoE( + plain_layer = FusedMoEFactory( num_experts=num_experts, top_k=layer.routed_experts.top_k, hidden_size=layer.routed_experts.hidden_size, @@ -210,7 +213,7 @@ def test_zero_expert_moe_output_decomposition(zero_expert_moe, num_tokens): expected, atol=4e-3, rtol=4e-3, - msg="FusedMoE output should equal plain FusedMoE output " + msg="FusedMoEFactory output should equal plain FusedMoEFactory output " "plus zero expert contribution", ) diff --git a/tests/lora/test_lora_manager.py b/tests/lora/test_lora_manager.py index 1db05b4b09..834273b8c5 100644 --- a/tests/lora/test_lora_manager.py +++ b/tests/lora/test_lora_manager.py @@ -198,7 +198,7 @@ def test_dedup_shared_module_across_paths(default_vllm_config, dist_init, dummy_ self.gate = gate # canonical path: "moe.gate" # Inner submodule holding the SAME gate instance under another - # path. This mirrors how FusedMoE.runner.gate references the + # path. This mirrors how MoERunner.runner.gate references the # block's gate in qwen3_moe. class _Runner(nn.Module): def __init__(self, g): diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 4a2db91af9..55ad9c6ddb 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -21,7 +21,7 @@ from vllm.model_executor.layers.attention.attention import ( Attention, set_default_quant_scales, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, Fp8KVCacheMethod, @@ -393,7 +393,7 @@ def test_fp8_reloading( method.use_marlin = use_marlin else: - layer = FusedMoE( + layer = FusedMoEFactory( num_experts=1, top_k=1, hidden_size=1, diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 065357addc..357e882834 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -1022,7 +1022,7 @@ def cutlass_fp4_moe_mm( An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs the gemms for each combination based on the specified problem sizes. - This is used as the MoE gemm during NVFP4 Quantized FusedMoE forward. + This is used as the MoE gemm during NVFP4 Quantized MoERunner forward. - a/b_tensors: the NVFP4 a_ptrs and b_ptrs tensors which are quantized input and expert weights. - a_/b_scales: The blockscales in FP8-E4M3 precision diff --git a/vllm/config/quantization.py b/vllm/config/quantization.py index 370fbaf273..4f2a40426c 100644 --- a/vllm/config/quantization.py +++ b/vllm/config/quantization.py @@ -88,7 +88,7 @@ class QuantizationConfigArgs: """Spec applied to ``LinearBase`` layers.""" moe: QuantSpec | None = None - """Spec applied to ``FusedMoE`` layers.""" + """Spec applied to ``FusedMoEFactory`` layers.""" ignore: list[str] = Field(default_factory=list) """Layers to skip quantization for.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py index db64b893e9..58195d1356 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py @@ -170,7 +170,7 @@ class ExampleConnector(KVConnectorBase_V1): # Only process layers that have kv_cache # attribute (attention layers) Skip non-attention - # layers like FusedMoE/MLP etc. + # layers like FusedMoEFactory/MLP etc. kv_cache_layer = getattr(layer, "kv_cache", None) if kv_cache_layer is None: continue diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index b447af8780..d80848b14b 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -44,7 +44,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): "Monolithic kernels are not supported for Fused MoE LoRA." ) - # Use the MoE-aware TP rank/size: when EP is active, FusedMoE collapses + # Use the MoE-aware TP rank/size: when EP is active, MoERunner collapses # moe_parallel_config.tp_size to 1 (experts are sharded across the # TP group instead). moe_parallel_config = self.moe_config.moe_parallel_config diff --git a/vllm/lora/lora_model.py b/vllm/lora/lora_model.py index 01d0496328..f094f11f09 100644 --- a/vllm/lora/lora_model.py +++ b/vllm/lora/lora_model.py @@ -24,7 +24,7 @@ logger = init_logger(__name__) @dataclass(frozen=True) class MoEEPLoadSpec: - """Per-expert-parallel slicing metadata for one FusedMoE LoRA module. + """Per-expert-parallel slicing metadata for one FusedMoEFactory LoRA module. Threaded into the LoRA loader so per-expert weights from EP ranks other than this one can be skipped before they ever hit CPU memory. @@ -193,7 +193,7 @@ class LoRAModel: skip_prefixes: List of module name prefixes to skip during loading. Models can define this to skip modules not used in inference (e.g., MTP layers). Format: ["mtp."] - moe_ep_spec: When 2D FusedMoE LoRA modules are present with + moe_ep_spec: When 2D FusedMoEFactory LoRA modules are present with expert parallelism enabled, the (ep_rank, local, global) slicing metadata shared across all MoE layers. Non-local expert weights are skipped at read time instead of being diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 2c4be56786..ba545bc9c7 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -739,7 +739,7 @@ class LoRAModelManager: def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None: for module_name, new_module_names in self.packed_modules.items(): - # For 2D FusedMoE modules with EP, narrow the per-expert + # For 2D RoutedExperts modules with EP, narrow the per-expert # sub-module list to this rank's owned experts so pack_moe # produces a tensor sized to local_num_experts directly. packed_module_names = new_module_names @@ -1096,7 +1096,7 @@ class LoRAModelManager: def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None: """ - Per-rank slicing metadata for 2D FusedMoE LoRA modules. + Per-rank slicing metadata for 2D RoutedEXperts LoRA modules. """ if not self._use_ep or not self._is_moe: return None diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 8f434272f5..17f287556b 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) from vllm.model_executor.layers.fused_moe.layer import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.modular_kernel import ( @@ -63,7 +63,7 @@ def get_config() -> dict[str, Any] | None: __all__ = [ - "FusedMoE", + "FusedMoEFactory", "FusedMoERouter", "FusedMoEConfig", "FusedMoEQuantConfig", diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index b4aed6acf8..f2ee520d09 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1134,9 +1134,9 @@ class FusedMoEParallelConfig: level's of parallelism to use in the fused moe layer. Args: - tp_size_ (int): `tp_size` passed into the FusedMoE constructor. - pcp_size_ (int): `pcp_size` passed into the FusedMoE constructor. - dp_size_ (int): `dp_size` passed into the FusedMoE constructor. + tp_size_ (int): `tp_size` passed into the FusedMoEFactory constructor. + pcp_size_ (int): `pcp_size` passed into the FusedMoEFactory constructor. + dp_size_ (int): `dp_size` passed into the FusedMoEFactory constructor. vllm_parallel_config (ParallelConfig): vLLM's parallel config object which contains the `enable_expert_parallel` flag. diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 6339ab171e..cfbadba7ac 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -96,8 +96,7 @@ def determine_expert_counts( return global_num_experts, logical_num_experts, num_fused_shared_experts -# TODO: rename this -def FusedMoE( +def FusedMoEFactory( num_experts: int, # Global number of experts top_k: int, hidden_size: int, diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 9d9d3aa765..1cd046ab26 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -1016,7 +1016,7 @@ def _process_weights_xpu( w2: [E, K, N] int4 (uint8 storage [E, K, N // 2]) w2_scales: [E, K, N // group_size] params_dtype - Input GPTQ layout from FusedMoE.weight_loader: + Input GPTQ layout from MoERunner.weight_loader: w13: [E, K // 8, 2*N] int32 (8 nibbles per int32 along the input dim) w13_scales: [E, K // group_size, 2*N] params_dtype w2: [E, N // 8, K] int32 @@ -1419,7 +1419,7 @@ def convert_to_wna16_moe_kernel_format( Args: backend: the selected ``WNA16MoEBackend``. - layer: the ``FusedMoE`` layer whose parameters are being prepared. + layer: the ``MoERunner`` layer whose parameters are being prepared. quant_config: the ``QuantizationConfig`` for this layer. input_dtype: optional activation dtype, usually should be 16 bit. """ diff --git a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py index c5553a90eb..c71ee30fb0 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py @@ -180,7 +180,7 @@ class RoutedExpertsCapturer: # (trailing rows are SP ceil-div padding). The TP group # is always initialized on real rollout workers, and # every rank in the group reaches this branch in - # lockstep (bind is per-FusedMoE layer, SP is a global + # lockstep (bind is per-FusedMoEFactory layer, SP is a global # condition), so a bare all_gather here will not # deadlock -- let it raise if the precondition is # violated rather than skip silently. diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index cdf436bcc5..202ecac9e5 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -353,7 +353,7 @@ class MoERunner(MoERunnerInterface): ) -> tuple[torch.Tensor, torch.Tensor | None]: """Apply transform for routed experts (e.g., latent projection). - This is called by FusedMoE.forward_native. The original hidden_states + This is called by MoERunner.forward_native. The original hidden_states is saved separately so shared experts get [S, hidden_size] while routed experts get the transformed [S, moe_latent_size]. @@ -646,7 +646,7 @@ class MoERunner(MoERunnerInterface): ): # If router/gate provided, then apply it here. # (Note: This code runs only when "overlapped mode" is on to allow - # parallel execution of shared experts with the FusedMoE via + # parallel execution of shared experts with the RoutedExperts via # separate cuda stream) if self._shared_experts is not None: assert shared_experts_input is not None diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py index 5b8e0a3bff..5129c9f22d 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py @@ -62,7 +62,7 @@ class MoERunnerInterface(PluggableLayer, ABC): ######################################################################## # - # FusedMoE layer methods + # FusedMoEFactory layer methods # ######################################################################## diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 260a8309ff..6fea370824 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -215,7 +215,7 @@ class CompressedTensorsConfig(QuantizationConfig): def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX """ Helper function to update target_scheme_map - since linear layers get fused into FusedMoE + since linear layers get fused into RoutedExperts targeting 'Linear' needs to also match RoutedExperts modules. """ diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 48ebf0fef5..1c94db5a2d 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -2363,7 +2363,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): if key.startswith(prefix_dot): return info["quant_algo"].upper() - # FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's + # RoutedExperts expert prefix is e.g. "...moe.experts", while ModelOpt's # quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj". if prefix.endswith(".experts"): parent_dot = prefix.rsplit(".experts", 1)[0] + "." diff --git a/vllm/model_executor/model_loader/bitsandbytes_loader.py b/vllm/model_executor/model_loader/bitsandbytes_loader.py index cc9af05af7..525de8d6a8 100644 --- a/vllm/model_executor/model_loader/bitsandbytes_loader.py +++ b/vllm/model_executor/model_loader/bitsandbytes_loader.py @@ -467,7 +467,7 @@ class BitsAndBytesModelLoader(BaseModelLoader): elif isinstance(module, RoutedExperts) and hasattr( module.quant_method, "quant_config" ): - # TODO: support FusedMoE with prequant and 8bit. + # TODO: support RoutedExperts with prequant and 8bit. if self.pre_quant and self.load_8bit: raise ValueError( "Prequant BitsAndBytes 8bit models with RoutedExperts " diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index 7818200dff..862604921e 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -43,7 +43,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -168,7 +168,7 @@ class AXK1MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index 0122d01958..6ed67861e9 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -18,7 +18,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, MoERunner, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -122,8 +122,8 @@ class AfmoeMoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - # Routed experts using FusedMoE - self.experts = FusedMoE( + # Routed experts using FusedMoEFactory + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.num_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 5117f541cf..d4423e83cf 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -13,7 +13,9 @@ from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, +) from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -222,7 +224,7 @@ class AriaTextMoELayer(nn.Module): bias=config.mlp_bias, ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.moe_num_experts, top_k=config.moe_topk, diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index 642d07ee65..926f4b829e 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -41,7 +41,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -289,7 +289,7 @@ class BailingMoE(nn.Module): else: self.shared_experts = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index 5cc057c3b4..87e1d4dba0 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -18,7 +18,7 @@ from vllm.distributed import ( from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -329,8 +329,8 @@ class BailingMoeV25(nn.Module): else: self.shared_experts = None - # Routed experts using FusedMoE - self.experts = FusedMoE( + # Routed experts using FusedMoEFactory + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, @@ -602,7 +602,7 @@ class BailingMoeV25Model(nn.Module): (".gate_up_proj", ".up_proj", 1), ] - # Expert parameter mappings from FusedMoE + # Expert parameter mappings from FusedMoEFactory expert_mappings = list(self.get_expert_mapping()) def load_param(name: str, tensor: torch.Tensor, shard_id=None) -> bool: diff --git a/vllm/model_executor/models/bert_with_rope.py b/vllm/model_executor/models/bert_with_rope.py index 020ef99347..02ba333ca6 100644 --- a/vllm/model_executor/models/bert_with_rope.py +++ b/vllm/model_executor/models/bert_with_rope.py @@ -320,7 +320,7 @@ class NomicMoE(nn.Module): # router_logits: (num_tokens, n_experts) router_logits, _ = self.router(hidden_states) # FIXME(Isotr0py): This implementation is too tricky, - # we should use FusedMoE instead in the future + # we should use FusedMoEFactory instead in the future # after supporting ungated activation for it. topk_weights, topk_ids, _ = fused_topk( hidden_states, router_logits, self.top_k, renormalize=False diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index b993247ec6..3365ab06fc 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -16,7 +16,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, @@ -300,7 +300,7 @@ class Cohere2Moe(nn.Module): self.shared_experts = None self.shared_expert_combination_strategy = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.num_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, @@ -318,7 +318,7 @@ class Cohere2Moe(nn.Module): orig_shape = hidden_states.shape hidden_states = hidden_states.view(-1, self.hidden_size) router_logits, _ = self.gate(hidden_states) - # FusedMoE handles shared expert overlap internally and returns + # FusedMoEFactory handles shared expert overlap internally and returns # shared_output + routed_output when shared_experts is set. final_hidden_states = self.experts(hidden_states, router_logits) if self.shared_expert_combination_strategy == "average": diff --git a/vllm/model_executor/models/dbrx.py b/vllm/model_executor/models/dbrx.py index c28cf93924..7f1ddc442a 100644 --- a/vllm/model_executor/models/dbrx.py +++ b/vllm/model_executor/models/dbrx.py @@ -16,7 +16,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, RoutedExperts, ) from vllm.model_executor.layers.linear import ( @@ -159,7 +159,7 @@ class DbrxMoE(nn.Module): self.router = DbrxRouter(config, self.params_dtype) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.ffn_config.moe_num_experts, top_k=config.ffn_config.moe_top_k, hidden_size=config.d_model, diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 2d1770e8d7..f076b4fb3f 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -50,7 +50,7 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention, RSWAAttention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -361,7 +361,7 @@ class DeepseekV2MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index fea390ca21..6cc6a056fd 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -41,7 +41,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner +from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -183,7 +183,7 @@ class Ernie4_5_MoeMoE(nn.Module): else: self.shared_experts = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.moe_num_experts, top_k=config.moe_k, diff --git a/vllm/model_executor/models/ernie45_vl_moe.py b/vllm/model_executor/models/ernie45_vl_moe.py index 0bdb567c0a..5d36ca6d37 100644 --- a/vllm/model_executor/models/ernie45_vl_moe.py +++ b/vllm/model_executor/models/ernie45_vl_moe.py @@ -37,7 +37,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -260,7 +260,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): prefix=f"{prefix}.text_experts_gate", ) - self.text_experts = FusedMoE( + self.text_experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.moe_num_experts[0], top_k=config.moe_k, @@ -297,7 +297,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): prefix=f"{prefix}.vision_experts_gate", ) - self.vision_experts = FusedMoE( + self.vision_experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.moe_num_experts[1], top_k=config.moe_k, diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index 086040e2ea..a4d4e9019d 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -29,9 +29,7 @@ from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, ) -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -126,7 +124,7 @@ class ExaoneMoe(nn.Module): else: self.shared_experts = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, num_experts=self.n_routed_experts, diff --git a/vllm/model_executor/models/flex_olmo.py b/vllm/model_executor/models/flex_olmo.py index 2ff9d86056..a0abf9e0fa 100644 --- a/vllm/model_executor/models/flex_olmo.py +++ b/vllm/model_executor/models/flex_olmo.py @@ -21,7 +21,7 @@ from vllm.config import VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear @@ -73,7 +73,7 @@ class FlexOlmoMoE(nn.Module): prefix=f"{prefix}.gate", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=hf_config.num_experts, top_k=hf_config.num_experts_per_tok, hidden_size=hf_config.hidden_size, diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index e0c3130bdd..0fe539886b 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -38,7 +38,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_and_mul_fn from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -305,14 +305,14 @@ class Gemma4Router(nn.Module): class Gemma4MoE(nn.Module): - """Mixture of Experts for Gemma4 using vLLM's FusedMoE. + """Mixture of Experts for Gemma4 using vLLM's MoERunner. - Wraps FusedMoE with custom routing. The router projection is + Wraps MoERunner with custom routing. The router projection is external (Gemma4Router) — this class only handles expert dispatch. Gemma4 routing: softmax over ALL experts → top-k → renormalize. per_expert_scale is folded into routing weights for mathematical - correctness with FusedMoE's fused kernel. + correctness with MoERunner's fused kernel. """ def __init__( @@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module): self.num_experts = config.num_experts # Per-expert output scale folded into routing weights so that - # FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e) + # MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e) self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts)) # Gemma4 routing: softmax over ALL experts → top-k → renormalize. - # FusedMoE's built-in fused_topk scopes softmax differently, so + # MoERunner's built-in fused_topk scopes softmax differently, so # a custom routing function is needed for numerical correctness. # NOTE: self.per_expert_scale is read at call time (not captured into # a local) so that torch.func.functional_call parameter substitution @@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module): gating_output, topk, self.per_expert_scale ) - # FusedMoE experts with custom Gemma4 routing - self.experts = FusedMoE( + # MoERunner experts with custom Gemma4 routing + self.experts = FusedMoEFactory( num_experts=config.num_experts, top_k=config.top_k_experts, hidden_size=config.hidden_size, @@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin): # MoE expert weight mapping: checkpoint can have either: # 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D) # 2. Already per-expert 2D weights (if quantized) - # Map to FusedMoE parameters: - # moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13) - # moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13) - # moe.experts.{id}.down_proj → FusedMoE w2 + # Map to MoERunner parameters: + # moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13) + # moe.experts.{id}.up_proj → MoERunner w3 (shard of w13) + # moe.experts.{id}.down_proj → MoERunner w2 num_experts = getattr(self.config, "num_experts", None) or 0 # Strategy A: dot-separated suffix # (standard AWQ/GPTQ e.g. .qweight, .scales, .weight) @@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin): continue param = params_dict[moe_name] # Expert weights are already in the correct - # orientation for FusedMoE after _weight_iterator: + # orientation for MoERunner after _weight_iterator: # gate/up: [I, H] → w1/w3 expects [I, H] # down: [H, I] → w2 expects [H, I] # Scales and other quantization params may be 1D or scalar. @@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM( # MoE expert weights: checkpoint stores as 3D packed # tensors. Explode into per-expert 2D weights for - # FusedMoE weight_loader. + # MoERunner weight_loader. # # Checkpoint format: # moe.gate_up_proj: [E, 2*I, H] (fused gate + up) # moe.down_proj: [E, H, I] # - # FusedMoE expects per-expert: + # MoERunner expects per-expert: # w1 (gate): [I, H] — first half of gate_up # w3 (up): [I, H] — second half of gate_up # w2 (down): [H, I] — as-is from checkpoint # # No transpose needed: checkpoint orientation already - # matches FusedMoE's expected layout. + # matches MoERunner's expected layout. if "moe.gate_up_proj" in name and weight.dim() == 3: num_experts = weight.size(0) intermediate_size = weight.size(1) // 2 diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index abb9970c40..615df7cef3 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -42,7 +42,9 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -164,7 +166,7 @@ class Glm4MoE(nn.Module): ) # AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py - # pattern (see Glm4MoE / FusedMoE wiring there). + # pattern (see Glm4MoE / MoERunner wiring there). self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled() self.is_fusion_moe_shared_experts_enabled = ( rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() @@ -183,7 +185,7 @@ class Glm4MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index 5fc7e62640..27982eb1f3 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -21,7 +21,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig @@ -211,7 +211,7 @@ class MLPBlock(torch.nn.Module): return_bias=False, ) assert config.intermediate_size % self.world_size == 0 - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, @@ -1010,7 +1010,7 @@ class GptOssModel(nn.Module, EagleModelMixin): tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size) # Use centralized weight remapping for MoE expert parameters. - # The FusedMoE refactor moved expert params under + # The MoERunner refactor moved expert params under # `mlp.experts.routed_experts.*`; this remaps checkpoint names so # MoE weight/bias keys resolve against params_dict. for name, weight in remap_moe_expert_weights(weights, params_dict): diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py index 5909604bd5..328ea9ce63 100644 --- a/vllm/model_executor/models/granitemoe.py +++ b/vllm/model_executor/models/granitemoe.py @@ -40,7 +40,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -101,7 +101,7 @@ class GraniteMoeMoE(nn.Module): prefix=f"{prefix}.gate", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py index d646b13258..af26b39ee0 100644 --- a/vllm/model_executor/models/hunyuan_v1.py +++ b/vllm/model_executor/models/hunyuan_v1.py @@ -43,7 +43,7 @@ from vllm.distributed import ( from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -441,7 +441,7 @@ class HunYuanSparseMoeBlock(nn.Module): else: self.shared_mlp = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_mlp, num_experts=self.n_routed_experts, top_k=top_k, diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py index f61618821d..f75fbb0041 100644 --- a/vllm/model_executor/models/hy_v3.py +++ b/vllm/model_executor/models/hy_v3.py @@ -44,7 +44,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -184,7 +184,7 @@ class HYV3MoEFused(nn.Module): scoring_func = "sigmoid" e_score_correction_bias = self.expert_bias - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=self.n_routed_experts, top_k=top_k, hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/interns1_pro.py b/vllm/model_executor/models/interns1_pro.py index ef7d3d5834..935abb04ed 100644 --- a/vllm/model_executor/models/interns1_pro.py +++ b/vllm/model_executor/models/interns1_pro.py @@ -42,7 +42,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -173,7 +173,7 @@ class InternS1ProMoeSparseMoeBlock(nn.Module): # For custom routing function self.n_groups = getattr(config, "router_n_groups", -1) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=self.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/jamba.py b/vllm/model_executor/models/jamba.py index 33a8c63641..baee41ae1b 100644 --- a/vllm/model_executor/models/jamba.py +++ b/vllm/model_executor/models/jamba.py @@ -14,7 +14,9 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -81,7 +83,7 @@ class JambaMoE(nn.Module): prefix=f"{prefix}.router", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( self.num_total_experts, self.top_k, self.hidden_size, diff --git a/vllm/model_executor/models/laguna.py b/vllm/model_executor/models/laguna.py index e71054f4da..3fcf03b7e6 100644 --- a/vllm/model_executor/models/laguna.py +++ b/vllm/model_executor/models/laguna.py @@ -19,7 +19,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -174,7 +174,7 @@ class LagunaMoE(nn.Module): prefix=f"{prefix}.gate", ) - # Shared expert (optional) - passed to FusedMoE for overlap optimization + # Shared expert (optional) - passed to FusedMoEFactory for overlap optimization self.shared_expert: LagunaMLP | None if config.shared_expert_intermediate_size > 0: self.shared_expert = LagunaMLP( @@ -191,20 +191,22 @@ class LagunaMoE(nn.Module): # Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The # checkpoint stores one [num_experts] tensor per MoE layer at # `mlp.experts.e_score_correction_bias`; registering it as a Parameter - # on the FusedMoE lets the weight loader pick it up and the router + # on the MoERunner lets the weight loader pick it up and the router # add it during top-k selection. The fused top-k bias router requires # float32 regardless of model dtype. e_score_correction_bias = torch.nn.Parameter( torch.zeros(config.num_experts, dtype=torch.float32), requires_grad=False, ) - - # FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the + # MoERunner with SIGMOID routing. Passing `shared_experts=` lets the # layer overlap the shared-expert compute with the all2all dispatch. - # `apply_routed_scale_to_output=True` makes FusedMoE handle the + # `apply_routed_scale_to_output=True` makes MoERunner handle the + # FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the + # layer overlap the shared-expert compute with the all2all dispatch. + # `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the # routed_scaling_factor, shared+routed combine, and TP all-reduce # internally, so forward() just returns the final hidden states. - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_expert, num_experts=config.num_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 9d1f7bd50c..0b41cb0f0f 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -15,7 +15,9 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -143,7 +145,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module): else: self.gate.e_score_correction_bias = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=self.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index 71df54a424..9577c27f54 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -37,7 +37,7 @@ from vllm.model_executor.layers.attention import ( ChunkedLocalAttention, ) from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -131,7 +131,7 @@ class Llama4MoE(nn.Module): self.n_physical_experts = self.n_local_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_expert, num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 18628a64ce..2c6dedae35 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -47,7 +47,7 @@ from vllm.distributed import get_pp_group from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -297,7 +297,7 @@ class LongcatMoe(nn.Module): ) assert config.zero_expert_type is not None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( zero_expert_type=config.zero_expert_type, e_score_correction_bias=self.router.e_score_correction_bias, num_experts=num_experts, @@ -317,7 +317,7 @@ class LongcatMoe(nn.Module): num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) - # Align to FusedMoE padded hidden size to avoid dim mismatch + # Align to MoERunner padded hidden size to avoid dim mismatch padded_hidden = self.experts.moe_config.hidden_dim if hidden_dim < padded_hidden: hidden_states_padded = torch.nn.functional.pad( @@ -333,7 +333,7 @@ class LongcatMoe(nn.Module): hidden_states_padded.to(self.router_params_dtype) ) - # FusedMoE handles routing memoization and zero expert computation + # MoERunner handles routing memoization and zero expert computation # internally. Pass full router_logits (including zero experts) so that # zero experts can be properly identified in routing. final_hidden_states = self.experts( diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index 4c2ebd958b..30ddd5eb12 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -24,7 +24,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -171,7 +171,7 @@ class MiMoV2MoE(nn.Module): torch.empty(config.n_routed_experts, dtype=self.gate_dtype) ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=self.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index 79bd8f439e..b94d6fef0a 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -39,9 +39,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -95,7 +93,7 @@ class MiniMaxM2MoE(nn.Module): else: self.e_score_correction_bias = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, scoring_func=config.scoring_func, diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py index 8305ecc330..991729354b 100644 --- a/vllm/model_executor/models/mixtral.py +++ b/vllm/model_executor/models/mixtral.py @@ -39,9 +39,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -123,7 +121,7 @@ class MixtralMoE(nn.Module): prefix=f"{prefix}.gate", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index 36d554f21f..d68a619eb2 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -33,7 +33,7 @@ from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.activation import ReLUSquaredActivation from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, activation_without_mul, fused_moe_make_expert_params_mapping, @@ -209,7 +209,7 @@ class NemotronHMoE(nn.Module): self.fc1_latent_proj = None self.fc2_latent_proj = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/olmoe.py b/vllm/model_executor/models/olmoe.py index a57ef9cf43..7867caa97c 100644 --- a/vllm/model_executor/models/olmoe.py +++ b/vllm/model_executor/models/olmoe.py @@ -32,9 +32,7 @@ from vllm.distributed import ( from vllm.distributed.utils import split_tensor_along_last_dim from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -94,7 +92,7 @@ class OlmoeMoE(nn.Module): prefix=f"{prefix}.gate", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index fd40033e71..6528b8488c 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -43,7 +43,7 @@ from vllm.model_executor.layers.attention import ( Attention, StaticSinkAttention, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -195,7 +195,7 @@ class OpenPanguMoE(nn.Module): else: self.shared_experts = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/param2moe.py b/vllm/model_executor/models/param2moe.py index 48c0c714ed..7830272c32 100644 --- a/vllm/model_executor/models/param2moe.py +++ b/vllm/model_executor/models/param2moe.py @@ -32,7 +32,9 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -325,7 +327,7 @@ class Param2MoEMoEBlock(nn.Module): else: self.shared_experts = None # type: ignore[assignment] - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, @@ -342,7 +344,7 @@ class Param2MoEMoEBlock(nn.Module): routed_scaling_factor=self.routed_scaling_factor, ) - def maybe_get_fused_moe(self) -> FusedMoE: + def maybe_get_fused_moe(self) -> FusedMoEFactory: return self.experts def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/vllm/model_executor/models/phimoe.py b/vllm/model_executor/models/phimoe.py index dc2af1e8d6..b3b7b995ee 100644 --- a/vllm/model_executor/models/phimoe.py +++ b/vllm/model_executor/models/phimoe.py @@ -35,9 +35,7 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, @@ -273,7 +271,7 @@ class PhiMoE(nn.Module): prefix=f"{prefix}.gate", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index a946159b8f..43274e4ba7 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -40,9 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -165,7 +163,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): else: self.shared_expert = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_expert, num_experts=config.num_experts, top_k=config.num_experts_per_tok, diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index 16a2275cec..2c29f9cc61 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -42,7 +42,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -201,7 +201,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module): self.shared_expert_gate = None self.shared_expert = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_expert, gate=self.gate, num_experts=self.n_routed_experts, @@ -228,13 +228,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module): hidden_states = sequence_parallel_chunk(hidden_states) if self.experts.is_internal_router: - # In this case, the gate/router runs inside the FusedMoE class + # In this case, the gate/router runs inside the MoERunner class final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states ) else: # Actually this will be dead code, since we always pass gate into - # FusedMoE in the current implementation. But we keep this code + # MoERunner in the current implementation. But we keep this code # here for clarity and future flexibility. router_logits, _ = self.gate(hidden_states) final_hidden_states = self.experts( diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 50bc4cce18..b43aea8795 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -20,9 +20,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate from vllm.model_executor.layers.layernorm import ( GemmaRMSNorm as Qwen3NextRMSNorm, @@ -175,7 +173,7 @@ class Qwen3NextSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_expert, gate=self.gate, num_experts=self.n_routed_experts, @@ -208,7 +206,7 @@ class Qwen3NextSparseMoeBlock(nn.Module): hidden_states = sequence_parallel_chunk(hidden_states) if self.experts.is_internal_router: - # In this case, the gate/router runs inside the FusedMoE class + # In this case, the gate/router runs inside the MoERunner class final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states ) diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index b9f9532ae1..2b2fbfe708 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -35,7 +35,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner +from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -334,7 +334,7 @@ class SarvamMLAMoE(nn.Module): else: self.shared_experts = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, diff --git a/vllm/model_executor/models/step3_text.py b/vllm/model_executor/models/step3_text.py index 7fb5a91705..446e66ca8b 100644 --- a/vllm/model_executor/models/step3_text.py +++ b/vllm/model_executor/models/step3_text.py @@ -19,7 +19,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -68,7 +68,7 @@ class FusedMoEBlock(nn.Module): f"the number of experts {config.moe_num_experts}." ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.moe_num_experts, top_k=config.moe_top_k, hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index 1f4fbecce5..0a699ad241 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -24,7 +24,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, MoERunner, fused_moe_make_expert_params_mapping, ) @@ -376,7 +376,7 @@ class FusedMoEBlock(nn.Module): quant_config=quant_config, prefix=f"{prefix}.share_expert", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.share_expert, gate=self.gate, num_experts=config.moe_num_experts, @@ -404,7 +404,7 @@ class FusedMoEBlock(nn.Module): hidden_states=hidden_states, router_logits=hidden_states ) else: - # TODO(bnell): this gate could be moved into the FusedMoE? + # TODO(bnell): this gate could be moved into the MoERunner? router_logits, _ = self.gate(hidden_states) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index ce96403697..e1c3733a0f 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -29,7 +29,11 @@ from vllm.distributed import get_dp_group, get_ep_group from vllm.forward_context import ForwardContext, get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer -from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + MoERunner, + RoutedExperts, +) from vllm.model_executor.models.interfaces import MixtureOfExperts from vllm.model_executor.models.transformers.fuser import get_fuser from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser @@ -53,7 +57,7 @@ class TransformersMoEState: # --8<-- [start:transformers_fused_moe] @PluggableLayer.register("transformers_fused_moe") class TransformersMoERunner(MoERunner): - """Custom FusedMoE for the Transformers modeling backend.""" + """Custom MoERunner for the Transformers modeling backend.""" # --8<-- [end:transformers_fused_moe] def __init__(self, *args, moe_state: TransformersMoEState, **kwargs): @@ -297,7 +301,7 @@ class MoEMixin(MixtureOfExperts): ) if fuser is not None and not reaches_fp16_trick: # MoE block forward is fully replaced. - # gate/router and shared expert (if any) runs in FusedMoE. + # gate/router and shared expert (if any) runs in MoERunner. shared_experts = fuser.shared_experts(moe_block, prefix) # Store shared experts for later down projection adjustment if shared_experts is not None: @@ -331,7 +335,7 @@ class MoEMixin(MixtureOfExperts): if fuser.shared_name: routed += " + shared experts" logger.info_once( - "Fused: %s (%s) -> FusedMoE (internal routing)", + "Fused: %s (%s) -> MoERunner (internal routing)", routed, moe_block_cls, ) @@ -373,10 +377,10 @@ class MoEMixin(MixtureOfExperts): runner_args={"moe_state": moe_state}, ) logger.info_once( - "Fused: experts (%s) -> FusedMoE (external routing)", + "Fused: experts (%s) -> MoERunner (external routing)", experts_cls, ) - fused_experts = FusedMoE(**kwargs) + fused_experts = FusedMoEFactory(**kwargs) moe_block.experts = fused_experts log_replacement(qual_name, experts, fused_experts) # Update MixtureOfExperts mixin state diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index cccc73a28a..44f2121659 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -108,7 +108,7 @@ def _extract_data_from_fused_moe_module( m_: torch.nn.Module, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: """ - Extract weights, weight scales and num_topk from FusedMoE module. + Extract weights, weight scales and num_topk from MoERunner module. """ assert isinstance(m_, MoERunner) m = m_.routed_experts diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index ae9d38eb0a..becfd1d4f7 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -18,7 +18,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -279,7 +279,7 @@ class DeepseekV4MoE(nn.Module): self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, n_shared_experts=( config.n_shared_experts if self.fuse_shared_experts else None @@ -308,7 +308,7 @@ class DeepseekV4MoE(nn.Module): org_shape = hidden_states.shape if self.experts.is_internal_router: - # In this case, the gate/router runs inside the FusedMoE class + # In this case, the gate/router runs inside the MoERunner class final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states, diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index eb94a86173..51479af238 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -51,7 +51,7 @@ from .model import DeepseekV4DecoderLayer logger = init_logger(__name__) # MoE expert scales are fused into per-layer w13/w2 tensors. The exact -# parameter suffix depends on which FusedMoE method handles the experts: +# parameter suffix depends on which MoERunner method handles the experts: # - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``; # - fp8 experts (Fp8MoEMethod with block_quant=True) register # ``w{1,2,3}_weight_scale_inv``. diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index bbda009372..237cebc25d 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -27,7 +27,7 @@ from vllm.model_executor.kernels.mhc.tilelang import ( ) from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.router.base_router import ( @@ -670,7 +670,7 @@ class DeepseekV4MoE(nn.Module): self.physical_expert_start = self.experts_start_idx self.physical_expert_end = self.experts_end_idx - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, @@ -736,7 +736,7 @@ class DeepseekV4MoE(nn.Module): ) -> torch.Tensor: org_shape = hidden_states.shape if self.experts.is_internal_router: - # In this case, the gate/router runs inside the FusedMoE class + # In this case, the gate/router runs inside the MoERunner class final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states, diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index ee036967a7..53951196b3 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -59,7 +59,7 @@ from .model import ( logger = init_logger(__name__) # MoE expert scales are fused into per-layer w13/w2 tensors. The exact -# parameter suffix depends on which FusedMoE method handles the experts: +# parameter suffix depends on which FusedMoEFactory method handles the experts: # - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``; # - fp8 experts (Fp8MoEMethod with block_quant=True) register # ``w{1,2,3}_weight_scale_inv``. diff --git a/vllm/models/deepseek_v4/xpu/model.py b/vllm/models/deepseek_v4/xpu/model.py index 4c6155c842..0c10404d10 100644 --- a/vllm/models/deepseek_v4/xpu/model.py +++ b/vllm/models/deepseek_v4/xpu/model.py @@ -18,7 +18,7 @@ from vllm.distributed import ( from vllm.forward_context import get_forward_context from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -729,7 +729,7 @@ class DeepseekV4MoE(nn.Module): self.n_local_experts = config.n_routed_experts // self.tp_size self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, diff --git a/vllm/models/deepseek_v4/xpu/mtp.py b/vllm/models/deepseek_v4/xpu/mtp.py index 35687ccf16..5c905ac22e 100644 --- a/vllm/models/deepseek_v4/xpu/mtp.py +++ b/vllm/models/deepseek_v4/xpu/mtp.py @@ -54,7 +54,7 @@ from .model import ( logger = init_logger(__name__) # MoE expert scales are fused into per-layer w13/w2 tensors. The exact -# parameter suffix depends on which FusedMoE method handles the experts: +# parameter suffix depends on which FusedMoEFactory method handles the experts: # - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``; # - fp8 experts (Fp8MoEMethod with block_quant=True) register # ``w{1,2,3}_weight_scale_inv``. diff --git a/vllm/models/inkling/amd/model.py b/vllm/models/inkling/amd/model.py index 56ffb1adcd..e37a54938d 100644 --- a/vllm/models/inkling/amd/model.py +++ b/vllm/models/inkling/amd/model.py @@ -646,7 +646,7 @@ def _load_inkling_weights( weight.shard_id = shard_id # MoE expert tensors (fused stacked, routed + shared sink): translate - # the checkpoint layout to per-expert FusedMoE loads. + # the checkpoint layout to per-expert MoERunner loads. moe_match = _MOE_EXPERT_WEIGHT_RE.match(name) if moe_match is not None and moe_match.group("mlp") in moe_modules: moe = moe_modules[moe_match.group("mlp")] diff --git a/vllm/models/inkling/amd/moe.py b/vllm/models/inkling/amd/moe.py index 90b00a3e5a..d8141eb4cf 100644 --- a/vllm/models/inkling/amd/moe.py +++ b/vllm/models/inkling/amd/moe.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction. +"""Inkling mixture-of-experts on vLLM's MoERunner abstraction. Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over the routed experts, log-sigmoid renormalization over the k routed + S shared "sink" logits, scaled by route_scale * global_scale. The routed top-k goes -through vLLM's FusedMoE (which handles TP/EP); the sink experts run in +through vLLM's MoERunner (which handles TP/EP); the sink experts run in :class:`InklingSinkExperts` -- replicated across EP ranks (every token activates every sink) and always bf16 (the checkpoint excludes every ``shared_experts`` from quantization). @@ -35,7 +35,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16 -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform @@ -265,7 +265,7 @@ class InklingGate(nn.Module): def _inkling_moe_ep_size() -> int: - """EP size the FusedMoE layer will run with (mirrors + """EP size the MoERunner layer will run with (mirrors FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when expert parallelism is enabled).""" parallel_config = get_current_vllm_config().parallel_config @@ -444,7 +444,7 @@ class InklingMoE(nn.Module): # The released MXFP4 checkpoint keeps the first routed-expert layer # in bf16 and lists its two expert weights explicitly in Quark's - # exclusion list. FusedMoE asks for a quant method at the module + # exclusion list. FusedMoEFactory asks for a quant method at the module # prefix, so an exact weight exclusion would otherwise be missed and # the bf16 tensors would be loaded into MXFP4 parameters. routed_quant_config = quant_config @@ -458,7 +458,7 @@ class InklingMoE(nn.Module): if routed_weights <= excluded: routed_quant_config = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, @@ -503,7 +503,7 @@ class InklingMoE(nn.Module): topk: int, renormalize: bool, ) -> tuple[torch.Tensor, torch.Tensor]: - """FusedMoE ``custom_routing_function``: the routed top-k slice of the + """MoERunner ``custom_routing_function``: the routed top-k slice of the full (routed + sink) selection. forward() stashes its selection (keyed by logits identity) so the @@ -523,7 +523,7 @@ class InklingMoE(nn.Module): router_logits = self.gate.compute_logits(x) num_tokens = x.shape[0] # One gate select per layer: the routed slice is stashed for the - # routing function inside the FusedMoE op; the sink gammas are the + # routing function inside the MoERunner op; the sink gammas are the # trailing columns. k = self.gate.topk weights, ids = self.gate.select_experts(router_logits) diff --git a/vllm/models/inkling/nvidia/moe.py b/vllm/models/inkling/nvidia/moe.py index 9ff3acaa14..36ffd07b92 100644 --- a/vllm/models/inkling/nvidia/moe.py +++ b/vllm/models/inkling/nvidia/moe.py @@ -34,7 +34,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16 -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from vllm.triton_utils import tl, tldevice, triton @@ -439,7 +439,7 @@ class InklingMoE(nn.Module): # power-of-two EP sizes (n_routed is a power of two). num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size() - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=num_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, diff --git a/vllm/models/kimi_k3/amd/linear.py b/vllm/models/kimi_k3/amd/linear.py index f27c81a24f..4e0088e9e1 100644 --- a/vllm/models/kimi_k3/amd/linear.py +++ b/vllm/models/kimi_k3/amd/linear.py @@ -15,7 +15,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear @@ -260,7 +260,7 @@ class KimiMoE(nn.Module): self.routed_expert_up_proj = None self.routed_output_transform = None - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=num_experts, top_k=num_experts_per_token, diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index ff7d06d3ff..3f3d5b624c 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -20,7 +20,7 @@ from vllm.forward_context import get_forward_context, is_forward_context_availab from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.router.base_router import ( @@ -560,7 +560,7 @@ class KimiMoE(nn.Module): current_platform.is_cuda() and current_platform.is_device_capability_family(100) ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, num_experts=num_experts, top_k=num_experts_per_token, @@ -578,7 +578,7 @@ class KimiMoE(nn.Module): scoring_func=config.moe_router_activation_func, e_score_correction_bias=self.gate.e_score_correction_bias, routed_scaling_factor=self.routed_scaling_factor, - # Down projection runs outside FusedMoE so it can overlap the + # Down projection runs outside MoERunner so it can overlap the # router gate on the aux stream (see forward()); the original # hidden states are passed to forward() as shared_experts_input # so shared experts still see the untransformed input. @@ -689,7 +689,7 @@ class KimiMoE(nn.Module): ) else: # Routed experts consume the down-projected latent; shared experts - # (inside FusedMoE) get the original hidden states via + # (inside MoERunner) get the original hidden states via # shared_experts_input. final_hidden_states = self.experts( hidden_states=routed_hidden_states, diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 324104e055..6960676666 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -41,7 +41,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import ( fused_allreduce_gemma_rms_norm, ) from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -391,7 +391,7 @@ class MiniMaxM3MoE(nn.Module): # always-on shared expert; aiter applies the routed scaling internally. # Every other path (vLLM top-k bias router, or no fusion) applies the # routed scaling to the MoE output here. - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index e29514ab9a..0b7d2704f5 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -31,7 +31,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import ( fused_allreduce_gemma_rms_norm, ) from vllm.model_executor.layers.fused_moe import ( - FusedMoE, + FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) @@ -244,7 +244,7 @@ class MiniMaxM3MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = FusedMoE( + self.experts = FusedMoEFactory( num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size,