[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -279,7 +279,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.fuse_shared_experts else None
|
||||
@@ -308,7 +308,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
|
||||
@@ -51,7 +51,7 @@ from .model import DeepseekV4DecoderLayer
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which MoERunner method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
@@ -670,7 +670,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.physical_expert_start = self.experts_start_idx
|
||||
self.physical_expert_end = self.experts_end_idx
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
@@ -736,7 +736,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
org_shape = hidden_states.shape
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
|
||||
@@ -59,7 +59,7 @@ from .model import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -729,7 +729,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.n_local_experts = config.n_routed_experts // self.tp_size
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -54,7 +54,7 @@ from .model import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors. The exact
|
||||
# parameter suffix depends on which FusedMoE method handles the experts:
|
||||
# parameter suffix depends on which FusedMoEFactory method handles the experts:
|
||||
# - fp4 experts (Mxfp4MoEMethod) register ``w{1,2,3}_weight_scale``;
|
||||
# - fp8 experts (Fp8MoEMethod with block_quant=True) register
|
||||
# ``w{1,2,3}_weight_scale_inv``.
|
||||
|
||||
@@ -646,7 +646,7 @@ def _load_inkling_weights(
|
||||
weight.shard_id = shard_id
|
||||
|
||||
# MoE expert tensors (fused stacked, routed + shared sink): translate
|
||||
# the checkpoint layout to per-expert FusedMoE loads.
|
||||
# the checkpoint layout to per-expert MoERunner loads.
|
||||
moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
|
||||
if moe_match is not None and moe_match.group("mlp") in moe_modules:
|
||||
moe = moe_modules[moe_match.group("mlp")]
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
|
||||
"""Inkling mixture-of-experts on vLLM's MoERunner abstraction.
|
||||
|
||||
Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
|
||||
the routed experts, log-sigmoid renormalization over the k routed + S shared
|
||||
"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
|
||||
through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
|
||||
through vLLM's MoERunner (which handles TP/EP); the sink experts run in
|
||||
:class:`InklingSinkExperts` -- replicated across EP ranks (every token
|
||||
activates every sink) and always bf16 (the checkpoint excludes every
|
||||
``shared_experts`` from quantization).
|
||||
@@ -35,7 +35,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -265,7 +265,7 @@ class InklingGate(nn.Module):
|
||||
|
||||
|
||||
def _inkling_moe_ep_size() -> int:
|
||||
"""EP size the FusedMoE layer will run with (mirrors
|
||||
"""EP size the MoERunner layer will run with (mirrors
|
||||
FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
|
||||
expert parallelism is enabled)."""
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
@@ -444,7 +444,7 @@ class InklingMoE(nn.Module):
|
||||
|
||||
# The released MXFP4 checkpoint keeps the first routed-expert layer
|
||||
# in bf16 and lists its two expert weights explicitly in Quark's
|
||||
# exclusion list. FusedMoE asks for a quant method at the module
|
||||
# exclusion list. FusedMoEFactory asks for a quant method at the module
|
||||
# prefix, so an exact weight exclusion would otherwise be missed and
|
||||
# the bf16 tensors would be loaded into MXFP4 parameters.
|
||||
routed_quant_config = quant_config
|
||||
@@ -458,7 +458,7 @@ class InklingMoE(nn.Module):
|
||||
if routed_weights <= excluded:
|
||||
routed_quant_config = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -503,7 +503,7 @@ class InklingMoE(nn.Module):
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""FusedMoE ``custom_routing_function``: the routed top-k slice of the
|
||||
"""MoERunner ``custom_routing_function``: the routed top-k slice of the
|
||||
full (routed + sink) selection.
|
||||
|
||||
forward() stashes its selection (keyed by logits identity) so the
|
||||
@@ -523,7 +523,7 @@ class InklingMoE(nn.Module):
|
||||
router_logits = self.gate.compute_logits(x)
|
||||
num_tokens = x.shape[0]
|
||||
# One gate select per layer: the routed slice is stashed for the
|
||||
# routing function inside the FusedMoE op; the sink gammas are the
|
||||
# routing function inside the MoERunner op; the sink gammas are the
|
||||
# trailing columns.
|
||||
k = self.gate.topk
|
||||
weights, ids = self.gate.select_experts(router_logits)
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
@@ -439,7 +439,7 @@ class InklingMoE(nn.Module):
|
||||
# power-of-two EP sizes (n_routed is a power of two).
|
||||
num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -15,7 +15,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
@@ -260,7 +260,7 @@ class KimiMoE(nn.Module):
|
||||
self.routed_expert_up_proj = None
|
||||
self.routed_output_transform = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=num_experts_per_token,
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.forward_context import get_forward_context, is_forward_context_availab
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
@@ -560,7 +560,7 @@ class KimiMoE(nn.Module):
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=num_experts,
|
||||
top_k=num_experts_per_token,
|
||||
@@ -578,7 +578,7 @@ class KimiMoE(nn.Module):
|
||||
scoring_func=config.moe_router_activation_func,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
# Down projection runs outside FusedMoE so it can overlap the
|
||||
# Down projection runs outside MoERunner so it can overlap the
|
||||
# router gate on the aux stream (see forward()); the original
|
||||
# hidden states are passed to forward() as shared_experts_input
|
||||
# so shared experts still see the untransformed input.
|
||||
@@ -689,7 +689,7 @@ class KimiMoE(nn.Module):
|
||||
)
|
||||
else:
|
||||
# Routed experts consume the down-projected latent; shared experts
|
||||
# (inside FusedMoE) get the original hidden states via
|
||||
# (inside MoERunner) get the original hidden states via
|
||||
# shared_experts_input.
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=routed_hidden_states,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -391,7 +391,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
# always-on shared expert; aiter applies the routed scaling internally.
|
||||
# Every other path (vLLM top-k bias router, or no fusion) applies the
|
||||
# routed scaling to the MoE output here.
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -31,7 +31,7 @@ from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -244,7 +244,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
Reference in New Issue
Block a user