[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -38,7 +38,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -305,14 +305,14 @@ class Gemma4Router(nn.Module):
|
||||
|
||||
|
||||
class Gemma4MoE(nn.Module):
|
||||
"""Mixture of Experts for Gemma4 using vLLM's FusedMoE.
|
||||
"""Mixture of Experts for Gemma4 using vLLM's MoERunner.
|
||||
|
||||
Wraps FusedMoE with custom routing. The router projection is
|
||||
Wraps MoERunner with custom routing. The router projection is
|
||||
external (Gemma4Router) — this class only handles expert dispatch.
|
||||
|
||||
Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
per_expert_scale is folded into routing weights for mathematical
|
||||
correctness with FusedMoE's fused kernel.
|
||||
correctness with MoERunner's fused kernel.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module):
|
||||
self.num_experts = config.num_experts
|
||||
|
||||
# Per-expert output scale folded into routing weights so that
|
||||
# FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
# MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts))
|
||||
|
||||
# Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
# FusedMoE's built-in fused_topk scopes softmax differently, so
|
||||
# MoERunner's built-in fused_topk scopes softmax differently, so
|
||||
# a custom routing function is needed for numerical correctness.
|
||||
# NOTE: self.per_expert_scale is read at call time (not captured into
|
||||
# a local) so that torch.func.functional_call parameter substitution
|
||||
@@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module):
|
||||
gating_output, topk, self.per_expert_scale
|
||||
)
|
||||
|
||||
# FusedMoE experts with custom Gemma4 routing
|
||||
self.experts = FusedMoE(
|
||||
# MoERunner experts with custom Gemma4 routing
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.top_k_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
# MoE expert weight mapping: checkpoint can have either:
|
||||
# 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D)
|
||||
# 2. Already per-expert 2D weights (if quantized)
|
||||
# Map to FusedMoE parameters:
|
||||
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → FusedMoE w2
|
||||
# Map to MoERunner parameters:
|
||||
# moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → MoERunner w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → MoERunner w2
|
||||
num_experts = getattr(self.config, "num_experts", None) or 0
|
||||
# Strategy A: dot-separated suffix
|
||||
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
|
||||
@@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
param = params_dict[moe_name]
|
||||
# Expert weights are already in the correct
|
||||
# orientation for FusedMoE after _weight_iterator:
|
||||
# orientation for MoERunner after _weight_iterator:
|
||||
# gate/up: [I, H] → w1/w3 expects [I, H]
|
||||
# down: [H, I] → w2 expects [H, I]
|
||||
# Scales and other quantization params may be 1D or scalar.
|
||||
@@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM(
|
||||
|
||||
# MoE expert weights: checkpoint stores as 3D packed
|
||||
# tensors. Explode into per-expert 2D weights for
|
||||
# FusedMoE weight_loader.
|
||||
# MoERunner weight_loader.
|
||||
#
|
||||
# Checkpoint format:
|
||||
# moe.gate_up_proj: [E, 2*I, H] (fused gate + up)
|
||||
# moe.down_proj: [E, H, I]
|
||||
#
|
||||
# FusedMoE expects per-expert:
|
||||
# MoERunner expects per-expert:
|
||||
# w1 (gate): [I, H] — first half of gate_up
|
||||
# w3 (up): [I, H] — second half of gate_up
|
||||
# w2 (down): [H, I] — as-is from checkpoint
|
||||
#
|
||||
# No transpose needed: checkpoint orientation already
|
||||
# matches FusedMoE's expected layout.
|
||||
# matches MoERunner's expected layout.
|
||||
if "moe.gate_up_proj" in name and weight.dim() == 3:
|
||||
num_experts = weight.size(0)
|
||||
intermediate_size = weight.size(1) // 2
|
||||
|
||||
Reference in New Issue
Block a user