[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
+12 -9
View File
@@ -16,6 +16,7 @@ from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -216,7 +217,7 @@ class AriaProjector(nn.Module):
return out
class AriaFusedMoE(FusedMoE):
class AriaRoutedExperts(RoutedExperts):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
) -> None:
@@ -225,13 +226,14 @@ class AriaFusedMoE(FusedMoE):
# up weights for each expert.
# Note: Loading expert weights with quantization is not supported
tp_rank = get_tensor_model_parallel_rank()
tp_size = self.moe_config.tp_size
if shard_id == "w13":
# the shape of loaded_weight is
# (num_experts, hidden_size, 2 * moe_intermediate_size)
if self.tp_size > 1:
if tp_size > 1:
up, gate = loaded_weight.chunk(2, dim=-1)
up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
up_and_gate = torch.cat(
[up_current_rank, gate_current_rank], dim=-1
).transpose(1, 2)
@@ -241,8 +243,8 @@ class AriaFusedMoE(FusedMoE):
elif shard_id == "w2":
# the shape of loaded_weight is
# (num_experts, moe_intermediate_size, hidden_size)
if self.tp_size > 1:
down_current_rank = loaded_weight.chunk(self.tp_size, dim=1)[tp_rank]
if tp_size > 1:
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
param.data.copy_(down_current_rank.transpose(1, 2))
else:
param.data.copy_(loaded_weight.transpose(1, 2))
@@ -278,7 +280,7 @@ class AriaTextMoELayer(nn.Module):
bias=config.mlp_bias,
)
self.experts = AriaFusedMoE(
self.experts = FusedMoE(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_topk,
@@ -286,6 +288,7 @@ class AriaTextMoELayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.experts",
routed_experts_cls=AriaRoutedExperts,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -351,8 +354,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
("experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.w2_weight", "experts.fc2.weight", "w2"),
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()