[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
+22 -10
View File
@@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -148,10 +152,14 @@ class AXK1MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -189,8 +197,9 @@ class AXK1MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -725,6 +734,12 @@ class AXK1Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
AXK1MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -789,9 +804,6 @@ class AXK1Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -822,7 +834,7 @@ class AXK1Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
(self.config.n_shared_experts or 0)
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -839,7 +851,7 @@ class AXK1Model(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping: