[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
+10 -6
View File
@@ -8,12 +8,14 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -167,6 +169,11 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.set_moe_parameters()
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def set_moe_parameters(self):
self.num_moe_layers = self.config.num_nextn_predict_layers
@@ -231,9 +238,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -250,7 +254,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -266,7 +270,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
+3 -43
View File
@@ -7,7 +7,6 @@ import torch
import torch.nn as nn
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_embed_norm import (
@@ -344,9 +343,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -360,12 +356,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
else 0
),
num_experts=self.config.n_routed_experts,
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
@@ -378,9 +369,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
if _try_load_fp8_indexer_wk(
@@ -398,8 +386,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
@@ -415,32 +401,10 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
break
else:
num_chunks = 1
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
total = loaded_weight.shape[split_dim]
assert total % num_chunks == 0
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
is_expert_weight = False
for mapping in expert_params_mapping:
@@ -462,10 +426,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
name = name_mapped
break
else:
if is_expert_weight:
@@ -485,8 +446,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
if not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
loaded_params.add(name)
loaded_layers: set[int] = set()
for param_name in loaded_params:
+44 -51
View File
@@ -16,12 +16,16 @@ from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -37,6 +41,9 @@ from vllm.model_executor.layers.mhc import (
MHCPreOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -60,6 +67,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
logger = init_logger(__name__)
class DeepseekV4MLP(nn.Module):
def __init__(
@@ -154,48 +163,11 @@ class DeepseekV4MLP(nn.Module):
return x
def _shared_experts_are_fp4(config, layer_idx: int | None = None) -> bool:
"""Whether the shared experts are MXFP4 and thus fusable.
``layer_idx=None`` resolves the model-wide default (global scheme), used by
the main-model weight loader / mapper callers that operate per-model.
"""
quant_cfg = getattr(config, "quantization_config", None)
if quant_cfg is None:
return False
if layer_idx is None:
base = None
elif layer_idx >= config.num_hidden_layers:
base = f"mtp.{layer_idx - config.num_hidden_layers}.ffn.shared_experts"
else:
base = f"layers.{layer_idx}.ffn.shared_experts"
if base and any(e.startswith(base) for e in (quant_cfg.get("exclude") or [])):
return False
entry = (
(quant_cfg.get("layer_quant_config") or {}).get(f"{base}.w1") if base else None
)
if entry is None:
entry = quant_cfg.get("global_quant_config")
return ((entry or {}).get("weight") or {}).get("dtype") == "fp4"
def _fuse_shared_experts_enabled(config, prefix: str = "") -> bool:
"""Whether to fuse the shared expert into the routed MXFP4 grouped GEMM.
Fusion fuses the shared expert into the routed experts' MXFP4 grouped GEMM,
so it only applies where the shared expert is the same precision as the
routed experts. Some layers may carry a shared expert in a different quantization
than the routed experts; when so, it runs as its own linear and must not be fused.
"""
if not (
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
def _fuse_shared_experts_enabled(config) -> bool:
return bool(
getattr(config, "n_shared_experts", None)
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
and not get_current_vllm_config().parallel_config.enable_expert_parallel
):
return False
return _shared_experts_are_fp4(
config, extract_layer_index(prefix) if prefix else None
)
@@ -255,9 +227,25 @@ class DeepseekV4MoE(nn.Module):
self.n_shared_experts = config.n_shared_experts
self.fuse_shared_experts = _fuse_shared_experts_enabled(config, prefix)
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
fse_requested = _fuse_shared_experts_enabled(config)
if fse_requested:
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.shared_experts",
)
if not fse_compatible:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
self.is_fused_shared_expert_enabled = fse_requested and fse_compatible
if config.n_shared_experts is None or self.fuse_shared_experts:
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -282,8 +270,9 @@ class DeepseekV4MoE(nn.Module):
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
n_shared_experts=(
config.n_shared_experts if self.fuse_shared_experts else None
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
@@ -581,6 +570,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV4MoE,
"ffn",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
@@ -763,7 +757,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# diverge from how the module was built if per-layer quantization ever
# mixes fused and non-fused layers.
fuse_by_layer = {
extract_layer_index(mod_name): mod.fuse_shared_experts
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
for mod_name, mod in self.named_modules()
if isinstance(mod, DeepseekV4MoE)
}
@@ -874,7 +868,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# redirected shared-expert weights route through the expert loader.
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.n_routed_experts + (
n_shared if _fuse_shared_experts_enabled(self.config) else 0
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
@@ -956,15 +950,14 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
config = vllm_config.model_config.hf_config
self.config = config
expert_dtype = getattr(config, "expert_dtype", "fp4")
fuse_shared_experts = _fuse_shared_experts_enabled(config)
if expert_dtype != "fp4" or fuse_shared_experts:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype, fuse_shared_experts=fuse_shared_experts
)
self.model = self.model_cls(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype,
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
+64 -30
View File
@@ -34,6 +34,7 @@ from vllm.config import (
)
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
@@ -45,6 +46,9 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
MinimaxM3QKVParallelLinearWithIndexer,
@@ -53,6 +57,9 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -108,23 +115,7 @@ from vllm.v1.kv_cache_interface import (
is_quantized_kv_cache,
)
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
"""Whether to fuse the shared expert with routed experts.
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
router-append fusion runs on both aiter and non-aiter MoE);
it is disabled under expert parallelism (the shared slot is appended to
the routed top-k, which the EP expert-mapping path does not handle).
"""
from vllm.platforms import current_platform
return bool(
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
and not get_current_vllm_config().parallel_config.enable_expert_parallel
)
logger = init_logger(__name__)
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
@@ -303,10 +294,12 @@ class MiniMaxM3MLP(nn.Module):
return x
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
def _aiter_moe_fused_shared_experts_enabled(
is_fused_shared_expert_enabled: bool,
) -> bool:
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
A strict sub-case of `is_fused_shared_expert_enabled`: shared-expert
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
and allowed (not under expert parallelism). When additionally on gfx950 with
an active aiter MoE backend, the shared expert is appended inside aiter's
@@ -314,11 +307,13 @@ def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
router.
"""
if not _fuse_shared_experts_enabled(config):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
return (
on_gfx950()
and is_fused_shared_expert_enabled
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
class MiniMaxM3MoE(nn.Module):
@@ -371,11 +366,46 @@ class MiniMaxM3MoE(nn.Module):
# MoE call as the last expert slot, so we don't build a separate module.
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
# It is disabled under expert parallelism (the shared slot is appended to
# the routed top-k, which the EP expert-mapping path does not handle).
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
# is checked to enable FSE for MiniMax-M3, despite AITER not being used.
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
fse_requested = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
self.is_fused_shared_expert_enabled = False
if (
fse_requested
and bool(getattr(config, "n_shared_experts", None))
and not get_current_vllm_config().parallel_config.enable_expert_parallel
):
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.shared_experts",
)
if not fse_compatible:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
else:
self.is_fused_shared_expert_enabled = True
# When additionally on gfx950 with an active aiter MoE backend, the shared
# expert is appended inside aiter's an active aiter MoE backend, the shared
# expert is appended inside aiter's biased grouped top-k kernel
# (``num_fused_shared_experts``) instead of the vLLM router's torch concat.
# Otherwise FSE still runs via the vLLM top-k bias router.
# TODO: `on_gfx950()` check here should not be MiniMax-M3 specific, and
# the check should be done on resolved MOE backend directly.
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(
self.is_fused_shared_expert_enabled
)
self.shared_experts: MiniMaxM3MLP | None = None
if self.n_shared_experts and not self.fuse_shared_experts:
if self.n_shared_experts and not self.is_fused_shared_expert_enabled:
self.shared_experts = MiniMaxM3MLP(
config=config,
intermediate_size=config.intermediate_size * self.n_shared_experts,
@@ -412,8 +442,9 @@ class MiniMaxM3MoE(nn.Module):
router_logits_dtype=self.gate.out_dtype,
shared_experts=self.shared_experts,
n_shared_experts=(
self.n_shared_experts if self.fuse_shared_experts else None
self.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
@@ -1101,6 +1132,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
MiniMaxM3MoE,
"block_sparse_moe",
)
if get_pp_group().is_last_rank:
self.norm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -1155,7 +1191,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# expert, include the appended slot (id == num_local_experts).
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.num_local_experts + (
n_shared if _fuse_shared_experts_enabled(self.config) else 0
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
@@ -1187,8 +1223,6 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = self.get_expert_mapping()
_fuse_shared = _fuse_shared_experts_enabled(self.config)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
@@ -1206,7 +1240,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# down->w2) so it loads via the routed expert loader. Runs before the
# stacked/dense mappings so shared_experts.gate_proj/up_proj are not
# captured by the dense gate_up_proj mapping.
if _fuse_shared and ".shared_experts." in name:
if self.is_fused_shared_expert_enabled and ".shared_experts." in name:
sid = self.config.num_local_experts
name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.")
name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")