[MOE] Standardize and abstract fused shared expert optimization selection (#51695)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -8,12 +8,14 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -167,6 +169,11 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.set_moe_parameters()
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
@@ -231,9 +238,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -250,7 +254,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -266,7 +270,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
@@ -344,9 +343,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -360,12 +356,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
@@ -378,9 +369,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if _try_load_fp8_indexer_wk(
|
||||
@@ -398,8 +386,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
@@ -415,32 +401,10 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
break
|
||||
else:
|
||||
num_chunks = 1
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
total = loaded_weight.shape[split_dim]
|
||||
assert total % num_chunks == 0
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
@@ -462,10 +426,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
@@ -485,8 +446,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(name)
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
|
||||
@@ -16,12 +16,16 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -37,6 +41,9 @@ from vllm.model_executor.layers.mhc import (
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -60,6 +67,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
@@ -154,48 +163,11 @@ class DeepseekV4MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _shared_experts_are_fp4(config, layer_idx: int | None = None) -> bool:
|
||||
"""Whether the shared experts are MXFP4 and thus fusable.
|
||||
|
||||
``layer_idx=None`` resolves the model-wide default (global scheme), used by
|
||||
the main-model weight loader / mapper callers that operate per-model.
|
||||
"""
|
||||
quant_cfg = getattr(config, "quantization_config", None)
|
||||
if quant_cfg is None:
|
||||
return False
|
||||
if layer_idx is None:
|
||||
base = None
|
||||
elif layer_idx >= config.num_hidden_layers:
|
||||
base = f"mtp.{layer_idx - config.num_hidden_layers}.ffn.shared_experts"
|
||||
else:
|
||||
base = f"layers.{layer_idx}.ffn.shared_experts"
|
||||
if base and any(e.startswith(base) for e in (quant_cfg.get("exclude") or [])):
|
||||
return False
|
||||
entry = (
|
||||
(quant_cfg.get("layer_quant_config") or {}).get(f"{base}.w1") if base else None
|
||||
)
|
||||
if entry is None:
|
||||
entry = quant_cfg.get("global_quant_config")
|
||||
return ((entry or {}).get("weight") or {}).get("dtype") == "fp4"
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config, prefix: str = "") -> bool:
|
||||
"""Whether to fuse the shared expert into the routed MXFP4 grouped GEMM.
|
||||
|
||||
Fusion fuses the shared expert into the routed experts' MXFP4 grouped GEMM,
|
||||
so it only applies where the shared expert is the same precision as the
|
||||
routed experts. Some layers may carry a shared expert in a different quantization
|
||||
than the routed experts; when so, it runs as its own linear and must not be fused.
|
||||
"""
|
||||
if not (
|
||||
current_platform.is_rocm()
|
||||
and getattr(config, "n_shared_experts", None)
|
||||
def _fuse_shared_experts_enabled(config) -> bool:
|
||||
return bool(
|
||||
getattr(config, "n_shared_experts", None)
|
||||
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
):
|
||||
return False
|
||||
return _shared_experts_are_fp4(
|
||||
config, extract_layer_index(prefix) if prefix else None
|
||||
)
|
||||
|
||||
|
||||
@@ -255,9 +227,25 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
|
||||
self.fuse_shared_experts = _fuse_shared_experts_enabled(config, prefix)
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
|
||||
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
|
||||
fse_requested = _fuse_shared_experts_enabled(config)
|
||||
if fse_requested:
|
||||
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.shared_experts",
|
||||
)
|
||||
if not fse_compatible:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
|
||||
if config.n_shared_experts is None or self.fuse_shared_experts:
|
||||
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = None
|
||||
else:
|
||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||
@@ -282,8 +270,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
config.n_shared_experts if self.fuse_shared_experts else None
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -581,6 +570,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV4MoE,
|
||||
"ffn",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
|
||||
@@ -763,7 +757,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# diverge from how the module was built if per-layer quantization ever
|
||||
# mixes fused and non-fused layers.
|
||||
fuse_by_layer = {
|
||||
extract_layer_index(mod_name): mod.fuse_shared_experts
|
||||
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
|
||||
for mod_name, mod in self.named_modules()
|
||||
if isinstance(mod, DeepseekV4MoE)
|
||||
}
|
||||
@@ -874,7 +868,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# redirected shared-expert weights route through the expert loader.
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.n_routed_experts + (
|
||||
n_shared if _fuse_shared_experts_enabled(self.config) else 0
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
@@ -956,15 +950,14 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
expert_dtype = getattr(config, "expert_dtype", "fp4")
|
||||
fuse_shared_experts = _fuse_shared_experts_enabled(config)
|
||||
if expert_dtype != "fp4" or fuse_shared_experts:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype, fuse_shared_experts=fuse_shared_experts
|
||||
)
|
||||
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
|
||||
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
|
||||
expert_dtype,
|
||||
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
@@ -45,6 +46,9 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
MinimaxM3QKVParallelLinearWithIndexer,
|
||||
@@ -53,6 +57,9 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -108,23 +115,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
"""Whether to fuse the shared expert with routed experts.
|
||||
|
||||
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
|
||||
router-append fusion runs on both aiter and non-aiter MoE);
|
||||
it is disabled under expert parallelism (the shared slot is appended to
|
||||
the routed top-k, which the EP expert-mapping path does not handle).
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return bool(
|
||||
current_platform.is_rocm()
|
||||
and getattr(config, "n_shared_experts", None)
|
||||
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
)
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
|
||||
@@ -303,10 +294,12 @@ class MiniMaxM3MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
def _aiter_moe_fused_shared_experts_enabled(
|
||||
is_fused_shared_expert_enabled: bool,
|
||||
) -> bool:
|
||||
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
|
||||
|
||||
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
|
||||
A strict sub-case of `is_fused_shared_expert_enabled`: shared-expert
|
||||
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
|
||||
and allowed (not under expert parallelism). When additionally on gfx950 with
|
||||
an active aiter MoE backend, the shared expert is appended inside aiter's
|
||||
@@ -314,11 +307,13 @@ def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
|
||||
router.
|
||||
"""
|
||||
if not _fuse_shared_experts_enabled(config):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
return (
|
||||
on_gfx950()
|
||||
and is_fused_shared_expert_enabled
|
||||
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
@@ -371,11 +366,46 @@ class MiniMaxM3MoE(nn.Module):
|
||||
# MoE call as the last expert slot, so we don't build a separate module.
|
||||
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
|
||||
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
|
||||
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
|
||||
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
|
||||
# It is disabled under expert parallelism (the shared slot is appended to
|
||||
# the routed top-k, which the EP expert-mapping path does not handle).
|
||||
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
|
||||
# is checked to enable FSE for MiniMax-M3, despite AITER not being used.
|
||||
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
|
||||
fse_requested = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if (
|
||||
fse_requested
|
||||
and bool(getattr(config, "n_shared_experts", None))
|
||||
and not get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
):
|
||||
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
|
||||
quant_config,
|
||||
f"{prefix}.experts",
|
||||
f"{prefix}.shared_experts",
|
||||
)
|
||||
if not fse_compatible:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
else:
|
||||
self.is_fused_shared_expert_enabled = True
|
||||
|
||||
# When additionally on gfx950 with an active aiter MoE backend, the shared
|
||||
# expert is appended inside aiter's an active aiter MoE backend, the shared
|
||||
# expert is appended inside aiter's biased grouped top-k kernel
|
||||
# (``num_fused_shared_experts``) instead of the vLLM router's torch concat.
|
||||
# Otherwise FSE still runs via the vLLM top-k bias router.
|
||||
# TODO: `on_gfx950()` check here should not be MiniMax-M3 specific, and
|
||||
# the check should be done on resolved MOE backend directly.
|
||||
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(
|
||||
self.is_fused_shared_expert_enabled
|
||||
)
|
||||
|
||||
self.shared_experts: MiniMaxM3MLP | None = None
|
||||
if self.n_shared_experts and not self.fuse_shared_experts:
|
||||
if self.n_shared_experts and not self.is_fused_shared_expert_enabled:
|
||||
self.shared_experts = MiniMaxM3MLP(
|
||||
config=config,
|
||||
intermediate_size=config.intermediate_size * self.n_shared_experts,
|
||||
@@ -412,8 +442,9 @@ class MiniMaxM3MoE(nn.Module):
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
self.n_shared_experts if self.fuse_shared_experts else None
|
||||
self.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
@@ -1101,6 +1132,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
MiniMaxM3MoE,
|
||||
"block_sparse_moe",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -1155,7 +1191,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# expert, include the appended slot (id == num_local_experts).
|
||||
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
|
||||
num_experts = self.config.num_local_experts + (
|
||||
n_shared if _fuse_shared_experts_enabled(self.config) else 0
|
||||
n_shared if self.is_fused_shared_expert_enabled else 0
|
||||
)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
@@ -1187,8 +1223,6 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
_fuse_shared = _fuse_shared_experts_enabled(self.config)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
@@ -1206,7 +1240,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# down->w2) so it loads via the routed expert loader. Runs before the
|
||||
# stacked/dense mappings so shared_experts.gate_proj/up_proj are not
|
||||
# captured by the dense gate_up_proj mapping.
|
||||
if _fuse_shared and ".shared_experts." in name:
|
||||
if self.is_fused_shared_expert_enabled and ".shared_experts." in name:
|
||||
sid = self.config.num_local_experts
|
||||
name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.")
|
||||
name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")
|
||||
|
||||
Reference in New Issue
Block a user