[MOE] Standardize and abstract fused shared expert optimization selection (#51695)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -7,13 +7,15 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
self.moe_mlp_layers.append(layer.mlp)
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.model.layers.values(),
|
||||
DeepseekV2MoE,
|
||||
"mtp_block.mlp",
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
@@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if rocm_aiter_moe_shared_expert_enabled
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
)
|
||||
@@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if spec_layer is None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user