[Refactor] Delete dead code in models (#51838)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -466,10 +466,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
|||||||
def runner(self) -> MoERunner:
|
def runner(self) -> MoERunner:
|
||||||
return self.base_layer
|
return self.base_layer
|
||||||
|
|
||||||
@property
|
|
||||||
def is_internal_router(self) -> bool:
|
|
||||||
return self.base_layer.is_internal_router
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def can_replace_layer(
|
def can_replace_layer(
|
||||||
cls,
|
cls,
|
||||||
|
|||||||
@@ -311,10 +311,6 @@ class MoERunner(MoERunnerInterface):
|
|||||||
def shared_experts(self) -> SharedExperts | None:
|
def shared_experts(self) -> SharedExperts | None:
|
||||||
return self._shared_experts
|
return self._shared_experts
|
||||||
|
|
||||||
@property
|
|
||||||
def is_internal_router(self) -> bool:
|
|
||||||
return self.gate is not None
|
|
||||||
|
|
||||||
# TODO(bnell): Temporary hack. Get rid of this.
|
# TODO(bnell): Temporary hack. Get rid of this.
|
||||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||||
self.routed_experts._replace_quant_method(quant_method)
|
self.routed_experts._replace_quant_method(quant_method)
|
||||||
|
|||||||
@@ -45,11 +45,6 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
|||||||
def shared_experts(self) -> SharedExperts | None:
|
def shared_experts(self) -> SharedExperts | None:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@property
|
|
||||||
@abstractmethod
|
|
||||||
def is_internal_router(self) -> bool:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def _quant_method(self) -> FusedMoEMethodBase:
|
def _quant_method(self) -> FusedMoEMethodBase:
|
||||||
|
|||||||
@@ -108,7 +108,6 @@ class AXK1MoE(nn.Module):
|
|||||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
assert config.n_routed_experts is not None
|
assert config.n_routed_experts is not None
|
||||||
assert config.num_experts_per_tok is not None
|
assert config.num_experts_per_tok is not None
|
||||||
@@ -148,11 +147,6 @@ class AXK1MoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||||
self.is_fusion_moe_shared_experts_enabled = (
|
self.is_fusion_moe_shared_experts_enabled = (
|
||||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||||
@@ -210,15 +204,9 @@ class AXK1MoE(nn.Module):
|
|||||||
if self.is_sequence_parallel:
|
if self.is_sequence_parallel:
|
||||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
final_hidden_states = self.experts(
|
hidden_states=hidden_states, router_logits=hidden_states
|
||||||
hidden_states=hidden_states, router_logits=hidden_states
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states, router_logits=router_logits
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.is_sequence_parallel:
|
if self.is_sequence_parallel:
|
||||||
final_hidden_states = tensor_model_parallel_all_gather(
|
final_hidden_states = tensor_model_parallel_all_gather(
|
||||||
|
|||||||
@@ -71,7 +71,6 @@ class AfmoeMoE(nn.Module):
|
|||||||
self.route_norm = config.route_norm
|
self.route_norm = config.route_norm
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = self.ep_group.rank()
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.num_experts
|
self.n_routed_experts: int = config.num_experts
|
||||||
self.n_shared_experts: int = config.num_shared_experts
|
self.n_shared_experts: int = config.num_shared_experts
|
||||||
@@ -103,11 +102,6 @@ class AfmoeMoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.shared_experts = None
|
self.shared_experts = None
|
||||||
# Shared experts
|
# Shared experts
|
||||||
if config.num_shared_experts > 0:
|
if config.num_shared_experts > 0:
|
||||||
|
|||||||
@@ -293,7 +293,6 @@ class DeepseekV2MoE(nn.Module):
|
|||||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.n_routed_experts
|
self.n_routed_experts: int = config.n_routed_experts
|
||||||
self.n_shared_experts: int = config.n_shared_experts
|
self.n_shared_experts: int = config.n_shared_experts
|
||||||
@@ -329,11 +328,6 @@ class DeepseekV2MoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||||
self.is_fusion_moe_shared_experts_enabled = (
|
self.is_fusion_moe_shared_experts_enabled = (
|
||||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||||
@@ -409,15 +403,9 @@ class DeepseekV2MoE(nn.Module):
|
|||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
final_hidden_states = self.experts(
|
hidden_states=hidden_states, router_logits=hidden_states
|
||||||
hidden_states=hidden_states, router_logits=hidden_states
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states, router_logits=router_logits
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
final_hidden_states = tensor_model_parallel_all_gather(
|
final_hidden_states = tensor_model_parallel_all_gather(
|
||||||
|
|||||||
@@ -129,7 +129,6 @@ class Ernie4_5_MoeMoE(nn.Module):
|
|||||||
|
|
||||||
self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None)
|
self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None)
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.moe_num_experts
|
self.n_routed_experts: int = config.moe_num_experts
|
||||||
self.n_shared_experts: int = self.moe_num_shared_experts
|
self.n_shared_experts: int = self.moe_num_shared_experts
|
||||||
@@ -143,10 +142,6 @@ class Ernie4_5_MoeMoE(nn.Module):
|
|||||||
self.n_logical_experts = self.n_routed_experts
|
self.n_logical_experts = self.n_routed_experts
|
||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
self.has_shared_experts = getattr(config, "moe_num_shared_experts", 0) > 0
|
self.has_shared_experts = getattr(config, "moe_num_shared_experts", 0) > 0
|
||||||
|
|
||||||
if self.tp_size > config.moe_num_experts:
|
if self.tp_size > config.moe_num_experts:
|
||||||
|
|||||||
@@ -76,7 +76,6 @@ class ExaoneMoe(nn.Module):
|
|||||||
self.routed_scaling_factor = config.routed_scaling_factor
|
self.routed_scaling_factor = config.routed_scaling_factor
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = self.ep_group.rank()
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -113,11 +112,6 @@ class ExaoneMoe(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
if getattr(config, "num_shared_experts", 0) > 0:
|
if getattr(config, "num_shared_experts", 0) > 0:
|
||||||
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
|
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
|
||||||
self.shared_experts = ExaoneMoeGatedMLP(
|
self.shared_experts = ExaoneMoeGatedMLP(
|
||||||
|
|||||||
@@ -127,7 +127,6 @@ class Glm4MoE(nn.Module):
|
|||||||
self.routed_scaling_factor = config.routed_scaling_factor
|
self.routed_scaling_factor = config.routed_scaling_factor
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.n_routed_experts
|
self.n_routed_experts: int = config.n_routed_experts
|
||||||
self.n_shared_experts: int = config.n_shared_experts
|
self.n_shared_experts: int = config.n_shared_experts
|
||||||
@@ -160,11 +159,6 @@ class Glm4MoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
|
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
|
||||||
# pattern (see Glm4MoE / MoERunner wiring there).
|
# pattern (see Glm4MoE / MoERunner wiring there).
|
||||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||||
|
|||||||
@@ -372,7 +372,6 @@ class HunYuanSparseMoeBlock(nn.Module):
|
|||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -408,11 +407,6 @@ class HunYuanSparseMoeBlock(nn.Module):
|
|||||||
self.n_redundant_experts = eplb_config.num_redundant_experts
|
self.n_redundant_experts = eplb_config.num_redundant_experts
|
||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
config.num_experts,
|
config.num_experts,
|
||||||
|
|||||||
@@ -134,7 +134,6 @@ class HYV3MoEFused(nn.Module):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
if self.tp_size > config.num_experts:
|
if self.tp_size > config.num_experts:
|
||||||
@@ -153,10 +152,6 @@ class HYV3MoEFused(nn.Module):
|
|||||||
self.n_redundant_experts = eplb_config.num_redundant_experts
|
self.n_redundant_experts = eplb_config.num_redundant_experts
|
||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
self.gate = GateLinear(
|
self.gate = GateLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
config.num_experts,
|
config.num_experts,
|
||||||
|
|||||||
@@ -144,7 +144,6 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
|
|||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -165,11 +164,6 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
# For custom routing function
|
# For custom routing function
|
||||||
self.n_groups = getattr(config, "router_n_groups", -1)
|
self.n_groups = getattr(config, "router_n_groups", -1)
|
||||||
|
|
||||||
|
|||||||
@@ -127,17 +127,10 @@ class InternS2MobiusMetaMoeBlock(nn.Module):
|
|||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
final_hidden_states = self.experts(
|
hidden_states=hidden_states,
|
||||||
hidden_states=hidden_states,
|
router_logits=hidden_states,
|
||||||
router_logits=hidden_states,
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states,
|
|
||||||
router_logits=router_logits,
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
final_hidden_states = tensor_model_parallel_all_gather(
|
final_hidden_states = tensor_model_parallel_all_gather(
|
||||||
|
|||||||
@@ -132,7 +132,6 @@ class LagunaMoE(nn.Module):
|
|||||||
|
|
||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = self.ep_group.rank()
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
|
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
@@ -160,11 +159,6 @@ class LagunaMoE(nn.Module):
|
|||||||
self.n_logical_experts = self.n_routed_experts
|
self.n_logical_experts = self.n_routed_experts
|
||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
# Router gate
|
# Router gate
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
|
|||||||
@@ -106,7 +106,6 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
|
|||||||
self.routed_scaling_factor = config.routed_scaling_factor
|
self.routed_scaling_factor = config.routed_scaling_factor
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -126,11 +125,6 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
config.num_experts,
|
config.num_experts,
|
||||||
|
|||||||
@@ -127,7 +127,6 @@ class MiMoV2MoE(nn.Module):
|
|||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.n_routed_experts
|
self.n_routed_experts = config.n_routed_experts
|
||||||
|
|
||||||
@@ -154,11 +153,6 @@ class MiMoV2MoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
dtype = getattr(config, "moe_router_dtype", "float32")
|
dtype = getattr(config, "moe_router_dtype", "float32")
|
||||||
self.gate_dtype = str_dtype_to_torch_dtype(dtype)
|
self.gate_dtype = str_dtype_to_torch_dtype(dtype)
|
||||||
self.gate = nn.Linear(
|
self.gate = nn.Linear(
|
||||||
|
|||||||
@@ -92,7 +92,6 @@ class MixtralMoE(nn.Module):
|
|||||||
self.hidden_size = hidden_size
|
self.hidden_size = hidden_size
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
|
|
||||||
# Expert Parallelism Load balancing settings.
|
# Expert Parallelism Load balancing settings.
|
||||||
@@ -105,11 +104,6 @@ class MixtralMoE(nn.Module):
|
|||||||
self.n_redundant_experts = parallel_config.eplb_config.num_redundant_experts
|
self.n_redundant_experts = parallel_config.eplb_config.num_redundant_experts
|
||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
# Gate always runs at half / full precision for now.
|
# Gate always runs at half / full precision for now.
|
||||||
|
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
|
|||||||
@@ -136,7 +136,6 @@ class NemotronHMoE(nn.Module):
|
|||||||
self.routed_scaling_factor = config.routed_scaling_factor
|
self.routed_scaling_factor = config.routed_scaling_factor
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = self.ep_group.rank()
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.n_routed_experts
|
self.n_routed_experts: int = config.n_routed_experts
|
||||||
self.n_shared_experts: int = config.n_shared_experts
|
self.n_shared_experts: int = config.n_shared_experts
|
||||||
@@ -166,11 +165,6 @@ class NemotronHMoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
if config.n_shared_experts is None or config.n_shared_experts == 0:
|
if config.n_shared_experts is None or config.n_shared_experts == 0:
|
||||||
self.shared_experts = None
|
self.shared_experts = None
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -142,7 +142,6 @@ class OpenPanguMoE(nn.Module):
|
|||||||
|
|
||||||
self.routed_scaling_factor = config.routed_scaling_factor
|
self.routed_scaling_factor = config.routed_scaling_factor
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = self.ep_group.rank()
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts: int = config.n_routed_experts
|
self.n_routed_experts: int = config.n_routed_experts
|
||||||
self.n_shared_experts: int = config.n_shared_experts
|
self.n_shared_experts: int = config.n_shared_experts
|
||||||
@@ -176,11 +175,6 @@ class OpenPanguMoE(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
if config.n_shared_experts is not None:
|
if config.n_shared_experts is not None:
|
||||||
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
|
||||||
self.shared_experts = OpenPanguMLP(
|
self.shared_experts = OpenPanguMLP(
|
||||||
|
|||||||
@@ -142,7 +142,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
|||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -164,11 +163,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
config.num_experts,
|
config.num_experts,
|
||||||
@@ -227,19 +221,9 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
|||||||
if self.is_sequence_parallel:
|
if self.is_sequence_parallel:
|
||||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
# In this case, the gate/router runs inside the MoERunner class
|
hidden_states=hidden_states, router_logits=hidden_states
|
||||||
final_hidden_states = self.experts(
|
)
|
||||||
hidden_states=hidden_states, router_logits=hidden_states
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Actually this will be dead code, since we always pass gate into
|
|
||||||
# MoERunner in the current implementation. But we keep this code
|
|
||||||
# here for clarity and future flexibility.
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states, router_logits=router_logits
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.is_sequence_parallel:
|
if self.is_sequence_parallel:
|
||||||
final_hidden_states = tensor_model_parallel_all_gather(
|
final_hidden_states = tensor_model_parallel_all_gather(
|
||||||
|
|||||||
@@ -109,7 +109,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
|||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
self.tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
|
||||||
self.ep_group = get_ep_group().device_group
|
self.ep_group = get_ep_group().device_group
|
||||||
self.ep_rank = get_ep_group().rank_in_group
|
|
||||||
self.ep_size = self.ep_group.size()
|
self.ep_size = self.ep_group.size()
|
||||||
self.n_routed_experts = config.num_experts
|
self.n_routed_experts = config.num_experts
|
||||||
|
|
||||||
@@ -130,11 +129,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
self.gate = ReplicatedLinear(
|
self.gate = ReplicatedLinear(
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
config.num_experts,
|
config.num_experts,
|
||||||
@@ -205,17 +199,9 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
|||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
# In this case, the gate/router runs inside the MoERunner class
|
hidden_states=hidden_states, router_logits=hidden_states
|
||||||
final_hidden_states = self.experts(
|
)
|
||||||
hidden_states=hidden_states, router_logits=hidden_states
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# router_logits: (num_tokens, n_experts)
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states, router_logits=router_logits
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||||
final_hidden_states = tensor_model_parallel_all_gather(
|
final_hidden_states = tensor_model_parallel_all_gather(
|
||||||
|
|||||||
@@ -303,7 +303,6 @@ class FusedMoEBlock(nn.Module):
|
|||||||
self.layer_idx = extract_layer_index(prefix)
|
self.layer_idx = extract_layer_index(prefix)
|
||||||
|
|
||||||
self.ep_size = get_ep_group().device_group.size()
|
self.ep_size = get_ep_group().device_group.size()
|
||||||
self.ep_rank = get_ep_group().device_group.rank()
|
|
||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
@@ -316,11 +315,6 @@ class FusedMoEBlock(nn.Module):
|
|||||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||||
|
|
||||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
|
||||||
self.physical_expert_end = (
|
|
||||||
self.physical_expert_start + self.n_local_physical_experts
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.tp_size > config.moe_num_experts:
|
if self.tp_size > config.moe_num_experts:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Tensor parallel size {self.tp_size} is greater than "
|
f"Tensor parallel size {self.tp_size} is greater than "
|
||||||
@@ -399,16 +393,9 @@ class FusedMoEBlock(nn.Module):
|
|||||||
num_tokens, hidden_dim = hidden_states.shape
|
num_tokens, hidden_dim = hidden_states.shape
|
||||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||||
|
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
final_hidden_states = self.experts(
|
hidden_states=hidden_states, router_logits=hidden_states
|
||||||
hidden_states=hidden_states, router_logits=hidden_states
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
# TODO(bnell): this gate could be moved into the MoERunner?
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states, router_logits=router_logits
|
|
||||||
)
|
|
||||||
|
|
||||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||||
|
|
||||||
|
|||||||
@@ -307,20 +307,11 @@ class DeepseekV4MoE(nn.Module):
|
|||||||
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
||||||
|
|
||||||
org_shape = hidden_states.shape
|
org_shape = hidden_states.shape
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
# In this case, the gate/router runs inside the MoERunner class
|
hidden_states=hidden_states,
|
||||||
final_hidden_states = self.experts(
|
router_logits=hidden_states,
|
||||||
hidden_states=hidden_states,
|
input_ids=input_ids,
|
||||||
router_logits=hidden_states,
|
)
|
||||||
input_ids=input_ids,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states,
|
|
||||||
router_logits=router_logits,
|
|
||||||
input_ids=input_ids,
|
|
||||||
)
|
|
||||||
|
|
||||||
return final_hidden_states.view(org_shape)
|
return final_hidden_states.view(org_shape)
|
||||||
|
|
||||||
|
|||||||
@@ -746,20 +746,11 @@ class DeepseekV4MoE(nn.Module):
|
|||||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
org_shape = hidden_states.shape
|
org_shape = hidden_states.shape
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
# In this case, the gate/router runs inside the MoERunner class
|
hidden_states=hidden_states,
|
||||||
final_hidden_states = self.experts(
|
router_logits=hidden_states,
|
||||||
hidden_states=hidden_states,
|
input_ids=input_ids,
|
||||||
router_logits=hidden_states,
|
)
|
||||||
input_ids=input_ids,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states,
|
|
||||||
router_logits=router_logits,
|
|
||||||
input_ids=input_ids,
|
|
||||||
)
|
|
||||||
|
|
||||||
return final_hidden_states.view(org_shape)
|
return final_hidden_states.view(org_shape)
|
||||||
|
|
||||||
|
|||||||
@@ -792,19 +792,11 @@ class DeepseekV4MoE(nn.Module):
|
|||||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
org_shape = hidden_states.shape
|
org_shape = hidden_states.shape
|
||||||
if self.experts.is_internal_router:
|
final_hidden_states = self.experts(
|
||||||
final_hidden_states = self.experts(
|
hidden_states=hidden_states,
|
||||||
hidden_states=hidden_states,
|
router_logits=hidden_states,
|
||||||
router_logits=hidden_states,
|
input_ids=input_ids,
|
||||||
input_ids=input_ids,
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
router_logits, _ = self.gate(hidden_states)
|
|
||||||
final_hidden_states = self.experts(
|
|
||||||
hidden_states=hidden_states,
|
|
||||||
router_logits=router_logits,
|
|
||||||
input_ids=input_ids,
|
|
||||||
)
|
|
||||||
|
|
||||||
return final_hidden_states.view(org_shape)
|
return final_hidden_states.view(org_shape)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user