[Refactor] Optimize select_experts (#28069)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -118,7 +118,7 @@ class FlashConfig(PretrainedConfig):
|
||||
router_dtype="float32",
|
||||
router_bias=False,
|
||||
topk_method=None,
|
||||
routed_scaling_factor=None,
|
||||
routed_scaling_factor=1.0,
|
||||
zero_expert_num=0,
|
||||
zero_expert_type=None,
|
||||
nextn_use_scmoe=False,
|
||||
|
||||
@@ -625,7 +625,7 @@ class OpenPanguDecoderLayer(nn.Module):
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", None)
|
||||
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.first_k_dense_replace = getattr(
|
||||
config, "first_k_dense_replace", self.num_hidden_layers
|
||||
|
||||
Reference in New Issue
Block a user