[Bugfix] Define router_logits_dtype for remaining MoE models (#33737)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -236,9 +236,9 @@ class FlashMLP(nn.Module):
|
||||
class LongcatRouter(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
zero_expert_num=0,
|
||||
rounter_params_dtype=torch.bfloat16,
|
||||
config: FlashConfig,
|
||||
zero_expert_num: int,
|
||||
rounter_params_dtype: torch.dtype,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -309,6 +309,7 @@ class LongcatMoe(nn.Module):
|
||||
prefix=f"{prefix}.experts",
|
||||
enable_eplb=enable_eplb,
|
||||
routed_scaling_factor=config.routed_scaling_factor,
|
||||
router_logits_dtype=self.rounter_params_dtype,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user