Add MoNe experts pruning support (#50415)
Signed-off-by: Ilya Boytsov <ieboytsov@nebius.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Ilya Boytsov <ieboytsov@nebius.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Ilya Boytsov
Codex
Misha Goin
Claude Opus 5.5
parent
1484266938
commit
189bbfb9f4
@@ -142,6 +142,7 @@ def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
require_decomposed_backend=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
|
||||
Reference in New Issue
Block a user