Enable B12x backend for non-gated MoEs (like Nemotron) (#43328)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
b136cc2c2c
commit
b1384f5ec6
@@ -55,6 +55,7 @@ def make_dummy_moe_config(
|
||||
intermediate_size: int = 1,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
max_num_tokens: int = 512,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> FusedMoEConfig:
|
||||
"""
|
||||
This is a dummy config for the mk constructor interface
|
||||
@@ -73,7 +74,7 @@ def make_dummy_moe_config(
|
||||
else num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
activation=activation,
|
||||
in_dtype=in_dtype,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
|
||||
Reference in New Issue
Block a user