Shard the K3 Latent-MoE up-projection on large batches (#50383)

This commit is contained in:
Jee Jee Li
2026-08-03 00:49:39 -07:00
committed by GitHub
parent 0a6446005d
commit 4635cc3e8f
2 changed files with 176 additions and 72 deletions
+2 -14
View File
@@ -472,8 +472,8 @@ class KimiMoE(nn.Module):
if self.num_shared_experts is not None:
shared_intermediate_size = moe_intermediate_size * self.num_shared_experts
self.shared_experts = KimiMLP(
hidden_size=config.hidden_size,
intermediate_size=shared_intermediate_size,
hidden_size=config.hidden_size, # 7618
intermediate_size=shared_intermediate_size, # 3072*2
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
@@ -553,13 +553,6 @@ class KimiMoE(nn.Module):
activation_linear_beta=activation_situ_linear_beta,
)
else:
# The tail-fusion kernels are tcgen05-based, so they require an
# SM100 NVIDIA device; the runner falls back to the default latent
# MoE path everywhere else.
enable_tail_fusion = (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
)
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
num_experts=num_experts,
@@ -586,11 +579,6 @@ class KimiMoE(nn.Module):
routed_output_transform=self.routed_output_transform,
is_sequence_parallel=use_sequence_parallel,
runner_cls=LatentMoERunner if self.use_latent_moe else None,
runner_args=(
{"enable_k3_latent_moe_tail_fusion": enable_tail_fusion}
if self.use_latent_moe
else None
),
)
if self.padded_moe_intermediate_size != moe_intermediate_size:
w13_weight = getattr(self.experts, "w13_weight", None)