[ROCm][Perf] Kimi-K3 Store only the current rank's shards in latent MoE up-proj (#59591)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
kliuae
2026-10-08 14:55:20 +00:00
committed by GitHub
parent d33872dfeb
commit d80ddaac73
3 changed files with 125 additions and 28 deletions
+13 -3
View File
@@ -36,18 +36,23 @@ class ROCmLatentMoERunner(MoERunner):
up_proj = getattr(transform, "up_proj", None)
tp_size = get_tensor_model_parallel_world_size()
self._up_proj_preshard = bool(getattr(transform, "row_sharded", False))
self._up_proj_shard_size = 0
self._tail_shardable = (
up_proj is not None
and tp_size > 1
and up_proj.weight.shape[0] % tp_size == 0
and (self._up_proj_preshard or up_proj.weight.shape[0] % tp_size == 0)
and self._shared_experts is not None
and not self.moe_config.is_sequence_parallel
and self.routed_scaling_factor == 1.0
)
if self._tail_shardable:
assert up_proj is not None
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
self._up_proj_shard_size = (
up_proj.weight.shape[0]
if self._up_proj_preshard
else up_proj.weight.shape[0] // tp_size
)
else:
logger.warning_once(
"K3 latent-MoE tail is not shardable under this config, "
@@ -80,7 +85,12 @@ class ROCmLatentMoERunner(MoERunner):
shard_size = self._up_proj_shard_size
shard_start = get_tensor_model_parallel_rank() * shard_size
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
weight = transform.up_proj.weight
up_proj_shard = (
weight
if self._up_proj_preshard
else weight.narrow(0, shard_start, shard_size)
)
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
+32 -9
View File
@@ -137,11 +137,13 @@ class KimiRoutedOutputTransform(nn.Module):
def __init__(
self,
norm: RMSNorm | None,
up_proj: ReplicatedLinear,
up_proj: ReplicatedLinear | ColumnParallelLinear,
row_sharded: bool = False,
) -> None:
super().__init__()
self.norm = norm
self.up_proj = up_proj
self.row_sharded = row_sharded
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.norm is not None:
@@ -265,7 +267,7 @@ class KimiMoE(nn.Module):
self.routed_expert_down_proj: ReplicatedLinear | None
self.routed_expert_norm: RMSNorm | None
self.routed_expert_up_proj: ReplicatedLinear | None
self.routed_expert_up_proj: ReplicatedLinear | ColumnParallelLinear | None
self.routed_output_transform: KimiRoutedOutputTransform | None
if self.use_latent_moe:
self.routed_expert_down_proj = ReplicatedLinear(
@@ -280,15 +282,36 @@ class KimiMoE(nn.Module):
if self.latent_moe_use_norm
else None
)
self.routed_expert_up_proj = ReplicatedLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
shard_up_proj = (
self.tp_size > 1
and hidden_size % self.tp_size == 0
and self.num_shared_experts is not None
and self.routed_scaling_factor == 1.0
)
if shard_up_proj:
# gather_output is a fallback restoring the full hidden dim for
# the unfused tail. The fused tail reads from .weight directly
# so it is not triggered
self.routed_expert_up_proj = ColumnParallelLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
gather_output=True,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
)
else:
self.routed_expert_up_proj = ReplicatedLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
)
self.routed_output_transform = KimiRoutedOutputTransform(
self.routed_expert_norm, self.routed_expert_up_proj
self.routed_expert_norm,
self.routed_expert_up_proj,
row_sharded=shard_up_proj,
)
else:
self.routed_expert_down_proj = None