[ROCm][Perf] Kimi-K3 Store only the current rank's shards in latent MoE up-proj (#59591)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -36,18 +36,23 @@ class ROCmLatentMoERunner(MoERunner):
|
||||
up_proj = getattr(transform, "up_proj", None)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
self._up_proj_preshard = bool(getattr(transform, "row_sharded", False))
|
||||
self._up_proj_shard_size = 0
|
||||
self._tail_shardable = (
|
||||
up_proj is not None
|
||||
and tp_size > 1
|
||||
and up_proj.weight.shape[0] % tp_size == 0
|
||||
and (self._up_proj_preshard or up_proj.weight.shape[0] % tp_size == 0)
|
||||
and self._shared_experts is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and self.routed_scaling_factor == 1.0
|
||||
)
|
||||
if self._tail_shardable:
|
||||
assert up_proj is not None
|
||||
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
|
||||
self._up_proj_shard_size = (
|
||||
up_proj.weight.shape[0]
|
||||
if self._up_proj_preshard
|
||||
else up_proj.weight.shape[0] // tp_size
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"K3 latent-MoE tail is not shardable under this config, "
|
||||
@@ -80,7 +85,12 @@ class ROCmLatentMoERunner(MoERunner):
|
||||
|
||||
shard_size = self._up_proj_shard_size
|
||||
shard_start = get_tensor_model_parallel_rank() * shard_size
|
||||
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
|
||||
weight = transform.up_proj.weight
|
||||
up_proj_shard = (
|
||||
weight
|
||||
if self._up_proj_preshard
|
||||
else weight.narrow(0, shard_start, shard_size)
|
||||
)
|
||||
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
|
||||
|
||||
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
|
||||
|
||||
@@ -137,11 +137,13 @@ class KimiRoutedOutputTransform(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
norm: RMSNorm | None,
|
||||
up_proj: ReplicatedLinear,
|
||||
up_proj: ReplicatedLinear | ColumnParallelLinear,
|
||||
row_sharded: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm = norm
|
||||
self.up_proj = up_proj
|
||||
self.row_sharded = row_sharded
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if self.norm is not None:
|
||||
@@ -265,7 +267,7 @@ class KimiMoE(nn.Module):
|
||||
|
||||
self.routed_expert_down_proj: ReplicatedLinear | None
|
||||
self.routed_expert_norm: RMSNorm | None
|
||||
self.routed_expert_up_proj: ReplicatedLinear | None
|
||||
self.routed_expert_up_proj: ReplicatedLinear | ColumnParallelLinear | None
|
||||
self.routed_output_transform: KimiRoutedOutputTransform | None
|
||||
if self.use_latent_moe:
|
||||
self.routed_expert_down_proj = ReplicatedLinear(
|
||||
@@ -280,15 +282,36 @@ class KimiMoE(nn.Module):
|
||||
if self.latent_moe_use_norm
|
||||
else None
|
||||
)
|
||||
self.routed_expert_up_proj = ReplicatedLinear(
|
||||
self.moe_hidden_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.routed_expert_up_proj",
|
||||
shard_up_proj = (
|
||||
self.tp_size > 1
|
||||
and hidden_size % self.tp_size == 0
|
||||
and self.num_shared_experts is not None
|
||||
and self.routed_scaling_factor == 1.0
|
||||
)
|
||||
if shard_up_proj:
|
||||
# gather_output is a fallback restoring the full hidden dim for
|
||||
# the unfused tail. The fused tail reads from .weight directly
|
||||
# so it is not triggered
|
||||
self.routed_expert_up_proj = ColumnParallelLinear(
|
||||
self.moe_hidden_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
gather_output=True,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.routed_expert_up_proj",
|
||||
)
|
||||
else:
|
||||
self.routed_expert_up_proj = ReplicatedLinear(
|
||||
self.moe_hidden_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.routed_expert_up_proj",
|
||||
)
|
||||
self.routed_output_transform = KimiRoutedOutputTransform(
|
||||
self.routed_expert_norm, self.routed_expert_up_proj
|
||||
self.routed_expert_norm,
|
||||
self.routed_expert_up_proj,
|
||||
row_sharded=shard_up_proj,
|
||||
)
|
||||
else:
|
||||
self.routed_expert_down_proj = None
|
||||
|
||||
Reference in New Issue
Block a user