[ROCm][Perf] Kimi-K3 Store only the current rank's shards in latent MoE up-proj (#59591)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
kliuae
2026-10-08 14:55:20 +00:00
committed by GitHub
parent d33872dfeb
commit d80ddaac73
3 changed files with 125 additions and 28 deletions
@@ -22,7 +22,7 @@ from tests.utils import (
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.runner import moe_runner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear
from vllm.models.kimi_k3.amd import latent_moe_runner
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
@@ -40,20 +40,42 @@ EPS = 1e-5
DTYPE = torch.bfloat16
def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
def _build_transform(
device: torch.device, *, row_sharded: bool = False
) -> tuple[KimiRoutedOutputTransform, torch.Tensor]:
norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE)
up_proj = ReplicatedLinear(
LATENT_SIZE,
HIDDEN_SIZE,
bias=False,
params_dtype=DTYPE,
prefix="routed_expert_up_proj",
).to(device=device)
torch.manual_seed(0)
norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight))
up_proj.weight.data.copy_(torch.randn_like(up_proj.weight) / LATENT_SIZE**0.5)
return KimiRoutedOutputTransform(norm, up_proj)
full_weight = (
torch.randn(HIDDEN_SIZE, LATENT_SIZE, device=device, dtype=DTYPE)
/ LATENT_SIZE**0.5
)
up_proj: ReplicatedLinear | ColumnParallelLinear
if row_sharded:
up_proj = ColumnParallelLinear(
LATENT_SIZE,
HIDDEN_SIZE,
bias=False,
gather_output=True,
params_dtype=DTYPE,
prefix="routed_expert_up_proj",
).to(device=device)
up_proj.weight_loader(up_proj.weight, full_weight)
else:
up_proj = ReplicatedLinear(
LATENT_SIZE,
HIDDEN_SIZE,
bias=False,
params_dtype=DTYPE,
prefix="routed_expert_up_proj",
).to(device=device)
up_proj.weight.data.copy_(full_weight)
return (
KimiRoutedOutputTransform(norm, up_proj, row_sharded=row_sharded),
full_weight,
)
def _tail_runner(
@@ -72,6 +94,7 @@ def _tail_runner(
runner = object.__new__(ROCmLatentMoERunner)
attrs = {
"routed_output_transform": transform,
"_up_proj_preshard": bool(getattr(transform, "row_sharded", False)),
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
"_logged_sharded_tail": False,
"moe_config": SimpleNamespace(
@@ -101,9 +124,13 @@ def _rank_partials(
def _check_matches_replicated(
device: torch.device, tp_world: int, rank: int, use_ep: bool = False
device: torch.device,
tp_world: int,
rank: int,
use_ep: bool = False,
row_sharded: bool = False,
) -> None:
transform = _build_transform(device)
transform, up_weight = _build_transform(device, row_sharded=row_sharded)
runner = _tail_runner(transform, tp_world, use_ep)
group = get_tp_group().device_group
@@ -118,7 +145,7 @@ def _check_matches_replicated(
transform.norm.weight,
EPS,
),
transform.up_proj.weight,
up_weight,
)
expected.add_(_all_reduced(shared_output, group))
@@ -136,12 +163,37 @@ def _check_matches_replicated_under_ep(
_check_matches_replicated(device, tp_world, rank, use_ep=True)
def _check_presharded_matches_replicated(
device: torch.device, tp_world: int, rank: int
) -> None:
_check_matches_replicated(device, tp_world, rank, row_sharded=True)
def _check_unfused_tail_gathers_full_hidden(
device: torch.device, tp_world: int, rank: int
) -> None:
transform, up_weight = _build_transform(device, row_sharded=True)
assert transform.up_proj.weight.shape[0] == HIDDEN_SIZE // tp_world
# all-gather reconstructs the full projection when all ranks feed the same
# input, so use one seed for all ranks.
torch.manual_seed(1234)
latent = torch.randn(4, LATENT_SIZE, device=device, dtype=DTYPE).mul_(0.01)
torch.testing.assert_close(
transform(latent),
F.linear(transform.norm(latent), up_weight),
atol=8e-2,
rtol=3e-2,
)
def _check_writes_only_its_own_shard(
device: torch.device, tp_world: int, rank: int
) -> None:
"""Two ranks that swapped both weight rows and write offsets still sum to
the right total, so inspect each slice before the final collective."""
transform = _build_transform(device)
transform, up_weight = _build_transform(device)
runner = _tail_runner(transform, tp_world)
group = get_tp_group().device_group
@@ -169,7 +221,7 @@ def _check_writes_only_its_own_shard(
shard = HIDDEN_SIZE // tp_world
start, end = rank * shard, (rank + 1) * shard
local = captured["states"]
projected = F.linear(latent, transform.up_proj.weight[start:end])
projected = F.linear(latent, up_weight[start:end])
torch.testing.assert_close(
local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2
@@ -181,6 +233,8 @@ def _check_writes_only_its_own_shard(
_CHECKS = {
"matches_replicated": _check_matches_replicated,
"matches_replicated_ep": _check_matches_replicated_under_ep,
"presharded_matches_replicated": _check_presharded_matches_replicated,
"unfused_gathers": _check_unfused_tail_gathers_full_hidden,
"own_shard_only": _check_writes_only_its_own_shard,
}
@@ -219,6 +273,16 @@ def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated_ep", 8)
@multi_gpu_test(num_gpus=4)
def test_presharded_tail_tp4_matches_replicated_projection() -> None:
_run_ranks("presharded_matches_replicated", 4)
@multi_gpu_test(num_gpus=4)
def test_presharded_unfused_tail_gathers_the_full_hidden_dim() -> None:
_run_ranks("unfused_gathers", 4)
@multi_gpu_test(num_gpus=4)
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
_run_ranks("own_shard_only", 4)
+13 -3
View File
@@ -36,18 +36,23 @@ class ROCmLatentMoERunner(MoERunner):
up_proj = getattr(transform, "up_proj", None)
tp_size = get_tensor_model_parallel_world_size()
self._up_proj_preshard = bool(getattr(transform, "row_sharded", False))
self._up_proj_shard_size = 0
self._tail_shardable = (
up_proj is not None
and tp_size > 1
and up_proj.weight.shape[0] % tp_size == 0
and (self._up_proj_preshard or up_proj.weight.shape[0] % tp_size == 0)
and self._shared_experts is not None
and not self.moe_config.is_sequence_parallel
and self.routed_scaling_factor == 1.0
)
if self._tail_shardable:
assert up_proj is not None
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
self._up_proj_shard_size = (
up_proj.weight.shape[0]
if self._up_proj_preshard
else up_proj.weight.shape[0] // tp_size
)
else:
logger.warning_once(
"K3 latent-MoE tail is not shardable under this config, "
@@ -80,7 +85,12 @@ class ROCmLatentMoERunner(MoERunner):
shard_size = self._up_proj_shard_size
shard_start = get_tensor_model_parallel_rank() * shard_size
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
weight = transform.up_proj.weight
up_proj_shard = (
weight
if self._up_proj_preshard
else weight.narrow(0, shard_start, shard_size)
)
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
+32 -9
View File
@@ -137,11 +137,13 @@ class KimiRoutedOutputTransform(nn.Module):
def __init__(
self,
norm: RMSNorm | None,
up_proj: ReplicatedLinear,
up_proj: ReplicatedLinear | ColumnParallelLinear,
row_sharded: bool = False,
) -> None:
super().__init__()
self.norm = norm
self.up_proj = up_proj
self.row_sharded = row_sharded
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.norm is not None:
@@ -265,7 +267,7 @@ class KimiMoE(nn.Module):
self.routed_expert_down_proj: ReplicatedLinear | None
self.routed_expert_norm: RMSNorm | None
self.routed_expert_up_proj: ReplicatedLinear | None
self.routed_expert_up_proj: ReplicatedLinear | ColumnParallelLinear | None
self.routed_output_transform: KimiRoutedOutputTransform | None
if self.use_latent_moe:
self.routed_expert_down_proj = ReplicatedLinear(
@@ -280,15 +282,36 @@ class KimiMoE(nn.Module):
if self.latent_moe_use_norm
else None
)
self.routed_expert_up_proj = ReplicatedLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
shard_up_proj = (
self.tp_size > 1
and hidden_size % self.tp_size == 0
and self.num_shared_experts is not None
and self.routed_scaling_factor == 1.0
)
if shard_up_proj:
# gather_output is a fallback restoring the full hidden dim for
# the unfused tail. The fused tail reads from .weight directly
# so it is not triggered
self.routed_expert_up_proj = ColumnParallelLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
gather_output=True,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
)
else:
self.routed_expert_up_proj = ReplicatedLinear(
self.moe_hidden_size,
hidden_size,
bias=False,
quant_config=None,
prefix=f"{prefix}.routed_expert_up_proj",
)
self.routed_output_transform = KimiRoutedOutputTransform(
self.routed_expert_norm, self.routed_expert_up_proj
self.routed_expert_norm,
self.routed_expert_up_proj,
row_sharded=shard_up_proj,
)
else:
self.routed_expert_down_proj = None