From 43d691ec6b1d26d3ef3d8725a7c7e4d8556eb984 Mon Sep 17 00:00:00 2001 From: kliuae <17350011+kliuae@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:11:18 +0800 Subject: [PATCH] [ROCm][Perf] Kimi-K3 Shard Latent MoE up-projection for ROCm path (#51253) Signed-off-by: kliuae --- tests/models/kimi_k3/__init__.py | 2 + .../kimi_k3/test_amd_latent_moe_runner.py | 297 ++++++++++++++++++ vllm/models/kimi_k3/amd/latent_moe_runner.py | 152 +++++++++ vllm/models/kimi_k3/amd/linear.py | 2 + 4 files changed, 453 insertions(+) create mode 100644 tests/models/kimi_k3/__init__.py create mode 100644 tests/models/kimi_k3/test_amd_latent_moe_runner.py create mode 100644 vllm/models/kimi_k3/amd/latent_moe_runner.py diff --git a/tests/models/kimi_k3/__init__.py b/tests/models/kimi_k3/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/models/kimi_k3/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/models/kimi_k3/test_amd_latent_moe_runner.py b/tests/models/kimi_k3/test_amd_latent_moe_runner.py new file mode 100644 index 0000000000..ec270c04fe --- /dev/null +++ b/tests/models/kimi_k3/test_amd_latent_moe_runner.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The ROCm latent-MoE tail must equal the replicated up-projection. + +A wrong shard offset or a dropped accumulation still runs and still produces +plausible text, so these pin the arithmetic rather than the behaviour. +""" + +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.multiprocessing import spawn + +from tests.utils import ( + ensure_current_vllm_config, + init_test_distributed_environment, + multi_gpu_test, +) +from vllm.distributed import get_tp_group +from vllm.model_executor.layers.fused_moe.runner import moe_runner +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner +from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), + reason="ROCmLatentMoERunner is only wired up on ROCm", +) + +HIDDEN_SIZE = 7168 +LATENT_SIZE = 3584 +EPS = 1e-5 +DTYPE = torch.bfloat16 + + +def _build_transform(device: torch.device) -> KimiRoutedOutputTransform: + norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE) + up_proj = ReplicatedLinear( + LATENT_SIZE, + HIDDEN_SIZE, + bias=False, + params_dtype=DTYPE, + prefix="routed_expert_up_proj", + ).to(device=device) + + torch.manual_seed(0) + norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight)) + up_proj.weight.data.copy_(torch.randn_like(up_proj.weight) / LATENT_SIZE**0.5) + return KimiRoutedOutputTransform(norm, up_proj) + + +def _tail_runner( + transform: KimiRoutedOutputTransform, tp_size: int +) -> ROCmLatentMoERunner: + """A runner carrying only what the tail reads, so no engine is needed. + + ``_maybe_reduce_final_output`` is left as the real base-class method, so the + all-reduce that stitches the shards is the real collective. + """ + runner = object.__new__(ROCmLatentMoERunner) + attrs = { + "routed_output_transform": transform, + "_up_proj_shard_size": HIDDEN_SIZE // tp_size, + "_logged_sharded_tail": False, + "moe_config": SimpleNamespace( + tp_size=tp_size, + ep_size=1, + is_sequence_parallel=False, + skip_final_all_reduce=False, + ), + } + for name, value in attrs.items(): + object.__setattr__(runner, name, value) + return runner + + +def _all_reduced(tensor: torch.Tensor, group) -> torch.Tensor: + reduced = tensor.clone() + dist.all_reduce(reduced, group=group) + return reduced + + +def _rank_partials( + num_tokens: int, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor]: + routed = torch.randn(num_tokens, LATENT_SIZE, device=device, dtype=DTYPE) + shared = torch.randn(num_tokens, HIDDEN_SIZE, device=device, dtype=DTYPE) + return routed.mul_(0.01), shared + + +def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None: + transform = _build_transform(device) + runner = _tail_runner(transform, tp_size) + group = get_tp_group().device_group + + for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)): + torch.manual_seed(100 * iteration + rank + 1) + routed_output, shared_output = _rank_partials(num_tokens, device) + + expected = F.linear( + F.rms_norm( + _all_reduced(routed_output, group), + (LATENT_SIZE,), + transform.norm.weight, + EPS, + ), + transform.up_proj.weight, + ) + expected.add_(_all_reduced(shared_output, group)) + + actual = runner._shard_up_proj_tail(routed_output, shared_output, None) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) + + +def _check_writes_only_its_own_shard( + device: torch.device, tp_size: int, rank: int +) -> None: + """Two ranks that swapped both weight rows and write offsets still sum to + the right total, so inspect each slice before the final collective.""" + transform = _build_transform(device) + runner = _tail_runner(transform, tp_size) + group = get_tp_group().device_group + + torch.manual_seed(rank + 1) + routed_output, shared_output = _rank_partials(8, device) + before = shared_output.clone() + latent = transform.norm(_all_reduced(routed_output, group)) + + captured: dict = {} + real_reduce = runner._maybe_reduce_final_output + + def _capture(states, trunc_size, output_is_reduced=None): + captured["states"] = states.clone() + captured["output_is_reduced"] = output_is_reduced + return real_reduce(states, trunc_size, output_is_reduced) + + object.__setattr__(runner, "_maybe_reduce_final_output", _capture) + runner._shard_up_proj_tail(routed_output, shared_output, None) + # Restoring the method breaks the runner -> _capture -> runner cycle, so the + # captured device tensor is freed before the process group is torn down. + object.__setattr__(runner, "_maybe_reduce_final_output", real_reduce) + + assert captured["output_is_reduced"] is False + + shard = HIDDEN_SIZE // tp_size + start, end = rank * shard, (rank + 1) * shard + local = captured["states"] + projected = F.linear(latent, transform.up_proj.weight[start:end]) + + torch.testing.assert_close( + local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2 + ) + torch.testing.assert_close(local[:, :start], before[:, :start], atol=0, rtol=0) + torch.testing.assert_close(local[:, end:], before[:, end:], atol=0, rtol=0) + + +_CHECKS = { + "matches_replicated": _check_matches_replicated, + "own_shard_only": _check_writes_only_its_own_shard, +} + + +def _worker(local_rank: int, world_size: int, port: str, check: str) -> None: + device = torch.device(f"cuda:{local_rank}") + torch.accelerator.set_device_index(device) + with ensure_current_vllm_config(): + init_test_distributed_environment( + world_size, 1, local_rank, port, local_rank=local_rank + ) + _CHECKS[check](device, world_size, local_rank) + + +def _run_ranks(check: str, tp_size: int) -> None: + spawn( + _worker, + args=(tp_size, str(get_open_port()), check), + nprocs=tp_size, + join=True, + ) + + +@multi_gpu_test(num_gpus=4) +def test_sharded_tail_tp4_matches_replicated_projection() -> None: + _run_ranks("matches_replicated", 4) + + +@multi_gpu_test(num_gpus=8) +def test_sharded_tail_tp8_matches_replicated_projection() -> None: + _run_ranks("matches_replicated", 8) + + +@multi_gpu_test(num_gpus=4) +def test_sharded_tail_tp4_writes_only_its_own_shard() -> None: + _run_ranks("own_shard_only", 4) + + +def _runner(**attrs) -> ROCmLatentMoERunner: + runner = object.__new__(ROCmLatentMoERunner) + for name, value in attrs.items(): + object.__setattr__(runner, name, value) + return runner + + +@pytest.mark.parametrize( + "shardable,pre_reduced,expect_sharded", + [ + (True, False, True), + (True, True, False), + (False, False, False), + (False, True, False), + ], +) +def test_forward_shards_only_when_the_tail_is_valid( + shardable: bool, + pre_reduced: bool, + expect_sharded: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The shard needs an un-reduced shared partial to accumulate into.""" + base_output, sharded_output = torch.zeros(1), torch.ones(1) + monkeypatch.setattr(moe_runner.MoERunner, "forward", lambda *a, **k: base_output) + monkeypatch.setattr( + ROCmLatentMoERunner, "_fused_forward", lambda *a, **k: sharded_output + ) + monkeypatch.setattr( + ROCmLatentMoERunner, "_fused_output_is_reduced", property(lambda _: pre_reduced) + ) + runner = _runner(_tail_shardable=shardable) + + result = runner.forward(torch.zeros(1), torch.zeros(1)) + + assert result is (sharded_output if expect_sharded else base_output) + + +@pytest.fixture +def build_runner(monkeypatch: pytest.MonkeyPatch): + """Construct through the real subclass ``__init__``, stubbing only the base.""" + + def _base_init( + self, + *, + tp_size: int = 8, + hidden: int = HIDDEN_SIZE, + is_sequence_parallel: bool = False, + has_up_proj: bool = True, + has_shared_experts: bool = True, + routed_scaling_factor: float = 1.0, + ) -> None: + self.moe_config = SimpleNamespace( + tp_size=tp_size, is_sequence_parallel=is_sequence_parallel + ) + self.routed_output_transform = SimpleNamespace( + norm=None, + up_proj=SimpleNamespace(weight=torch.zeros(hidden, LATENT_SIZE)) + if has_up_proj + else None, + ) + self.routed_scaling_factor = routed_scaling_factor + self._shared_experts = object() if has_shared_experts else None + + monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init) + return ROCmLatentMoERunner + + +def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None: + runner = build_runner() + + assert runner._tail_shardable + assert runner._up_proj_shard_size == HIDDEN_SIZE // 8 + + +@pytest.mark.parametrize( + "override", + [ + pytest.param({"tp_size": 1}, id="no-tp"), + pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"), + pytest.param({"has_up_proj": False}, id="no-up-proj"), + pytest.param({"has_shared_experts": False}, id="no-shared-partial"), + pytest.param({"is_sequence_parallel": True}, id="sequence-parallel"), + pytest.param({"routed_scaling_factor": 2.0}, id="routed-scaling-factor"), + ], +) +def test_falls_back_when_the_config_breaks_the_shard( + build_runner, override: dict +) -> None: + """Each of these makes the sharded tail wrong, not merely slower.""" + runner = build_runner(**override) + + assert not runner._tail_shardable + assert runner._up_proj_shard_size == 0 diff --git a/vllm/models/kimi_k3/amd/latent_moe_runner.py b/vllm/models/kimi_k3/amd/latent_moe_runner.py new file mode 100644 index 0000000000..db1c078b5d --- /dev/null +++ b/vllm/models/kimi_k3/amd/latent_moe_runner.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch + +from vllm.distributed import ( + get_tensor_model_parallel_rank, + tensor_model_parallel_all_reduce, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner, _unpack + +logger = init_logger(__name__) + + +class ROCmLatentMoERunner(MoERunner): + """MoE runner for latent MoE with a replicated routed up-projection. + + Mirrors CUDA's LatentMoERunner, but currently only the up projection + -sharded path is implemented. (Tier 2) + + Native path: the replicated up-proj produces the full hidden dim on every + rank, so the base runner combines routed + shared correctly at any TP size. + """ + + def __init__( + self, + *args, + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + + transform = self.routed_output_transform + up_proj = getattr(transform, "up_proj", None) + tp_size = self.moe_config.tp_size + + self._up_proj_shard_size = 0 + self._tail_shardable = ( + up_proj is not None + and tp_size > 1 + and up_proj.weight.shape[0] % tp_size == 0 + and self._shared_experts is not None + and not self.moe_config.is_sequence_parallel + and self.routed_scaling_factor == 1.0 + ) + if self._tail_shardable: + assert up_proj is not None + self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size + else: + logger.warning_once( + "K3 latent-MoE tail is not shardable under this config, " + "falling back to the replicated up-projection.", + scope="global", + ) + self._logged_sharded_tail = False + + def _shard_up_proj_tail( + self, + fused_output: torch.Tensor, + shared_output: torch.Tensor, + trunc_size: int | None, + ) -> torch.Tensor: + """ + Tier 2: column-parallel up-projection folded into the final reduce. + """ + if not self._logged_sharded_tail: + self._logged_sharded_tail = True + logger.info_once( + "Kimi-K3 latent-MoE tail: up-projecting only this rank's " + "hidden shard into the shared output.", + scope="global", + ) + + transform = self.routed_output_transform + assert transform is not None + + latent = tensor_model_parallel_all_reduce(fused_output) + if transform.norm is not None: + latent = transform.norm(latent) + + shard_size = self._up_proj_shard_size + shard_start = get_tensor_model_parallel_rank() * shard_size + up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size) + hidden_shard = shared_output.narrow(-1, shard_start, shard_size) + + # hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's + # beta-add epilogue so folding in the shared partial costs no kernel. + hidden_shard.addmm_(latent, up_proj_shard.t()) + + return self._maybe_reduce_final_output( + shared_output, trunc_size, output_is_reduced=False + ) + + def forward( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + input_ids: torch.Tensor | None = None, + shared_experts_input: torch.Tensor | None = None, + ) -> torch.Tensor: + if self._tail_shardable and not self._fused_output_is_reduced: + return self._fused_forward( + hidden_states, router_logits, input_ids, shared_experts_input + ) + return super().forward( + hidden_states, router_logits, input_ids, shared_experts_input + ) + + def _fused_forward( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + input_ids: torch.Tensor | None, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor: + # When the caller pre-applies the routed input transform outside the + # runner (e.g. to overlap it on a separate stream), it passes the + # already-transformed routed input as ``hidden_states`` and the original + # hidden states as ``shared_experts_input``; skip the transform then. + if shared_experts_input is None: + hidden_states, shared_experts_input = self.apply_routed_input_transform( + hidden_states + ) + + hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = ( + self._maybe_pad_hidden_states( + shared_experts_input, + hidden_states, + ) + ) + + result = self._forward_entry( + hidden_states, + router_logits, + shared_experts_input, + input_ids, + self._encode_layer_name(), + self.moe_config.hidden_dim_unpadded + if self._quant_method.has_unpadded_output + else 0, + ) + + shared_output, fused_output = _unpack(result) + assert shared_output is not None + + if og_hidden_dim_pre_xform is not None: + fused_output = fused_output[..., :og_hidden_dim_pre_xform] + + result = self._shard_up_proj_tail( + fused_output, shared_output, og_hidden_dim_post_xform + ) + + return self._maybe_add_zero_expert_output(result) diff --git a/vllm/models/kimi_k3/amd/linear.py b/vllm/models/kimi_k3/amd/linear.py index 71c1ff1081..5cb916ef54 100644 --- a/vllm/models/kimi_k3/amd/linear.py +++ b/vllm/models/kimi_k3/amd/linear.py @@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention +from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner from vllm.models.kimi_k3.amd.ops.attn_res import attn_res from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig @@ -289,6 +290,7 @@ class KimiMoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, routed_input_transform=self.routed_expert_down_proj, routed_output_transform=self.routed_output_transform, + runner_cls=ROCmLatentMoERunner if self.use_latent_moe else None, ) if self.padded_moe_intermediate_size != moe_intermediate_size: w13_weight = getattr(self.experts, "w13_weight", None)