[ROCm][Perf] Kimi-K3 Shard Latent MoE up-projection for ROCm path (#51253)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
kliuae
2026-08-07 06:11:18 +00:00
committed by GitHub
parent b706fd1628
commit 43d691ec6b
4 changed files with 453 additions and 0 deletions
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed import (
get_tensor_model_parallel_rank,
tensor_model_parallel_all_reduce,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner, _unpack
logger = init_logger(__name__)
class ROCmLatentMoERunner(MoERunner):
"""MoE runner for latent MoE with a replicated routed up-projection.
Mirrors CUDA's LatentMoERunner, but currently only the up projection
-sharded path is implemented. (Tier 2)
Native path: the replicated up-proj produces the full hidden dim on every
rank, so the base runner combines routed + shared correctly at any TP size.
"""
def __init__(
self,
*args,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
transform = self.routed_output_transform
up_proj = getattr(transform, "up_proj", None)
tp_size = self.moe_config.tp_size
self._up_proj_shard_size = 0
self._tail_shardable = (
up_proj is not None
and tp_size > 1
and up_proj.weight.shape[0] % tp_size == 0
and self._shared_experts is not None
and not self.moe_config.is_sequence_parallel
and self.routed_scaling_factor == 1.0
)
if self._tail_shardable:
assert up_proj is not None
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
else:
logger.warning_once(
"K3 latent-MoE tail is not shardable under this config, "
"falling back to the replicated up-projection.",
scope="global",
)
self._logged_sharded_tail = False
def _shard_up_proj_tail(
self,
fused_output: torch.Tensor,
shared_output: torch.Tensor,
trunc_size: int | None,
) -> torch.Tensor:
"""
Tier 2: column-parallel up-projection folded into the final reduce.
"""
if not self._logged_sharded_tail:
self._logged_sharded_tail = True
logger.info_once(
"Kimi-K3 latent-MoE tail: up-projecting only this rank's "
"hidden shard into the shared output.",
scope="global",
)
transform = self.routed_output_transform
assert transform is not None
latent = tensor_model_parallel_all_reduce(fused_output)
if transform.norm is not None:
latent = transform.norm(latent)
shard_size = self._up_proj_shard_size
shard_start = get_tensor_model_parallel_rank() * shard_size
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
# beta-add epilogue so folding in the shared partial costs no kernel.
hidden_shard.addmm_(latent, up_proj_shard.t())
return self._maybe_reduce_final_output(
shared_output, trunc_size, output_is_reduced=False
)
def forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
if self._tail_shardable and not self._fused_output_is_reduced:
return self._fused_forward(
hidden_states, router_logits, input_ids, shared_experts_input
)
return super().forward(
hidden_states, router_logits, input_ids, shared_experts_input
)
def _fused_forward(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
# When the caller pre-applies the routed input transform outside the
# runner (e.g. to overlap it on a separate stream), it passes the
# already-transformed routed input as ``hidden_states`` and the original
# hidden states as ``shared_experts_input``; skip the transform then.
if shared_experts_input is None:
hidden_states, shared_experts_input = self.apply_routed_input_transform(
hidden_states
)
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
self._maybe_pad_hidden_states(
shared_experts_input,
hidden_states,
)
)
result = self._forward_entry(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
self._encode_layer_name(),
self.moe_config.hidden_dim_unpadded
if self._quant_method.has_unpadded_output
else 0,
)
shared_output, fused_output = _unpack(result)
assert shared_output is not None
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
result = self._shard_up_proj_tail(
fused_output, shared_output, og_hidden_dim_post_xform
)
return self._maybe_add_zero_expert_output(result)
+2
View File
@@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
@@ -289,6 +290,7 @@ class KimiMoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
routed_input_transform=self.routed_expert_down_proj,
routed_output_transform=self.routed_output_transform,
runner_cls=ROCmLatentMoERunner if self.use_latent_moe else None,
)
if self.padded_moe_intermediate_size != moe_intermediate_size:
w13_weight = getattr(self.experts, "w13_weight", None)