[ROCm][Perf] Kimi-K3 Shard Latent MoE up-projection for ROCm path (#51253)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner, _unpack
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ROCmLatentMoERunner(MoERunner):
|
||||
"""MoE runner for latent MoE with a replicated routed up-projection.
|
||||
|
||||
Mirrors CUDA's LatentMoERunner, but currently only the up projection
|
||||
-sharded path is implemented. (Tier 2)
|
||||
|
||||
Native path: the replicated up-proj produces the full hidden dim on every
|
||||
rank, so the base runner combines routed + shared correctly at any TP size.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
transform = self.routed_output_transform
|
||||
up_proj = getattr(transform, "up_proj", None)
|
||||
tp_size = self.moe_config.tp_size
|
||||
|
||||
self._up_proj_shard_size = 0
|
||||
self._tail_shardable = (
|
||||
up_proj is not None
|
||||
and tp_size > 1
|
||||
and up_proj.weight.shape[0] % tp_size == 0
|
||||
and self._shared_experts is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and self.routed_scaling_factor == 1.0
|
||||
)
|
||||
if self._tail_shardable:
|
||||
assert up_proj is not None
|
||||
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
|
||||
else:
|
||||
logger.warning_once(
|
||||
"K3 latent-MoE tail is not shardable under this config, "
|
||||
"falling back to the replicated up-projection.",
|
||||
scope="global",
|
||||
)
|
||||
self._logged_sharded_tail = False
|
||||
|
||||
def _shard_up_proj_tail(
|
||||
self,
|
||||
fused_output: torch.Tensor,
|
||||
shared_output: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Tier 2: column-parallel up-projection folded into the final reduce.
|
||||
"""
|
||||
if not self._logged_sharded_tail:
|
||||
self._logged_sharded_tail = True
|
||||
logger.info_once(
|
||||
"Kimi-K3 latent-MoE tail: up-projecting only this rank's "
|
||||
"hidden shard into the shared output.",
|
||||
scope="global",
|
||||
)
|
||||
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
|
||||
latent = tensor_model_parallel_all_reduce(fused_output)
|
||||
if transform.norm is not None:
|
||||
latent = transform.norm(latent)
|
||||
|
||||
shard_size = self._up_proj_shard_size
|
||||
shard_start = get_tensor_model_parallel_rank() * shard_size
|
||||
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
|
||||
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
|
||||
|
||||
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
|
||||
# beta-add epilogue so folding in the shared partial costs no kernel.
|
||||
hidden_shard.addmm_(latent, up_proj_shard.t())
|
||||
|
||||
return self._maybe_reduce_final_output(
|
||||
shared_output, trunc_size, output_is_reduced=False
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self._tail_shardable and not self._fused_output_is_reduced:
|
||||
return self._fused_forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
return super().forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
|
||||
def _fused_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
# When the caller pre-applies the routed input transform outside the
|
||||
# runner (e.g. to overlap it on a separate stream), it passes the
|
||||
# already-transformed routed input as ``hidden_states`` and the original
|
||||
# hidden states as ``shared_experts_input``; skip the transform then.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
||||
self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
)
|
||||
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
self.moe_config.hidden_dim_unpadded
|
||||
if self._quant_method.has_unpadded_output
|
||||
else 0,
|
||||
)
|
||||
|
||||
shared_output, fused_output = _unpack(result)
|
||||
assert shared_output is not None
|
||||
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
result = self._shard_up_proj_tail(
|
||||
fused_output, shared_output, og_hidden_dim_post_xform
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
@@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
@@ -289,6 +290,7 @@ class KimiMoE(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
routed_input_transform=self.routed_expert_down_proj,
|
||||
routed_output_transform=self.routed_output_transform,
|
||||
runner_cls=ROCmLatentMoERunner if self.use_latent_moe else None,
|
||||
)
|
||||
if self.padded_moe_intermediate_size != moe_intermediate_size:
|
||||
w13_weight = getattr(self.experts, "w13_weight", None)
|
||||
|
||||
Reference in New Issue
Block a user