[ROCm][Perf] Kimi-K3 Shard Latent MoE up-projection for ROCm path (#51253)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm latent-MoE tail must equal the replicated up-projection.
|
||||
|
||||
A wrong shard offset or a dropped accumulation still runs and still produces
|
||||
plausible text, so these pin the arithmetic rather than the behaviour.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
from torch.multiprocessing import spawn
|
||||
|
||||
from tests.utils import (
|
||||
ensure_current_vllm_config,
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
)
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.runner import moe_runner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCmLatentMoERunner is only wired up on ROCm",
|
||||
)
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
EPS = 1e-5
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
|
||||
norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE)
|
||||
up_proj = ReplicatedLinear(
|
||||
LATENT_SIZE,
|
||||
HIDDEN_SIZE,
|
||||
bias=False,
|
||||
params_dtype=DTYPE,
|
||||
prefix="routed_expert_up_proj",
|
||||
).to(device=device)
|
||||
|
||||
torch.manual_seed(0)
|
||||
norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight))
|
||||
up_proj.weight.data.copy_(torch.randn_like(up_proj.weight) / LATENT_SIZE**0.5)
|
||||
return KimiRoutedOutputTransform(norm, up_proj)
|
||||
|
||||
|
||||
def _tail_runner(
|
||||
transform: KimiRoutedOutputTransform, tp_size: int
|
||||
) -> ROCmLatentMoERunner:
|
||||
"""A runner carrying only what the tail reads, so no engine is needed.
|
||||
|
||||
``_maybe_reduce_final_output`` is left as the real base-class method, so the
|
||||
all-reduce that stitches the shards is the real collective.
|
||||
"""
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
attrs = {
|
||||
"routed_output_transform": transform,
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_size,
|
||||
"_logged_sharded_tail": False,
|
||||
"moe_config": SimpleNamespace(
|
||||
tp_size=tp_size,
|
||||
ep_size=1,
|
||||
is_sequence_parallel=False,
|
||||
skip_final_all_reduce=False,
|
||||
),
|
||||
}
|
||||
for name, value in attrs.items():
|
||||
object.__setattr__(runner, name, value)
|
||||
return runner
|
||||
|
||||
|
||||
def _all_reduced(tensor: torch.Tensor, group) -> torch.Tensor:
|
||||
reduced = tensor.clone()
|
||||
dist.all_reduce(reduced, group=group)
|
||||
return reduced
|
||||
|
||||
|
||||
def _rank_partials(
|
||||
num_tokens: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routed = torch.randn(num_tokens, LATENT_SIZE, device=device, dtype=DTYPE)
|
||||
shared = torch.randn(num_tokens, HIDDEN_SIZE, device=device, dtype=DTYPE)
|
||||
return routed.mul_(0.01), shared
|
||||
|
||||
|
||||
def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None:
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
torch.manual_seed(100 * iteration + rank + 1)
|
||||
routed_output, shared_output = _rank_partials(num_tokens, device)
|
||||
|
||||
expected = F.linear(
|
||||
F.rms_norm(
|
||||
_all_reduced(routed_output, group),
|
||||
(LATENT_SIZE,),
|
||||
transform.norm.weight,
|
||||
EPS,
|
||||
),
|
||||
transform.up_proj.weight,
|
||||
)
|
||||
expected.add_(_all_reduced(shared_output, group))
|
||||
|
||||
actual = runner._shard_up_proj_tail(routed_output, shared_output, None)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_size: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
torch.manual_seed(rank + 1)
|
||||
routed_output, shared_output = _rank_partials(8, device)
|
||||
before = shared_output.clone()
|
||||
latent = transform.norm(_all_reduced(routed_output, group))
|
||||
|
||||
captured: dict = {}
|
||||
real_reduce = runner._maybe_reduce_final_output
|
||||
|
||||
def _capture(states, trunc_size, output_is_reduced=None):
|
||||
captured["states"] = states.clone()
|
||||
captured["output_is_reduced"] = output_is_reduced
|
||||
return real_reduce(states, trunc_size, output_is_reduced)
|
||||
|
||||
object.__setattr__(runner, "_maybe_reduce_final_output", _capture)
|
||||
runner._shard_up_proj_tail(routed_output, shared_output, None)
|
||||
# Restoring the method breaks the runner -> _capture -> runner cycle, so the
|
||||
# captured device tensor is freed before the process group is torn down.
|
||||
object.__setattr__(runner, "_maybe_reduce_final_output", real_reduce)
|
||||
|
||||
assert captured["output_is_reduced"] is False
|
||||
|
||||
shard = HIDDEN_SIZE // tp_size
|
||||
start, end = rank * shard, (rank + 1) * shard
|
||||
local = captured["states"]
|
||||
projected = F.linear(latent, transform.up_proj.weight[start:end])
|
||||
|
||||
torch.testing.assert_close(
|
||||
local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2
|
||||
)
|
||||
torch.testing.assert_close(local[:, :start], before[:, :start], atol=0, rtol=0)
|
||||
torch.testing.assert_close(local[:, end:], before[:, end:], atol=0, rtol=0)
|
||||
|
||||
|
||||
_CHECKS = {
|
||||
"matches_replicated": _check_matches_replicated,
|
||||
"own_shard_only": _check_writes_only_its_own_shard,
|
||||
}
|
||||
|
||||
|
||||
def _worker(local_rank: int, world_size: int, port: str, check: str) -> None:
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
with ensure_current_vllm_config():
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
_CHECKS[check](device, world_size, local_rank)
|
||||
|
||||
|
||||
def _run_ranks(check: str, tp_size: int) -> None:
|
||||
spawn(
|
||||
_worker,
|
||||
args=(tp_size, str(get_open_port()), check),
|
||||
nprocs=tp_size,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 4)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_sharded_tail_tp8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
|
||||
_run_ranks("own_shard_only", 4)
|
||||
|
||||
|
||||
def _runner(**attrs) -> ROCmLatentMoERunner:
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
for name, value in attrs.items():
|
||||
object.__setattr__(runner, name, value)
|
||||
return runner
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shardable,pre_reduced,expect_sharded",
|
||||
[
|
||||
(True, False, True),
|
||||
(True, True, False),
|
||||
(False, False, False),
|
||||
(False, True, False),
|
||||
],
|
||||
)
|
||||
def test_forward_shards_only_when_the_tail_is_valid(
|
||||
shardable: bool,
|
||||
pre_reduced: bool,
|
||||
expect_sharded: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The shard needs an un-reduced shared partial to accumulate into."""
|
||||
base_output, sharded_output = torch.zeros(1), torch.ones(1)
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "forward", lambda *a, **k: base_output)
|
||||
monkeypatch.setattr(
|
||||
ROCmLatentMoERunner, "_fused_forward", lambda *a, **k: sharded_output
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ROCmLatentMoERunner, "_fused_output_is_reduced", property(lambda _: pre_reduced)
|
||||
)
|
||||
runner = _runner(_tail_shardable=shardable)
|
||||
|
||||
result = runner.forward(torch.zeros(1), torch.zeros(1))
|
||||
|
||||
assert result is (sharded_output if expect_sharded else base_output)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base."""
|
||||
|
||||
def _base_init(
|
||||
self,
|
||||
*,
|
||||
tp_size: int = 8,
|
||||
hidden: int = HIDDEN_SIZE,
|
||||
is_sequence_parallel: bool = False,
|
||||
has_up_proj: bool = True,
|
||||
has_shared_experts: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> None:
|
||||
self.moe_config = SimpleNamespace(
|
||||
tp_size=tp_size, is_sequence_parallel=is_sequence_parallel
|
||||
)
|
||||
self.routed_output_transform = SimpleNamespace(
|
||||
norm=None,
|
||||
up_proj=SimpleNamespace(weight=torch.zeros(hidden, LATENT_SIZE))
|
||||
if has_up_proj
|
||||
else None,
|
||||
)
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self._shared_experts = object() if has_shared_experts else None
|
||||
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
|
||||
return ROCmLatentMoERunner
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
runner = build_runner()
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
pytest.param({"tp_size": 1}, id="no-tp"),
|
||||
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
|
||||
pytest.param({"has_up_proj": False}, id="no-up-proj"),
|
||||
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
|
||||
pytest.param({"is_sequence_parallel": True}, id="sequence-parallel"),
|
||||
pytest.param({"routed_scaling_factor": 2.0}, id="routed-scaling-factor"),
|
||||
],
|
||||
)
|
||||
def test_falls_back_when_the_config_breaks_the_shard(
|
||||
build_runner, override: dict
|
||||
) -> None:
|
||||
"""Each of these makes the sharded tail wrong, not merely slower."""
|
||||
runner = build_runner(**override)
|
||||
|
||||
assert not runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == 0
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner, _unpack
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ROCmLatentMoERunner(MoERunner):
|
||||
"""MoE runner for latent MoE with a replicated routed up-projection.
|
||||
|
||||
Mirrors CUDA's LatentMoERunner, but currently only the up projection
|
||||
-sharded path is implemented. (Tier 2)
|
||||
|
||||
Native path: the replicated up-proj produces the full hidden dim on every
|
||||
rank, so the base runner combines routed + shared correctly at any TP size.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
transform = self.routed_output_transform
|
||||
up_proj = getattr(transform, "up_proj", None)
|
||||
tp_size = self.moe_config.tp_size
|
||||
|
||||
self._up_proj_shard_size = 0
|
||||
self._tail_shardable = (
|
||||
up_proj is not None
|
||||
and tp_size > 1
|
||||
and up_proj.weight.shape[0] % tp_size == 0
|
||||
and self._shared_experts is not None
|
||||
and not self.moe_config.is_sequence_parallel
|
||||
and self.routed_scaling_factor == 1.0
|
||||
)
|
||||
if self._tail_shardable:
|
||||
assert up_proj is not None
|
||||
self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size
|
||||
else:
|
||||
logger.warning_once(
|
||||
"K3 latent-MoE tail is not shardable under this config, "
|
||||
"falling back to the replicated up-projection.",
|
||||
scope="global",
|
||||
)
|
||||
self._logged_sharded_tail = False
|
||||
|
||||
def _shard_up_proj_tail(
|
||||
self,
|
||||
fused_output: torch.Tensor,
|
||||
shared_output: torch.Tensor,
|
||||
trunc_size: int | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Tier 2: column-parallel up-projection folded into the final reduce.
|
||||
"""
|
||||
if not self._logged_sharded_tail:
|
||||
self._logged_sharded_tail = True
|
||||
logger.info_once(
|
||||
"Kimi-K3 latent-MoE tail: up-projecting only this rank's "
|
||||
"hidden shard into the shared output.",
|
||||
scope="global",
|
||||
)
|
||||
|
||||
transform = self.routed_output_transform
|
||||
assert transform is not None
|
||||
|
||||
latent = tensor_model_parallel_all_reduce(fused_output)
|
||||
if transform.norm is not None:
|
||||
latent = transform.norm(latent)
|
||||
|
||||
shard_size = self._up_proj_shard_size
|
||||
shard_start = get_tensor_model_parallel_rank() * shard_size
|
||||
up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size)
|
||||
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
|
||||
|
||||
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
|
||||
# beta-add epilogue so folding in the shared partial costs no kernel.
|
||||
hidden_shard.addmm_(latent, up_proj_shard.t())
|
||||
|
||||
return self._maybe_reduce_final_output(
|
||||
shared_output, trunc_size, output_is_reduced=False
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self._tail_shardable and not self._fused_output_is_reduced:
|
||||
return self._fused_forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
return super().forward(
|
||||
hidden_states, router_logits, input_ids, shared_experts_input
|
||||
)
|
||||
|
||||
def _fused_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
# When the caller pre-applies the routed input transform outside the
|
||||
# runner (e.g. to overlap it on a separate stream), it passes the
|
||||
# already-transformed routed input as ``hidden_states`` and the original
|
||||
# hidden states as ``shared_experts_input``; skip the transform then.
|
||||
if shared_experts_input is None:
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
||||
self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
)
|
||||
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
self.moe_config.hidden_dim_unpadded
|
||||
if self._quant_method.has_unpadded_output
|
||||
else 0,
|
||||
)
|
||||
|
||||
shared_output, fused_output = _unpack(result)
|
||||
assert shared_output is not None
|
||||
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
result = self._shard_up_proj_tail(
|
||||
fused_output, shared_output, og_hidden_dim_post_xform
|
||||
)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
@@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
@@ -289,6 +290,7 @@ class KimiMoE(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
routed_input_transform=self.routed_expert_down_proj,
|
||||
routed_output_transform=self.routed_output_transform,
|
||||
runner_cls=ROCmLatentMoERunner if self.use_latent_moe else None,
|
||||
)
|
||||
if self.padded_moe_intermediate_size != moe_intermediate_size:
|
||||
w13_weight = getattr(self.experts, "w13_weight", None)
|
||||
|
||||
Reference in New Issue
Block a user