[ROCm][Perf] Kimi-K3 Enable sharded latent MoE up-projection under EP (#54956)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cd63bf9996
commit
a7ff44355c
@@ -23,6 +23,7 @@ from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.runner import moe_runner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.models.kimi_k3.amd import latent_moe_runner
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
|
||||
from vllm.platforms import current_platform
|
||||
@@ -56,21 +57,26 @@ def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
|
||||
|
||||
|
||||
def _tail_runner(
|
||||
transform: KimiRoutedOutputTransform, tp_size: int
|
||||
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
|
||||
) -> ROCmLatentMoERunner:
|
||||
"""A runner carrying only what the tail reads, so no engine is needed.
|
||||
|
||||
``_maybe_reduce_final_output`` is left as the real base-class method, so the
|
||||
all-reduce that stitches the shards is the real collective.
|
||||
|
||||
``tp_world`` is the size of the TP process group, which is what the hidden
|
||||
dim is split by. Under expert parallelism the MoE config reports
|
||||
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
|
||||
that the final reduce still fires.
|
||||
"""
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
attrs = {
|
||||
"routed_output_transform": transform,
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_size,
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
|
||||
"_logged_sharded_tail": False,
|
||||
"moe_config": SimpleNamespace(
|
||||
tp_size=tp_size,
|
||||
ep_size=1,
|
||||
tp_size=1 if use_ep else tp_world,
|
||||
ep_size=tp_world if use_ep else 1,
|
||||
is_sequence_parallel=False,
|
||||
skip_final_all_reduce=False,
|
||||
),
|
||||
@@ -94,9 +100,11 @@ def _rank_partials(
|
||||
return routed.mul_(0.01), shared
|
||||
|
||||
|
||||
def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None:
|
||||
def _check_matches_replicated(
|
||||
device: torch.device, tp_world: int, rank: int, use_ep: bool = False
|
||||
) -> None:
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
runner = _tail_runner(transform, tp_world, use_ep)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
@@ -119,13 +127,22 @@ def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) ->
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _check_matches_replicated_under_ep(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""The served config: the routed experts are expert-parallel, so the MoE
|
||||
config reports no tensor parallelism at all for a group that still splits
|
||||
the hidden dim ``tp_world`` ways."""
|
||||
_check_matches_replicated(device, tp_world, rank, use_ep=True)
|
||||
|
||||
|
||||
def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_size: int, rank: int
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
runner = _tail_runner(transform, tp_world)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
torch.manual_seed(rank + 1)
|
||||
@@ -149,7 +166,7 @@ def _check_writes_only_its_own_shard(
|
||||
|
||||
assert captured["output_is_reduced"] is False
|
||||
|
||||
shard = HIDDEN_SIZE // tp_size
|
||||
shard = HIDDEN_SIZE // tp_world
|
||||
start, end = rank * shard, (rank + 1) * shard
|
||||
local = captured["states"]
|
||||
projected = F.linear(latent, transform.up_proj.weight[start:end])
|
||||
@@ -163,6 +180,7 @@ def _check_writes_only_its_own_shard(
|
||||
|
||||
_CHECKS = {
|
||||
"matches_replicated": _check_matches_replicated,
|
||||
"matches_replicated_ep": _check_matches_replicated_under_ep,
|
||||
"own_shard_only": _check_writes_only_its_own_shard,
|
||||
}
|
||||
|
||||
@@ -196,6 +214,11 @@ def test_sharded_tail_tp8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated_ep", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
|
||||
_run_ranks("own_shard_only", 4)
|
||||
@@ -241,12 +264,19 @@ def test_forward_shards_only_when_the_tail_is_valid(
|
||||
|
||||
@pytest.fixture
|
||||
def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base."""
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base.
|
||||
|
||||
``tp_world`` is the size of the TP process group and is varied independently
|
||||
of ``moe_config.tp_size``, because under expert parallelism those two are
|
||||
different numbers for the same eight ranks.
|
||||
"""
|
||||
world = {"size": 8}
|
||||
|
||||
def _base_init(
|
||||
self,
|
||||
*,
|
||||
tp_size: int = 8,
|
||||
ep_size: int = 1,
|
||||
hidden: int = HIDDEN_SIZE,
|
||||
is_sequence_parallel: bool = False,
|
||||
has_up_proj: bool = True,
|
||||
@@ -254,7 +284,9 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> None:
|
||||
self.moe_config = SimpleNamespace(
|
||||
tp_size=tp_size, is_sequence_parallel=is_sequence_parallel
|
||||
tp_size=tp_size,
|
||||
ep_size=ep_size,
|
||||
is_sequence_parallel=is_sequence_parallel,
|
||||
)
|
||||
self.routed_output_transform = SimpleNamespace(
|
||||
norm=None,
|
||||
@@ -266,20 +298,63 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
self._shared_experts = object() if has_shared_experts else None
|
||||
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
|
||||
return ROCmLatentMoERunner
|
||||
monkeypatch.setattr(
|
||||
latent_moe_runner,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: world["size"],
|
||||
)
|
||||
|
||||
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
|
||||
world["size"] = tp_world
|
||||
return ROCmLatentMoERunner(**kwargs)
|
||||
|
||||
return _build
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
runner = build_runner()
|
||||
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
|
||||
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
|
||||
|
||||
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
|
||||
but the hidden dim is still split across all eight TP ranks, so the tail
|
||||
must shard exactly as it does without EP.
|
||||
"""
|
||||
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
|
||||
reports 4 for a TP process group of 2.
|
||||
|
||||
Only those 2 ranks share a hidden dim and take part in the final
|
||||
all-reduce, so sharding by 4 would leave half of it with no routed
|
||||
contribution on either rank.
|
||||
"""
|
||||
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
pytest.param({"tp_size": 1}, id="no-tp"),
|
||||
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
|
||||
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
|
||||
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
|
||||
pytest.param({"has_up_proj": False}, id="no-up-proj"),
|
||||
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
|
||||
@@ -290,7 +365,12 @@ def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
def test_falls_back_when_the_config_breaks_the_shard(
|
||||
build_runner, override: dict
|
||||
) -> None:
|
||||
"""Each of these makes the sharded tail wrong, not merely slower."""
|
||||
"""Each of these makes the sharded tail wrong, not merely slower.
|
||||
|
||||
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
|
||||
so there is nothing to shard and no all-reduce to stitch shards back
|
||||
together, however large ``moe_config.tp_size`` claims to be.
|
||||
"""
|
||||
runner = build_runner(**override)
|
||||
|
||||
assert not runner._tail_shardable
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -33,7 +34,7 @@ class ROCmLatentMoERunner(MoERunner):
|
||||
|
||||
transform = self.routed_output_transform
|
||||
up_proj = getattr(transform, "up_proj", None)
|
||||
tp_size = self.moe_config.tp_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
self._up_proj_shard_size = 0
|
||||
self._tail_shardable = (
|
||||
|
||||
Reference in New Issue
Block a user