diff --git a/tests/models/kimi_k3/test_amd_latent_moe_runner.py b/tests/models/kimi_k3/test_amd_latent_moe_runner.py index fd5bcfd22b..e47b5fc08e 100644 --- a/tests/models/kimi_k3/test_amd_latent_moe_runner.py +++ b/tests/models/kimi_k3/test_amd_latent_moe_runner.py @@ -22,7 +22,7 @@ from tests.utils import ( from vllm.distributed import get_tp_group from vllm.model_executor.layers.fused_moe.runner import moe_runner from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear from vllm.models.kimi_k3.amd import latent_moe_runner from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform @@ -40,20 +40,42 @@ EPS = 1e-5 DTYPE = torch.bfloat16 -def _build_transform(device: torch.device) -> KimiRoutedOutputTransform: +def _build_transform( + device: torch.device, *, row_sharded: bool = False +) -> tuple[KimiRoutedOutputTransform, torch.Tensor]: norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE) - up_proj = ReplicatedLinear( - LATENT_SIZE, - HIDDEN_SIZE, - bias=False, - params_dtype=DTYPE, - prefix="routed_expert_up_proj", - ).to(device=device) torch.manual_seed(0) norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight)) - up_proj.weight.data.copy_(torch.randn_like(up_proj.weight) / LATENT_SIZE**0.5) - return KimiRoutedOutputTransform(norm, up_proj) + full_weight = ( + torch.randn(HIDDEN_SIZE, LATENT_SIZE, device=device, dtype=DTYPE) + / LATENT_SIZE**0.5 + ) + + up_proj: ReplicatedLinear | ColumnParallelLinear + if row_sharded: + up_proj = ColumnParallelLinear( + LATENT_SIZE, + HIDDEN_SIZE, + bias=False, + gather_output=True, + params_dtype=DTYPE, + prefix="routed_expert_up_proj", + ).to(device=device) + up_proj.weight_loader(up_proj.weight, full_weight) + else: + up_proj = ReplicatedLinear( + LATENT_SIZE, + HIDDEN_SIZE, + bias=False, + params_dtype=DTYPE, + prefix="routed_expert_up_proj", + ).to(device=device) + up_proj.weight.data.copy_(full_weight) + return ( + KimiRoutedOutputTransform(norm, up_proj, row_sharded=row_sharded), + full_weight, + ) def _tail_runner( @@ -72,6 +94,7 @@ def _tail_runner( runner = object.__new__(ROCmLatentMoERunner) attrs = { "routed_output_transform": transform, + "_up_proj_preshard": bool(getattr(transform, "row_sharded", False)), "_up_proj_shard_size": HIDDEN_SIZE // tp_world, "_logged_sharded_tail": False, "moe_config": SimpleNamespace( @@ -101,9 +124,13 @@ def _rank_partials( def _check_matches_replicated( - device: torch.device, tp_world: int, rank: int, use_ep: bool = False + device: torch.device, + tp_world: int, + rank: int, + use_ep: bool = False, + row_sharded: bool = False, ) -> None: - transform = _build_transform(device) + transform, up_weight = _build_transform(device, row_sharded=row_sharded) runner = _tail_runner(transform, tp_world, use_ep) group = get_tp_group().device_group @@ -118,7 +145,7 @@ def _check_matches_replicated( transform.norm.weight, EPS, ), - transform.up_proj.weight, + up_weight, ) expected.add_(_all_reduced(shared_output, group)) @@ -136,12 +163,37 @@ def _check_matches_replicated_under_ep( _check_matches_replicated(device, tp_world, rank, use_ep=True) +def _check_presharded_matches_replicated( + device: torch.device, tp_world: int, rank: int +) -> None: + _check_matches_replicated(device, tp_world, rank, row_sharded=True) + + +def _check_unfused_tail_gathers_full_hidden( + device: torch.device, tp_world: int, rank: int +) -> None: + transform, up_weight = _build_transform(device, row_sharded=True) + assert transform.up_proj.weight.shape[0] == HIDDEN_SIZE // tp_world + + # all-gather reconstructs the full projection when all ranks feed the same + # input, so use one seed for all ranks. + torch.manual_seed(1234) + latent = torch.randn(4, LATENT_SIZE, device=device, dtype=DTYPE).mul_(0.01) + + torch.testing.assert_close( + transform(latent), + F.linear(transform.norm(latent), up_weight), + atol=8e-2, + rtol=3e-2, + ) + + def _check_writes_only_its_own_shard( device: torch.device, tp_world: int, rank: int ) -> None: """Two ranks that swapped both weight rows and write offsets still sum to the right total, so inspect each slice before the final collective.""" - transform = _build_transform(device) + transform, up_weight = _build_transform(device) runner = _tail_runner(transform, tp_world) group = get_tp_group().device_group @@ -169,7 +221,7 @@ def _check_writes_only_its_own_shard( shard = HIDDEN_SIZE // tp_world start, end = rank * shard, (rank + 1) * shard local = captured["states"] - projected = F.linear(latent, transform.up_proj.weight[start:end]) + projected = F.linear(latent, up_weight[start:end]) torch.testing.assert_close( local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2 @@ -181,6 +233,8 @@ def _check_writes_only_its_own_shard( _CHECKS = { "matches_replicated": _check_matches_replicated, "matches_replicated_ep": _check_matches_replicated_under_ep, + "presharded_matches_replicated": _check_presharded_matches_replicated, + "unfused_gathers": _check_unfused_tail_gathers_full_hidden, "own_shard_only": _check_writes_only_its_own_shard, } @@ -219,6 +273,16 @@ def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None: _run_ranks("matches_replicated_ep", 8) +@multi_gpu_test(num_gpus=4) +def test_presharded_tail_tp4_matches_replicated_projection() -> None: + _run_ranks("presharded_matches_replicated", 4) + + +@multi_gpu_test(num_gpus=4) +def test_presharded_unfused_tail_gathers_the_full_hidden_dim() -> None: + _run_ranks("unfused_gathers", 4) + + @multi_gpu_test(num_gpus=4) def test_sharded_tail_tp4_writes_only_its_own_shard() -> None: _run_ranks("own_shard_only", 4) diff --git a/vllm/models/kimi_k3/amd/latent_moe_runner.py b/vllm/models/kimi_k3/amd/latent_moe_runner.py index dbebfd4182..f6508d6b55 100644 --- a/vllm/models/kimi_k3/amd/latent_moe_runner.py +++ b/vllm/models/kimi_k3/amd/latent_moe_runner.py @@ -36,18 +36,23 @@ class ROCmLatentMoERunner(MoERunner): up_proj = getattr(transform, "up_proj", None) tp_size = get_tensor_model_parallel_world_size() + self._up_proj_preshard = bool(getattr(transform, "row_sharded", False)) self._up_proj_shard_size = 0 self._tail_shardable = ( up_proj is not None and tp_size > 1 - and up_proj.weight.shape[0] % tp_size == 0 + and (self._up_proj_preshard or up_proj.weight.shape[0] % tp_size == 0) and self._shared_experts is not None and not self.moe_config.is_sequence_parallel and self.routed_scaling_factor == 1.0 ) if self._tail_shardable: assert up_proj is not None - self._up_proj_shard_size = up_proj.weight.shape[0] // tp_size + self._up_proj_shard_size = ( + up_proj.weight.shape[0] + if self._up_proj_preshard + else up_proj.weight.shape[0] // tp_size + ) else: logger.warning_once( "K3 latent-MoE tail is not shardable under this config, " @@ -80,7 +85,12 @@ class ROCmLatentMoERunner(MoERunner): shard_size = self._up_proj_shard_size shard_start = get_tensor_model_parallel_rank() * shard_size - up_proj_shard = transform.up_proj.weight.narrow(0, shard_start, shard_size) + weight = transform.up_proj.weight + up_proj_shard = ( + weight + if self._up_proj_preshard + else weight.narrow(0, shard_start, shard_size) + ) hidden_shard = shared_output.narrow(-1, shard_start, shard_size) # hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's diff --git a/vllm/models/kimi_k3/amd/linear.py b/vllm/models/kimi_k3/amd/linear.py index 3a1d827abe..6827731893 100644 --- a/vllm/models/kimi_k3/amd/linear.py +++ b/vllm/models/kimi_k3/amd/linear.py @@ -137,11 +137,13 @@ class KimiRoutedOutputTransform(nn.Module): def __init__( self, norm: RMSNorm | None, - up_proj: ReplicatedLinear, + up_proj: ReplicatedLinear | ColumnParallelLinear, + row_sharded: bool = False, ) -> None: super().__init__() self.norm = norm self.up_proj = up_proj + self.row_sharded = row_sharded def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: if self.norm is not None: @@ -265,7 +267,7 @@ class KimiMoE(nn.Module): self.routed_expert_down_proj: ReplicatedLinear | None self.routed_expert_norm: RMSNorm | None - self.routed_expert_up_proj: ReplicatedLinear | None + self.routed_expert_up_proj: ReplicatedLinear | ColumnParallelLinear | None self.routed_output_transform: KimiRoutedOutputTransform | None if self.use_latent_moe: self.routed_expert_down_proj = ReplicatedLinear( @@ -280,15 +282,36 @@ class KimiMoE(nn.Module): if self.latent_moe_use_norm else None ) - self.routed_expert_up_proj = ReplicatedLinear( - self.moe_hidden_size, - hidden_size, - bias=False, - quant_config=None, - prefix=f"{prefix}.routed_expert_up_proj", + shard_up_proj = ( + self.tp_size > 1 + and hidden_size % self.tp_size == 0 + and self.num_shared_experts is not None + and self.routed_scaling_factor == 1.0 ) + if shard_up_proj: + # gather_output is a fallback restoring the full hidden dim for + # the unfused tail. The fused tail reads from .weight directly + # so it is not triggered + self.routed_expert_up_proj = ColumnParallelLinear( + self.moe_hidden_size, + hidden_size, + bias=False, + gather_output=True, + quant_config=None, + prefix=f"{prefix}.routed_expert_up_proj", + ) + else: + self.routed_expert_up_proj = ReplicatedLinear( + self.moe_hidden_size, + hidden_size, + bias=False, + quant_config=None, + prefix=f"{prefix}.routed_expert_up_proj", + ) self.routed_output_transform = KimiRoutedOutputTransform( - self.routed_expert_norm, self.routed_expert_up_proj + self.routed_expert_norm, + self.routed_expert_up_proj, + row_sharded=shard_up_proj, ) else: self.routed_expert_down_proj = None