diff --git a/tests/models/kimi_k3/test_sequence_parallel.py b/tests/models/kimi_k3/test_sequence_parallel.py index 9e8faea099..97d4f7b74a 100644 --- a/tests/models/kimi_k3/test_sequence_parallel.py +++ b/tests/models/kimi_k3/test_sequence_parallel.py @@ -267,21 +267,19 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch): @pytest.mark.parametrize( - ("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"), + ("enabled", "use_sequence_parallel", "tp_size", "expected"), [ - (True, True, True, 8, True), - (False, True, True, 8, False), # opt-in only - (True, False, True, 8, False), # replication only exists under SP - (True, True, False, 8, False), # FusedMoE path owns the reduction - (True, True, True, 1, False), # nothing to shard - (True, True, True, 5, False), # 6144 % 5 -- would fail divide() + (True, True, 8, True), + (False, True, 8, False), # opt-in only + (True, False, 8, False), # replication only exists under SP + (True, True, 1, False), # nothing to shard + (True, True, 5, False), # 6144 % 5 -- would fail divide() ], ) def test_shard_sequence_parallel_mlp_gating( monkeypatch, enabled: bool, use_sequence_parallel: bool, - eligible: bool, tp_size: int, expected: bool, ): @@ -295,7 +293,6 @@ def test_shard_sequence_parallel_mlp_gating( hidden_size=7168, intermediate_size=6144, use_sequence_parallel=use_sequence_parallel, - eligible=eligible, ) is expected ) diff --git a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py index dc4c99ea3e..df1b4ccc1d 100644 --- a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py +++ b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py @@ -81,6 +81,9 @@ class SharedExperts(torch.nn.Module): # - we are using eplb with non-default backend, because of correctness issues # - we are using flashinfer with DP, since there nothing to gain parallel_config = self._moe_config.moe_parallel_config + if getattr(self._layer, "shard_sequence_parallel", False): + # TODO: we may enable this to optimize further + return True return ( parallel_config.enable_eplb and parallel_config.all2all_backend != "allgather_reducescatter" diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index a47d4ed08b..7e06e23107 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -133,7 +133,6 @@ def shard_sequence_parallel_mlp( hidden_size: int, intermediate_size: int, use_sequence_parallel: bool, - eligible: bool, ) -> bool: """Whether to TP-shard a sequence-parallel MLP instead of replicating it. @@ -141,7 +140,7 @@ def shard_sequence_parallel_mlp( the trade-off and :mod:`vllm.envs` for when it is worth enabling. """ enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT - if not (use_sequence_parallel and eligible and enabled): + if not (use_sequence_parallel and enabled): return False tp_size = get_tensor_model_parallel_world_size() return ( @@ -173,7 +172,6 @@ class KimiMLP(nn.Module): quant_config: QuantizationConfig | None = None, reduce_results: bool = True, use_sequence_parallel: bool = False, - can_shard_sequence_parallel: bool = False, prefix: str = "", activation_situ_beta: float | None = None, activation_situ_linear_beta: float | None = None, @@ -184,7 +182,6 @@ class KimiMLP(nn.Module): hidden_size, intermediate_size, use_sequence_parallel, - can_shard_sequence_parallel, ) replicate = use_sequence_parallel and not self.shard_sequence_parallel @@ -532,10 +529,6 @@ class KimiMoE(nn.Module): quant_config=quant_config, reduce_results=False, use_sequence_parallel=use_sequence_parallel, - # Only the MegaMoE path calls the shared experts directly; the - # FusedMoE path below hands them to the runner, which fuses - # their reduction and assumes the replicated layout. - can_shard_sequence_parallel=self.use_mega_moe, prefix=f"{prefix}.shared_experts", activation_situ_beta=activation_situ_beta, activation_situ_linear_beta=activation_situ_linear_beta, @@ -846,7 +839,6 @@ class KimiDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.mlp", use_sequence_parallel=self.use_sequence_parallel, - can_shard_sequence_parallel=True, activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, )