[Kimi K3 Perf] option to shard the shared expert for non mega case, 16.98 GiB memory/GPU saved (#50912)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-04 14:00:44 -04:00
committed by GitHub
parent 7cab4368f2
commit d31de3c421
3 changed files with 10 additions and 18 deletions
@@ -267,21 +267,19 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
@pytest.mark.parametrize(
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
("enabled", "use_sequence_parallel", "tp_size", "expected"),
[
(True, True, True, 8, True),
(False, True, True, 8, False), # opt-in only
(True, False, True, 8, False), # replication only exists under SP
(True, True, False, 8, False), # FusedMoE path owns the reduction
(True, True, True, 1, False), # nothing to shard
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
(True, True, 8, True),
(False, True, 8, False), # opt-in only
(True, False, 8, False), # replication only exists under SP
(True, True, 1, False), # nothing to shard
(True, True, 5, False), # 6144 % 5 -- would fail divide()
],
)
def test_shard_sequence_parallel_mlp_gating(
monkeypatch,
enabled: bool,
use_sequence_parallel: bool,
eligible: bool,
tp_size: int,
expected: bool,
):
@@ -295,7 +293,6 @@ def test_shard_sequence_parallel_mlp_gating(
hidden_size=7168,
intermediate_size=6144,
use_sequence_parallel=use_sequence_parallel,
eligible=eligible,
)
is expected
)
@@ -81,6 +81,9 @@ class SharedExperts(torch.nn.Module):
# - we are using eplb with non-default backend, because of correctness issues
# - we are using flashinfer with DP, since there nothing to gain
parallel_config = self._moe_config.moe_parallel_config
if getattr(self._layer, "shard_sequence_parallel", False):
# TODO: we may enable this to optimize further
return True
return (
parallel_config.enable_eplb
and parallel_config.all2all_backend != "allgather_reducescatter"
+1 -9
View File
@@ -133,7 +133,6 @@ def shard_sequence_parallel_mlp(
hidden_size: int,
intermediate_size: int,
use_sequence_parallel: bool,
eligible: bool,
) -> bool:
"""Whether to TP-shard a sequence-parallel MLP instead of replicating it.
@@ -141,7 +140,7 @@ def shard_sequence_parallel_mlp(
the trade-off and :mod:`vllm.envs` for when it is worth enabling.
"""
enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT
if not (use_sequence_parallel and eligible and enabled):
if not (use_sequence_parallel and enabled):
return False
tp_size = get_tensor_model_parallel_world_size()
return (
@@ -173,7 +172,6 @@ class KimiMLP(nn.Module):
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
use_sequence_parallel: bool = False,
can_shard_sequence_parallel: bool = False,
prefix: str = "",
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
@@ -184,7 +182,6 @@ class KimiMLP(nn.Module):
hidden_size,
intermediate_size,
use_sequence_parallel,
can_shard_sequence_parallel,
)
replicate = use_sequence_parallel and not self.shard_sequence_parallel
@@ -532,10 +529,6 @@ class KimiMoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
use_sequence_parallel=use_sequence_parallel,
# Only the MegaMoE path calls the shared experts directly; the
# FusedMoE path below hands them to the runner, which fuses
# their reduction and assumes the replicated layout.
can_shard_sequence_parallel=self.use_mega_moe,
prefix=f"{prefix}.shared_experts",
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
@@ -846,7 +839,6 @@ class KimiDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
use_sequence_parallel=self.use_sequence_parallel,
can_shard_sequence_parallel=True,
activation_situ_beta=config.activation_situ_beta,
activation_situ_linear_beta=config.activation_situ_linear_beta,
)