[Kimi K3 Perf] option to shard the shared expert for non mega case, 16.98 GiB memory/GPU saved (#50912)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -267,21 +267,19 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
|
||||
("enabled", "use_sequence_parallel", "tp_size", "expected"),
|
||||
[
|
||||
(True, True, True, 8, True),
|
||||
(False, True, True, 8, False), # opt-in only
|
||||
(True, False, True, 8, False), # replication only exists under SP
|
||||
(True, True, False, 8, False), # FusedMoE path owns the reduction
|
||||
(True, True, True, 1, False), # nothing to shard
|
||||
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
(True, True, 8, True),
|
||||
(False, True, 8, False), # opt-in only
|
||||
(True, False, 8, False), # replication only exists under SP
|
||||
(True, True, 1, False), # nothing to shard
|
||||
(True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
],
|
||||
)
|
||||
def test_shard_sequence_parallel_mlp_gating(
|
||||
monkeypatch,
|
||||
enabled: bool,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
tp_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
@@ -295,7 +293,6 @@ def test_shard_sequence_parallel_mlp_gating(
|
||||
hidden_size=7168,
|
||||
intermediate_size=6144,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
eligible=eligible,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
@@ -81,6 +81,9 @@ class SharedExperts(torch.nn.Module):
|
||||
# - we are using eplb with non-default backend, because of correctness issues
|
||||
# - we are using flashinfer with DP, since there nothing to gain
|
||||
parallel_config = self._moe_config.moe_parallel_config
|
||||
if getattr(self._layer, "shard_sequence_parallel", False):
|
||||
# TODO: we may enable this to optimize further
|
||||
return True
|
||||
return (
|
||||
parallel_config.enable_eplb
|
||||
and parallel_config.all2all_backend != "allgather_reducescatter"
|
||||
|
||||
@@ -133,7 +133,6 @@ def shard_sequence_parallel_mlp(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
) -> bool:
|
||||
"""Whether to TP-shard a sequence-parallel MLP instead of replicating it.
|
||||
|
||||
@@ -141,7 +140,7 @@ def shard_sequence_parallel_mlp(
|
||||
the trade-off and :mod:`vllm.envs` for when it is worth enabling.
|
||||
"""
|
||||
enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT
|
||||
if not (use_sequence_parallel and eligible and enabled):
|
||||
if not (use_sequence_parallel and enabled):
|
||||
return False
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
return (
|
||||
@@ -173,7 +172,6 @@ class KimiMLP(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
use_sequence_parallel: bool = False,
|
||||
can_shard_sequence_parallel: bool = False,
|
||||
prefix: str = "",
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
@@ -184,7 +182,6 @@ class KimiMLP(nn.Module):
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
use_sequence_parallel,
|
||||
can_shard_sequence_parallel,
|
||||
)
|
||||
replicate = use_sequence_parallel and not self.shard_sequence_parallel
|
||||
|
||||
@@ -532,10 +529,6 @@ class KimiMoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
# Only the MegaMoE path calls the shared experts directly; the
|
||||
# FusedMoE path below hands them to the runner, which fuses
|
||||
# their reduction and assumes the replicated layout.
|
||||
can_shard_sequence_parallel=self.use_mega_moe,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
@@ -846,7 +839,6 @@ class KimiDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
can_shard_sequence_parallel=True,
|
||||
activation_situ_beta=config.activation_situ_beta,
|
||||
activation_situ_linear_beta=config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user