[Perf] Enable Kimi K3 shared expert sharding with DeepEP v2 (#56875)

Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Elvir Crnčević
2026-10-08 16:42:15 +00:00
committed by GitHub
co-authored by Misha Goin
parent f4dde3132c
commit 8674a794c7
2 changed files with 41 additions and 4 deletions
@@ -301,6 +301,27 @@ def test_shard_sequence_parallel_mlp_gating(
)
@pytest.mark.parametrize(
("moe_backend", "all2all_backend", "expected"),
[
("deep_gemm_mega_moe", "naive", True),
("auto", "deepep_v2", True),
("auto", "deepep_low_latency", False),
],
)
def test_shared_expert_sharding_backend_gate(
moe_backend: str,
all2all_backend: str,
expected: bool,
):
vllm_config = SimpleNamespace(
kernel_config=SimpleNamespace(moe_backend=moe_backend),
parallel_config=SimpleNamespace(all2all_backend=all2all_backend),
)
assert kimi_model.can_shard_sequence_parallel_shared_expert(vllm_config) is expected
def test_sharded_sequence_parallel_mlp_matches_replicated(default_vllm_config):
"""Sharded SP MLP must reproduce the replicated result for every token.
+20 -4
View File
@@ -156,6 +156,20 @@ def shard_sequence_parallel_mlp(
)
def can_shard_sequence_parallel_shared_expert(vllm_config: VllmConfig) -> bool:
"""Whether the active MoE backend preserves the SP shared-expert contract.
MegaMoE invokes the shared expert directly. DeepEP v2 invokes it through
``MoERunner``, but its EP combine returns the routed output in the local
sequence-sharded layout, so the shared expert can use the same
all-gather/partial-GEMM/reduce-scatter implementation.
"""
return (
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
or vllm_config.parallel_config.all2all_backend == "deepep_v2"
)
def maybe_init_gemm_rs_ar(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool:
# Both feature flags may be enabled; the worker's static SP topology binds
# its singleton to exactly one mode.
@@ -617,10 +631,12 @@ class KimiMoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
use_sequence_parallel=use_sequence_parallel,
# Only the MegaMoE path calls the shared experts directly; the
# FusedMoE path below hands them to the runner, which fuses
# their reduction and assumes the replicated layout.
can_shard_sequence_parallel=self.use_mega_moe,
# MegaMoE calls the shared expert directly. DeepEP v2 hands it
# to MoERunner, but preserves the local SP output layout needed
# by the sharded shared-expert path.
can_shard_sequence_parallel=can_shard_sequence_parallel_shared_expert(
vllm_config
),
run_gemm_rs_ar=run_gemm_rs_ar,
prefix=f"{prefix}.shared_experts",
activation_situ_beta=activation_situ_beta,