[Kimi-K3] Add GEMM-RS for sequence parallelism (#52079)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-08-13 09:02:28 -07:00
committed by GitHub
parent 96acd473a7
commit 6014f9e672
11 changed files with 1590 additions and 12 deletions
@@ -267,19 +267,21 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
@pytest.mark.parametrize(
("enabled", "use_sequence_parallel", "tp_size", "expected"),
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
[
(True, True, 8, True),
(False, True, 8, False), # opt-in only
(True, False, 8, False), # replication only exists under SP
(True, True, 1, False), # nothing to shard
(True, True, 5, False), # 6144 % 5 -- would fail divide()
(True, True, True, 8, True),
(False, True, True, 8, False), # opt-in only
(True, False, True, 8, False), # replication only exists under SP
(True, True, False, 8, False), # FusedMoE path owns the reduction
(True, True, True, 1, False), # nothing to shard
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
],
)
def test_shard_sequence_parallel_mlp_gating(
monkeypatch,
enabled: bool,
use_sequence_parallel: bool,
eligible: bool,
tp_size: int,
expected: bool,
):
@@ -293,6 +295,7 @@ def test_shard_sequence_parallel_mlp_gating(
hidden_size=7168,
intermediate_size=6144,
use_sequence_parallel=use_sequence_parallel,
eligible=eligible,
)
is expected
)