[Kimi-K3] Add GEMM-RS for sequence parallelism (#52079)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -267,19 +267,21 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "use_sequence_parallel", "tp_size", "expected"),
|
||||
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
|
||||
[
|
||||
(True, True, 8, True),
|
||||
(False, True, 8, False), # opt-in only
|
||||
(True, False, 8, False), # replication only exists under SP
|
||||
(True, True, 1, False), # nothing to shard
|
||||
(True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
(True, True, True, 8, True),
|
||||
(False, True, True, 8, False), # opt-in only
|
||||
(True, False, True, 8, False), # replication only exists under SP
|
||||
(True, True, False, 8, False), # FusedMoE path owns the reduction
|
||||
(True, True, True, 1, False), # nothing to shard
|
||||
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
],
|
||||
)
|
||||
def test_shard_sequence_parallel_mlp_gating(
|
||||
monkeypatch,
|
||||
enabled: bool,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
tp_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
@@ -293,6 +295,7 @@ def test_shard_sequence_parallel_mlp_gating(
|
||||
hidden_size=7168,
|
||||
intermediate_size=6144,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
eligible=eligible,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user