[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
+117
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import ray
@@ -9,6 +11,8 @@ import torch
import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from ..utils import (
@@ -23,6 +27,59 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def test_sp16_dispatches_only_to_mnnvl_lamport(
monkeypatch: pytest.MonkeyPatch,
):
"""SP16 uses the MNNVL Lamport kernels and rejects same-host dispatch."""
comm = CustomAllreduce.__new__(CustomAllreduce)
comm.disabled = False
comm.world_size = 16
comm.fully_connected = False
comm.mnnvl_only = True
comm._IS_CAPTURING = False
comm.max_mnnvl_all_gather_size = 2 * 1024 * 1024
comm.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
comm.mnnvl_multicast_ptr = 1
comm.mnnvl_lamport_ag_local_ptr = 1
comm.mnnvl_lamport_ag_multicast_ptr = 1
comm.mnnvl_lamport_ag_epoch_ptr = 1
comm.mnnvl_lamport_rs_local_ptr = 1
comm.mnnvl_lamport_rs_epoch_ptr = 1
comm.mnnvl_buffer_size = 32 * 1024 * 1024
comm._ptr = 0
lamport_all_gather = Mock()
lamport_reduce_scatter = Mock()
monkeypatch.setattr(custom_all_reduce.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
custom_all_reduce,
"ops",
SimpleNamespace(
mnnvl_lamport_all_gather=lamport_all_gather,
mnnvl_lamport_reduce_scatter=lamport_reduce_scatter,
),
)
gathered = comm.custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
scattered = comm.custom_reduce_scatter(torch.empty((16, 8), dtype=torch.bfloat16))
assert gathered is not None
assert scattered is not None
lamport_all_gather.assert_called_once()
lamport_reduce_scatter.assert_called_once()
assert not comm.should_custom_ar(torch.empty(8, dtype=torch.bfloat16))
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.int32))
assert not comm.should_custom_all_gather(
torch.empty((131073, 8), dtype=torch.bfloat16)
)
comm.mnnvl_only = False
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
assert not comm.should_custom_reduce_scatter(
torch.empty((16, 8), dtype=torch.bfloat16)
)
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
@@ -80,6 +137,32 @@ def graph_allreduce(
torch.testing.assert_close(out1, inp1)
torch.testing.assert_close(out2, inp2)
fa = get_tp_group().device_communicator.ca_comm
tp_rank = rank % tp_size
with graph_capture(device=device) as graph_capture_context:
local = torch.full(
(512, 4096), tp_rank + 1, dtype=torch.bfloat16, device=device
)
reduce_input = torch.full(
(512 * tp_size, 4096),
tp_rank + 1,
dtype=torch.bfloat16,
device=device,
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
gathered = fa.custom_all_gather(local)
scattered = fa.custom_reduce_scatter(reduce_input)
graph.replay()
assert gathered is not None
assert scattered is not None
expected_gather = torch.cat(
[torch.full_like(local, peer_rank + 1) for peer_rank in range(tp_size)]
)
expected_scatter = torch.full_like(local, tp_size * (tp_size + 1) // 2)
torch.testing.assert_close(gathered, expected_gather)
torch.testing.assert_close(scattered, expected_scatter)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
@@ -110,6 +193,29 @@ def eager_allreduce(
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
group = get_tp_group().device_group
tp_rank = rank % tp_size
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
local = torch.full((64, 4096), tp_rank + 1, dtype=dtype, device=device)
expected_gather = torch.empty(
(64 * tp_size, 4096), dtype=dtype, device=device
)
dist.all_gather_into_tensor(expected_gather, local, group=group)
gathered = fa.custom_all_gather(local)
assert gathered is not None
torch.testing.assert_close(gathered, expected_gather)
reduce_input = torch.full(
(64 * tp_size, 4096), tp_rank + 1, dtype=dtype, device=device
)
expected_scatter = torch.empty((64, 4096), dtype=dtype, device=device)
dist.reduce_scatter_tensor(
expected_scatter, reduce_input.clone(), group=group
)
scattered = fa.custom_reduce_scatter(reduce_input)
assert scattered is not None
torch.testing.assert_close(scattered, expected_scatter)
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
out = inp
for _ in range(num_communication):
@@ -130,3 +236,14 @@ def test_custom_allreduce(
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_custom_collectives_world_size_four(
monkeypatch: pytest.MonkeyPatch,
test_target,
):
"""Exercise the four-rank kernel specialization used by Kimi SP."""
if torch.accelerator.device_count() < 4:
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, 4, 1, test_target)