[Kimi-K3] Extend GEMM-RS to GEMM-AR (#53053)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Thien Tran
2026-08-21 03:44:15 +00:00
committed by GitHub
co-authored by OpenAI Codex wzhao18
parent b00f475f09
commit 2785c72a14
9 changed files with 488 additions and 239 deletions
@@ -31,6 +31,7 @@ def _reference(
weight: torch.Tensor,
world_size: int,
group: dist.ProcessGroup,
all_reduce: bool,
) -> torch.Tensor:
M = x.shape[0]
padded_M = (M + world_size - 1) // world_size * world_size
@@ -43,6 +44,10 @@ def _reference(
if padded_M > M:
partial[M:].zero_()
if all_reduce:
dist.all_reduce(partial, group=group)
return partial[:M]
output = torch.empty(
(padded_M // world_size, weight.shape[0]),
dtype=x.dtype,
@@ -57,7 +62,11 @@ def _assert_valid_rows_close(
expected: torch.Tensor,
M: int,
rank: int,
all_reduce: bool,
) -> None:
if all_reduce:
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=4.0)
return
rank_start = rank * actual.shape[0]
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
torch.testing.assert_close(
@@ -68,11 +77,110 @@ def _assert_valid_rows_close(
)
def _run_mode(
*,
all_reduce: bool,
device: torch.device,
group: dist.ProcessGroup,
rank: int,
world_size: int,
weights: dict[int, torch.Tensor],
) -> None:
# cute_dsl is unavailable off CUDA, so import it only inside the GPU worker.
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr
gemm_rs_ar = GemmRsAr(
max_M=max(M for M, _ in _SHAPES),
N=_N,
all_reduce=all_reduce,
)
input_generator = torch.Generator(device=device)
# Alternating shapes exercise producer-flag reuse across different grids,
# CTA-group choices, and both BN=128 and BN=256 dispatches.
for M, K in (*_SHAPES, *_SHAPES[::-1]):
input_generator.manual_seed(2000 + M + K)
x = torch.randn(
M,
K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
expected = _reference(x, weights[K], world_size, group, all_reduce)
actual = gemm_rs_ar(x, weights[K])
torch.accelerator.synchronize(device)
_assert_valid_rows_close(actual, expected, M, rank, all_reduce)
if all_reduce:
# AR outputs must remain valid after the symmetric workspace is reused.
lifetime_M, lifetime_K = 257, 768
input_generator.manual_seed(2500)
lifetime_x = torch.randn(
lifetime_M,
lifetime_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
first_output = gemm_rs_ar(lifetime_x, weights[lifetime_K])
first_snapshot = first_output.clone()
second_output = gemm_rs_ar(-lifetime_x, weights[lifetime_K])
torch.accelerator.synchronize(device)
assert first_output.data_ptr() != second_output.data_ptr()
torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0)
graph_M, graph_K = 1025, 4224
input_generator.manual_seed(3000)
graph_x = torch.randn(
graph_M,
graph_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
graph_expected = _reference(
graph_x,
weights[graph_K],
world_size,
group,
all_reduce,
)
graph_output = torch.empty_like(graph_expected)
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(capture_stream):
for _ in range(3):
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
capture_stream.synchronize()
dist.barrier(group=group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
torch.cuda.current_stream().wait_stream(capture_stream)
dist.barrier(group=group)
for _ in range(3):
graph.replay()
torch.accelerator.synchronize(device)
_assert_valid_rows_close(
graph_output,
graph_expected,
graph_M,
rank,
all_reduce,
)
dist.barrier(group=group)
del gemm_rs_ar, capture_stream, graph, graph_output
def _worker(local_rank: int, world_size: int, master_port: int) -> None:
# This module pulls in cute_dsl, which is unavailable off CUDA, so importing
# it at module level would fail collection. Import it where it is used, as
# `kda.py` does.
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
@@ -92,10 +200,8 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
group = dist.group.WORLD
rank = dist.get_rank(group)
gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N)
weight_generator = torch.Generator(device=device)
input_generator = torch.Generator(device=device)
weights = {}
for K in {K for _, K in _SHAPES}:
weight_generator.manual_seed(1000 + rank * 10 + K)
@@ -107,68 +213,29 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
generator=weight_generator,
)
# Alternating shapes exercise producer-flag reuse across different grids,
# CTA-group choices, and both BN=128 and BN=256 dispatches.
for M, K in (*_SHAPES, *_SHAPES[::-1]):
input_generator.manual_seed(2000 + M + K)
x = torch.randn(
M,
K,
dtype=torch.bfloat16,
# Production binds one mode per worker; exercise both mode-bound instances
# sequentially in this test without implying that both are initialized.
for all_reduce in (False, True):
_run_mode(
all_reduce=all_reduce,
device=device,
generator=input_generator,
group=group,
rank=rank,
world_size=world_size,
weights=weights,
)
expected = _reference(x, weights[K], world_size, group)
actual = gemm_rs(x, weights[K])
torch.accelerator.synchronize(device)
_assert_valid_rows_close(actual, expected, M, rank)
# Exercise GEMM-RS under CUDA graph capture and replay.
graph_M, graph_K = 1025, 4224
input_generator.manual_seed(3000)
graph_x = torch.randn(
graph_M,
graph_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
graph_expected = _reference(graph_x, weights[graph_K], world_size, group)
graph_output = torch.empty_like(graph_expected)
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(capture_stream):
for _ in range(3):
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
capture_stream.synchronize()
dist.barrier(group=group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
torch.cuda.current_stream().wait_stream(capture_stream)
dist.barrier(group=group)
for _ in range(3):
graph.replay()
torch.accelerator.synchronize(device)
_assert_valid_rows_close(graph_output, graph_expected, graph_M, rank)
dist.barrier(group=group)
del gemm_rs
cleanup_dist_env_and_memory()
@pytest.mark.distributed(num_gpus=2)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 GEMM-RS requires SM100",
reason="Kimi-K3 GEMM-RS/AR requires SM100",
)
def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None:
def test_kimi_k3_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch) -> None:
world_size = 2
if torch.accelerator.device_count() < world_size:
pytest.skip("GEMM-RS requires two GPUs")
pytest.skip("GEMM-RS/AR requires two GPUs")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")