Files
vLLM/tests/kernels/test_kimi_k3_gemm_rs.py
T

179 lines
5.2 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from tests.utils import ensure_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
_SHAPES = (
(129, 768),
(257, 768),
(1023, 4224),
(1024, 4224),
(8191, 4224),
)
_N = 512
def _reference(
x: torch.Tensor,
weight: torch.Tensor,
world_size: int,
group: dist.ProcessGroup,
) -> torch.Tensor:
M = x.shape[0]
padded_M = (M + world_size - 1) // world_size * world_size
partial = torch.empty(
(padded_M, weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
torch.mm(x, weight.T, out=partial[:M])
if padded_M > M:
partial[M:].zero_()
output = torch.empty(
(padded_M // world_size, weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
dist.reduce_scatter_single(output, partial, group=group)
return output
def _assert_valid_rows_close(
actual: torch.Tensor,
expected: torch.Tensor,
M: int,
rank: int,
) -> None:
rank_start = rank * actual.shape[0]
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
torch.testing.assert_close(
actual[:valid_rows],
expected[:valid_rows],
rtol=5e-2,
atol=4.0,
)
def _worker(local_rank: int, world_size: int, master_port: int) -> None:
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
}
)
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
group = dist.group.WORLD
rank = dist.get_rank(group)
gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N)
weight_generator = torch.Generator(device=device)
input_generator = torch.Generator(device=device)
weights = {}
for K in {K for _, K in _SHAPES}:
weight_generator.manual_seed(1000 + rank * 10 + K)
weights[K] = torch.randn(
_N,
K,
dtype=torch.bfloat16,
device=device,
generator=weight_generator,
)
# Alternating shapes exercise producer-flag reuse across different grids,
# CTA-group choices, and both BN=128 and BN=256 dispatches.
for M, K in (*_SHAPES, *_SHAPES[::-1]):
input_generator.manual_seed(2000 + M + K)
x = torch.randn(
M,
K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
expected = _reference(x, weights[K], world_size, group)
actual = gemm_rs(x, weights[K])
torch.accelerator.synchronize(device)
_assert_valid_rows_close(actual, expected, M, rank)
# Exercise GEMM-RS under CUDA graph capture and replay.
graph_M, graph_K = 1025, 4224
input_generator.manual_seed(3000)
graph_x = torch.randn(
graph_M,
graph_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
graph_expected = _reference(graph_x, weights[graph_K], world_size, group)
graph_output = torch.empty_like(graph_expected)
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(capture_stream):
for _ in range(3):
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
capture_stream.synchronize()
dist.barrier(group=group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
torch.cuda.current_stream().wait_stream(capture_stream)
dist.barrier(group=group)
for _ in range(3):
graph.replay()
torch.accelerator.synchronize(device)
_assert_valid_rows_close(graph_output, graph_expected, graph_M, rank)
dist.barrier(group=group)
del gemm_rs
cleanup_dist_env_and_memory()
@pytest.mark.distributed(num_gpus=2)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 GEMM-RS requires SM100",
)
def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None:
world_size = 2
if torch.accelerator.device_count() < world_size:
pytest.skip("GEMM-RS requires two GPUs")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
try:
mp.spawn(
_worker,
args=(world_size, get_open_port()),
nprocs=world_size,
)
finally:
cleanup_dist_env_and_memory()