From 6014f9e67291ea96548e263b8784a33a3f055954 Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Fri, 14 Aug 2026 00:02:28 +0800 Subject: [PATCH] [Kimi-K3] Add GEMM-RS for sequence parallelism (#52079) Signed-off-by: Thien Tran --- .buildkite/test_areas/distributed.yaml | 1 + .../kernels/benchmark_kimi_k3_gemm_rs.py | 433 ++++++++++ tests/kernels/test_kimi_k3_gemm_rs.py | 178 ++++ .../models/kimi_k3/test_sequence_parallel.py | 15 +- vllm/cute_utils/__init__.py | 12 + vllm/cute_utils/mbarrier.py | 47 ++ vllm/envs.py | 4 + vllm/models/kimi_k3/nvidia/kda.py | 16 + vllm/models/kimi_k3/nvidia/mla.py | 22 +- vllm/models/kimi_k3/nvidia/model.py | 84 +- .../kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py | 790 ++++++++++++++++++ 11 files changed, 1590 insertions(+), 12 deletions(-) create mode 100644 benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py create mode 100644 tests/kernels/test_kimi_k3_gemm_rs.py create mode 100644 vllm/cute_utils/mbarrier.py create mode 100644 vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 0c6fbdae21..fb0329d583 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -242,6 +242,7 @@ steps: commands: - pytest -v -s tests/distributed/test_context_parallel.py - pytest -v -s tests/distributed/test_nccl_symm_mem.py + - pytest -v -s tests/kernels/test_kimi_k3_gemm_rs.py - pytest -v -s tests/v1/distributed/test_dbo.py - pytest -v -s tests/distributed/test_mnnvl_alltoall.py diff --git a/benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py b/benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py new file mode 100644 index 0000000000..cb04bf86e3 --- /dev/null +++ b/benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Benchmark the SM100 Kimi-K3 GEMM-RS kernel. + +All ranks must belong to one NVLink domain. For example, run a TP8 sweep with: + + torchrun --nproc-per-node=8 \ + benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py +""" + +import argparse +import os +import statistics +from collections.abc import Callable +from dataclasses import dataclass + +import pandas as pd +import torch +import torch.distributed as dist +import torch.distributed._symmetric_memory as symm_mem + +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.distributed import cleanup_dist_env_and_memory +from vllm.distributed.parallel_state import ( + get_tp_group, + init_distributed_environment, + initialize_model_parallel, +) +from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS + +# Shared-expert down-proj and attention O-proj. +_KIMI_K3_PROJECTION_K = (6144, 12288) + + +@dataclass +class Candidate: + name: str + runs: list[Callable[[], torch.Tensor]] + check_correctness: bool = True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--m", + type=int, + nargs="+", + default=[128, 512, 2048, 8192, 32768], + help="Global token counts to benchmark.", + ) + parser.add_argument( + "--k", + type=int, + nargs="+", + help=( + "Per-rank input dimensions. By default, derive the Kimi-K3 " + "shared-expert down-proj and O-proj dimensions from the TP " + "world size." + ), + ) + parser.add_argument("--n", type=int, default=7168) + parser.add_argument( + "--num-workspaces", + type=int, + default=10, + help="Pointer-distinct inputs and CUDA graphs to rotate.", + ) + parser.add_argument("--warmup-replays", type=int, default=5) + parser.add_argument("--samples", type=int, default=20) + return parser.parse_args() + + +def capture_graph( + op: Callable[[], torch.Tensor], + stream: torch.cuda.Stream, + cpu_group: dist.ProcessGroup, +) -> tuple[torch.cuda.CUDAGraph, list[torch.Tensor | None]]: + result: list[torch.Tensor | None] = [None] + stream.wait_stream(torch.cuda.current_stream()) + dist.barrier(group=cpu_group) + with torch.cuda.stream(stream): + for _ in range(3): + result[0] = op() + stream.synchronize() + dist.barrier(group=cpu_group) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + result[0] = op() + torch.cuda.current_stream().wait_stream(stream) + dist.barrier(group=cpu_group) + return graph, result + + +def benchmark_graphs( + candidate_graphs: dict[str, list[torch.cuda.CUDAGraph]], + warmup_replays: int, + samples: int, + device_group: dist.ProcessGroup, + device_barrier: Callable[[], None], +) -> dict[str, float]: + candidate_names = list(candidate_graphs) + for round_index in range(warmup_replays): + for candidate_index in range(len(candidate_names)): + candidate_id = (round_index + candidate_index) % len(candidate_names) + name = candidate_names[candidate_id] + graphs = candidate_graphs[name] + device_barrier() + graphs[round_index % len(graphs)].replay() + torch.accelerator.synchronize() + + timings: dict[str, list[float]] = {name: [] for name in candidate_names} + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + for sample_index in range(samples): + for candidate_index in range(len(candidate_names)): + candidate_id = (sample_index + candidate_index) % len(candidate_names) + name = candidate_names[candidate_id] + graphs = candidate_graphs[name] + device_barrier() + start.record() + graphs[sample_index % len(graphs)].replay() + end.record() + end.synchronize() + + elapsed = torch.tensor( + start.elapsed_time(end) * 1000, + dtype=torch.float64, + device=torch.accelerator.current_device_index(), + ) + dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group) + timings[name].append(elapsed.item()) + return {name: statistics.median(values) for name, values in timings.items()} + + +def valid_rows(M: int, local_M: int, rank: int) -> int: + return min(max(M - rank * local_M, 0), local_M) + + +def benchmark_shape( + gemm_rs: GemmRS, + M: int, + N: int, + K: int, + num_workspaces: int, + warmup_replays: int, + samples: int, + device_group: dist.ProcessGroup, + cpu_group: dist.ProcessGroup, + device_barrier: Callable[[], None], +) -> dict[str, float | int]: + world_size = dist.get_world_size(device_group) + rank = dist.get_rank(device_group) + device = torch.device("cuda", torch.accelerator.current_device_index()) + padded_M = (M + world_size - 1) // world_size * world_size + local_M = padded_M // world_size + + rng = torch.Generator(device=device) + rng.manual_seed(1000 + rank * 10 + M + K) + inputs = [ + torch.randn(M, K, dtype=torch.bfloat16, device=device, generator=rng) + for _ in range(num_workspaces) + ] + weights = [ + torch.randn(N, K, dtype=torch.bfloat16, device=device, generator=rng) + for _ in range(num_workspaces) + ] + + partial = torch.empty((padded_M, N), dtype=torch.bfloat16, device=device) + symm_partial = symm_mem.empty((padded_M, N), dtype=torch.bfloat16, device=device) + symm_partial_handle = symm_mem.rendezvous(symm_partial, device_group) + rs_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)] + symm_rs_inputs = [] + symm_rs_handles = [] + for _ in range(num_workspaces): + rs_input = symm_mem.empty( + (padded_M, N), + dtype=torch.bfloat16, + device=device, + ) + symm_rs_inputs.append(rs_input) + symm_rs_handles.append(symm_mem.rendezvous(rs_input, device_group)) + torch_output = torch.empty((local_M, N), dtype=torch.bfloat16, device=device) + symm_output = torch.empty_like(torch_output) + gemm_output = torch.empty((M, N), dtype=torch.bfloat16, device=device) + + if padded_M > M: + partial[M:].zero_() + symm_partial[M:].zero_() + + def make_torch_ring_ll_gemm_rs( + x: torch.Tensor, weight: torch.Tensor + ) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + torch.mm(x, weight.T, out=partial[:M]) + dist.reduce_scatter_single(torch_output, partial, group=device_group) + return torch_output + + return run + + def make_torch_ldmc_gemm_rs( + x: torch.Tensor, weight: torch.Tensor + ) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + torch.mm(x, weight.T, out=symm_partial[:M]) + dist.reduce_scatter_single( + symm_output, + symm_partial, + group=device_group, + ) + return symm_output + + return run + + def make_fused_gemm_rs( + x: torch.Tensor, weight: torch.Tensor + ) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + return gemm_rs(x, weight) + + return run + + def make_torch_gemm( + x: torch.Tensor, + weight: torch.Tensor, + ) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + return torch.mm(x, weight.T, out=gemm_output) + + return run + + def make_ring_ll_rs(rs_input: torch.Tensor) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + dist.reduce_scatter_single(torch_output, rs_input, group=device_group) + return torch_output + + return run + + def make_ldmc_rs( + rs_input: torch.Tensor, + ) -> Callable[[], torch.Tensor]: + def run() -> torch.Tensor: + dist.reduce_scatter_single(symm_output, rs_input, group=device_group) + return symm_output + + return run + + candidates = ( + Candidate( + "ring_ll_us", + [make_torch_ring_ll_gemm_rs(x, w) for x, w in zip(inputs, weights)], + ), + Candidate( + "ldmc_us", + [make_torch_ldmc_gemm_rs(x, w) for x, w in zip(inputs, weights)], + ), + Candidate( + "gemm_rs_us", + [make_fused_gemm_rs(x, w) for x, w in zip(inputs, weights)], + ), + Candidate( + "torch_gemm_us", + [make_torch_gemm(x, w) for x, w in zip(inputs, weights)], + check_correctness=False, + ), + Candidate( + "ring_ll_rs_us", + [make_ring_ll_rs(x) for x in rs_inputs], + check_correctness=False, + ), + Candidate( + "ldmc_rs_us", + [make_ldmc_rs(x) for x in symm_rs_inputs], + check_correctness=False, + ), + ) + + expected = candidates[0].runs[0]() + rows = valid_rows(M, local_M, rank) + for candidate in candidates[1:]: + if not candidate.check_correctness: + continue + actual = candidate.runs[0]() + torch.accelerator.synchronize(device) + torch.testing.assert_close( + actual[:rows], + expected[:rows], + rtol=5e-2, + atol=4.0, + ) + + candidate_graphs = {} + graph_keepalive: list[object] = [symm_partial_handle, *symm_rs_handles] + for candidate in candidates: + stream = torch.cuda.Stream() + bundles = [capture_graph(run, stream, cpu_group) for run in candidate.runs] + candidate_graphs[candidate.name] = [graph for graph, _ in bundles] + graph_keepalive.extend(bundles) + graph_keepalive.append(stream) + + times = benchmark_graphs( + candidate_graphs, + warmup_replays, + samples, + device_group, + device_barrier, + ) + + best_nccl_rs_us = min(times["ring_ll_rs_us"], times["ldmc_rs_us"]) + return { + "M": M, + "N": N, + "K": K, + **times, + "best_nccl_rs_us": best_nccl_rs_us, + "speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_us"], + "speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_us"], + } + + +def print_results(results: list[dict[str, float | int]]) -> None: + results_df = pd.DataFrame(results) + end_to_end = results_df[ + [ + "M", + "N", + "K", + "ring_ll_us", + "ldmc_us", + "gemm_rs_us", + "speedup_vs_ring_ll", + "speedup_vs_ldmc", + ] + ].rename( + columns={ + "ring_ll_us": "Torch GEMM + NCCL RS (RING_LL) (us)", + "ldmc_us": "Torch GEMM + NCCL RS (LDMC) (us)", + "gemm_rs_us": "GEMM-RS (us)", + "speedup_vs_ring_ll": "Speedup vs RING_LL", + "speedup_vs_ldmc": "Speedup vs LDMC", + } + ) + end_to_end = end_to_end.round( + { + "Torch GEMM + NCCL RS (RING_LL) (us)": 2, + "Torch GEMM + NCCL RS (LDMC) (us)": 2, + "GEMM-RS (us)": 2, + "Speedup vs RING_LL": 3, + "Speedup vs LDMC": 3, + } + ) + components = results_df[ + ["M", "N", "K", "torch_gemm_us", "best_nccl_rs_us", "gemm_rs_us"] + ].rename( + columns={ + "torch_gemm_us": "Torch GEMM (us)", + "best_nccl_rs_us": "NCCL RS (best) (us)", + "gemm_rs_us": "GEMM-RS (us)", + } + ) + components = components.round(2) + + print("### End-to-end latency") + print(end_to_end.to_markdown(index=False)) + print("\n### Component latency") + print(components.to_markdown(index=False)) + print("\nNCCL RS (best) is the faster of RING_LL and LDMC for each shape.") + + +def main() -> None: + args = parse_args() + assert args.m and min(args.m) >= 128 + assert args.n % 256 == 0 + assert args.num_workspaces > 0 + assert args.warmup_replays >= 0 + assert args.samples > 0 + + local_rank = int(os.environ["LOCAL_RANK"]) + torch.accelerator.set_device_index(local_rank) + init_distributed_environment() + world_size = dist.get_world_size() + if args.k is None: + assert all(K % world_size == 0 for K in _KIMI_K3_PROJECTION_K) + K_values = [K // world_size for K in _KIMI_K3_PROJECTION_K] + else: + K_values = args.k + assert all(K % 64 == 0 for K in K_values) + # Reserve symmetric memory for the NCCL-managed benchmark allocations. + os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0" + # NCCL-managed symmetric allocations select the NVLS/LDMC collective path. + symm_mem.set_backend("NCCL") + with set_current_vllm_config(VllmConfig()): + initialize_model_parallel(tensor_model_parallel_size=world_size) + + tp_group = get_tp_group() + group_warmup = torch.zeros(1, device=torch.accelerator.current_device_index()) + dist.all_reduce(group_warmup, group=tp_group.device_group) + pynccl_comm = tp_group.device_communicator.pynccl_comm + assert pynccl_comm is not None + sync_input = torch.zeros(1, device=torch.accelerator.current_device_index()) + sync_output = torch.empty_like(sync_input) + + def device_barrier() -> None: + pynccl_comm.all_reduce(sync_input, sync_output) + + gemm_rs = GemmRS(max_M=max(args.m), N=args.n) + results = [ + benchmark_shape( + gemm_rs, + M, + args.n, + K, + args.num_workspaces, + args.warmup_replays, + args.samples, + tp_group.device_group, + tp_group.cpu_group, + device_barrier, + ) + for K in K_values + for M in args.m + ] + + if tp_group.rank_in_group == 0: + print_results(results) + + dist.barrier(group=tp_group.cpu_group) + del gemm_rs + cleanup_dist_env_and_memory() + + +if __name__ == "__main__": + main() diff --git a/tests/kernels/test_kimi_k3_gemm_rs.py b/tests/kernels/test_kimi_k3_gemm_rs.py new file mode 100644 index 0000000000..5ca54d2ca4 --- /dev/null +++ b/tests/kernels/test_kimi_k3_gemm_rs.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from tests.utils import ensure_current_vllm_config +from vllm.distributed import cleanup_dist_env_and_memory +from vllm.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, +) +from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port +from vllm.utils.system_utils import update_environment_variables + +_SHAPES = ( + (129, 768), + (257, 768), + (1023, 4224), + (1024, 4224), + (8191, 4224), +) +_N = 512 + + +def _reference( + x: torch.Tensor, + weight: torch.Tensor, + world_size: int, + group: dist.ProcessGroup, +) -> torch.Tensor: + M = x.shape[0] + padded_M = (M + world_size - 1) // world_size * world_size + partial = torch.empty( + (padded_M, weight.shape[0]), + dtype=x.dtype, + device=x.device, + ) + torch.mm(x, weight.T, out=partial[:M]) + if padded_M > M: + partial[M:].zero_() + + output = torch.empty( + (padded_M // world_size, weight.shape[0]), + dtype=x.dtype, + device=x.device, + ) + dist.reduce_scatter_single(output, partial, group=group) + return output + + +def _assert_valid_rows_close( + actual: torch.Tensor, + expected: torch.Tensor, + M: int, + rank: int, +) -> None: + rank_start = rank * actual.shape[0] + valid_rows = min(max(M - rank_start, 0), actual.shape[0]) + torch.testing.assert_close( + actual[:valid_rows], + expected[:valid_rows], + rtol=5e-2, + atol=4.0, + ) + + +def _worker(local_rank: int, world_size: int, master_port: int) -> None: + device = torch.device("cuda", local_rank) + torch.accelerator.set_device_index(device) + update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": str(master_port), + } + ) + + init_distributed_environment() + with ensure_current_vllm_config(): + initialize_model_parallel(tensor_model_parallel_size=world_size) + + group = dist.group.WORLD + rank = dist.get_rank(group) + gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N) + + weight_generator = torch.Generator(device=device) + input_generator = torch.Generator(device=device) + weights = {} + for K in {K for _, K in _SHAPES}: + weight_generator.manual_seed(1000 + rank * 10 + K) + weights[K] = torch.randn( + _N, + K, + dtype=torch.bfloat16, + device=device, + generator=weight_generator, + ) + + # Alternating shapes exercise producer-flag reuse across different grids, + # CTA-group choices, and both BN=128 and BN=256 dispatches. + for M, K in (*_SHAPES, *_SHAPES[::-1]): + input_generator.manual_seed(2000 + M + K) + x = torch.randn( + M, + K, + dtype=torch.bfloat16, + device=device, + generator=input_generator, + ) + expected = _reference(x, weights[K], world_size, group) + actual = gemm_rs(x, weights[K]) + torch.accelerator.synchronize(device) + _assert_valid_rows_close(actual, expected, M, rank) + + # Exercise GEMM-RS under CUDA graph capture and replay. + graph_M, graph_K = 1025, 4224 + input_generator.manual_seed(3000) + graph_x = torch.randn( + graph_M, + graph_K, + dtype=torch.bfloat16, + device=device, + generator=input_generator, + ) + graph_expected = _reference(graph_x, weights[graph_K], world_size, group) + graph_output = torch.empty_like(graph_expected) + + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(capture_stream): + for _ in range(3): + graph_output.copy_(gemm_rs(graph_x, weights[graph_K])) + capture_stream.synchronize() + dist.barrier(group=group) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + graph_output.copy_(gemm_rs(graph_x, weights[graph_K])) + torch.cuda.current_stream().wait_stream(capture_stream) + dist.barrier(group=group) + + for _ in range(3): + graph.replay() + torch.accelerator.synchronize(device) + _assert_valid_rows_close(graph_output, graph_expected, graph_M, rank) + + dist.barrier(group=group) + del gemm_rs + cleanup_dist_env_and_memory() + + +@pytest.mark.distributed(num_gpus=2) +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), + reason="Kimi-K3 GEMM-RS requires SM100", +) +def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None: + world_size = 2 + if torch.accelerator.device_count() < world_size: + pytest.skip("GEMM-RS requires two GPUs") + + monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1") + monkeypatch.setenv("NCCL_NVLS_ENABLE", "1") + try: + mp.spawn( + _worker, + args=(world_size, get_open_port()), + nprocs=world_size, + ) + finally: + cleanup_dist_env_and_memory() diff --git a/tests/models/kimi_k3/test_sequence_parallel.py b/tests/models/kimi_k3/test_sequence_parallel.py index 97d4f7b74a..9e8faea099 100644 --- a/tests/models/kimi_k3/test_sequence_parallel.py +++ b/tests/models/kimi_k3/test_sequence_parallel.py @@ -267,19 +267,21 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch): @pytest.mark.parametrize( - ("enabled", "use_sequence_parallel", "tp_size", "expected"), + ("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"), [ - (True, True, 8, True), - (False, True, 8, False), # opt-in only - (True, False, 8, False), # replication only exists under SP - (True, True, 1, False), # nothing to shard - (True, True, 5, False), # 6144 % 5 -- would fail divide() + (True, True, True, 8, True), + (False, True, True, 8, False), # opt-in only + (True, False, True, 8, False), # replication only exists under SP + (True, True, False, 8, False), # FusedMoE path owns the reduction + (True, True, True, 1, False), # nothing to shard + (True, True, True, 5, False), # 6144 % 5 -- would fail divide() ], ) def test_shard_sequence_parallel_mlp_gating( monkeypatch, enabled: bool, use_sequence_parallel: bool, + eligible: bool, tp_size: int, expected: bool, ): @@ -293,6 +295,7 @@ def test_shard_sequence_parallel_mlp_gating( hidden_size=7168, intermediate_size=6144, use_sequence_parallel=use_sequence_parallel, + eligible=eligible, ) is expected ) diff --git a/vllm/cute_utils/__init__.py b/vllm/cute_utils/__init__.py index d86a550eea..23721aac44 100644 --- a/vllm/cute_utils/__init__.py +++ b/vllm/cute_utils/__init__.py @@ -40,6 +40,18 @@ def recast_val(x, dtype, *, loc=None, ip=None): return dtype(llvm.bitcast(dtype.mlir_type, x.ir_value(loc=loc, ip=ip))) +@dsl_user_op +def to_cta0_smem(ptr: cute.Pointer, *, loc=None, ip=None): + return cute.make_ptr( + ptr.dtype, + ptr.toint(loc=loc, ip=ip) & 0xFEFF_FFFF, + cute.AddressSpace.smem, + assumed_align=8, + loc=loc, + ip=ip, + ) + + def simple_tma_copy(atom, src, dst, mbar=None, cache_policy=None): """A simple helper that wraps group_modes() and tma_partition() NOTE: this should be called WITHOUT cute.elect_one() diff --git a/vllm/cute_utils/mbarrier.py b/vllm/cute_utils/mbarrier.py new file mode 100644 index 0000000000..d913525810 --- /dev/null +++ b/vllm/cute_utils/mbarrier.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from cutlass import Int32 +from cutlass._mlir.dialects import nvvm +from cutlass.cutlass_dsl import dsl_user_op + +_SPACE_MAP = { + "cta": nvvm.MBarrierSpaceKind.CTA, + "cluster": nvvm.MBarrierSpaceKind.CLUSTER, +} + +_MEMORY_ORDER_MAP = { + "weak": nvvm.MemOrderKind.WEAK, + "relaxed": nvvm.MemOrderKind.RELAXED, + "acquire": nvvm.MemOrderKind.ACQUIRE, + "release": nvvm.MemOrderKind.RELEASE, + "acq_rel": nvvm.MemOrderKind.ACQ_REL, +} + + +@dsl_user_op +def arrive(mbar, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None): + nvvm.mbarrier_txn( + mbar.to_llvm_ptr(loc=loc, ip=ip), + Int32(1).ir_value(loc=loc, ip=ip), + kind=nvvm.MBarrierTxnKind.ARRIVE, + space=_SPACE_MAP[space], + order=_MEMORY_ORDER_MAP[order], + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def arrive_expect_tx( + mbar, size, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None +): + nvvm.mbarrier_txn( + mbar.to_llvm_ptr(loc=loc, ip=ip), + Int32(size).ir_value(loc=loc, ip=ip), + kind=nvvm.MBarrierTxnKind.ARRIVE_EXPECT_TX, + space=_SPACE_MAP[space], + order=_MEMORY_ORDER_MAP[order], + loc=loc, + ip=ip, + ) diff --git a/vllm/envs.py b/vllm/envs.py index 3382fa1d67..5715508d12 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -206,6 +206,7 @@ if TYPE_CHECKING: VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True VLLM_MOE_SKIP_PADDING: bool = True VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False + VLLM_KIMI_K3_GEMM_RS: bool = False VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_USE_FLASHINFER_MOE_INT4: bool = False VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None @@ -1594,6 +1595,9 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool( int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0")) ), + # Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel + # row-parallel projections. All TP ranks must belong to one NVLink domain. + "VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))), # Allow use of FlashInfer FP8 block-scale GEMM for linear layers. # This uses TensorRT-LLM kernels and requires SM90+ (Hopper). "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool( diff --git a/vllm/models/kimi_k3/nvidia/kda.py b/vllm/models/kimi_k3/nvidia/kda.py index b59293caeb..c17a8ec4e9 100644 --- a/vllm/models/kimi_k3/nvidia/kda.py +++ b/vllm/models/kimi_k3/nvidia/kda.py @@ -305,6 +305,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): config: KimiLinearConfig, vllm_config: VllmConfig, prefix: str = "", + run_gemm_rs: bool = False, ) -> None: super().__init__(config, vllm_config, prefix) @@ -470,7 +471,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): quant_config=self.quant_config, prefix=f"{prefix}.o_proj", ) + self.run_gemm_rs = run_gemm_rs + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj) + if not self.run_gemm_rs: + logger.warning_once( + "GEMM-RS is disabled for %s due to an incompatible projection.", + prefix, + ) compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") @@ -511,6 +521,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): core_attn_out=core_attn_out, ) core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)") + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + + gemm_rs = get_gemm_rs() + if gemm_rs.should_run(core_attn_out): + return gemm_rs(core_attn_out, self.o_proj.weight) return self.o_proj(core_attn_out)[0] @eager_break_during_capture diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 708104b073..17fd1f2658 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -134,6 +134,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): aux_stream: torch.cuda.Stream | None = None, use_rope: bool = False, non_causal_multi_token_decode: bool = False, + run_gemm_rs: bool = False, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -270,6 +271,16 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): quant_config=quant_config, prefix=f"{prefix}.o_proj", ) + self.run_gemm_rs = run_gemm_rs + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + + self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj) + if not self.run_gemm_rs: + logger.warning_once( + "GEMM-RS is disabled for %s due to an incompatible projection.", + prefix, + ) # ---- Attention backend / impl / KV cache ---- self.quant_config = quant_config @@ -541,10 +552,13 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): if gate is not None: attn_out = _gate_sigmoid_mul(attn_out, gate) - # ``o_proj`` (RowParallelLinear + out-of-place all-reduce) returns a - # fresh private tensor, so return it directly rather than copying into a - # caller buffer -- the previous ``output[:] = ...`` convention forced an - # extra [num_tokens, hidden] copy per layer. + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + + gemm_rs = get_gemm_rs() + if gemm_rs.should_run(attn_out): + return gemm_rs(attn_out, self.o_proj.weight) + return self.o_proj(attn_out)[0] @eager_break_during_capture diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index 59682fe32a..826fb0d349 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -133,6 +133,7 @@ def shard_sequence_parallel_mlp( hidden_size: int, intermediate_size: int, use_sequence_parallel: bool, + eligible: bool, ) -> bool: """Whether to TP-shard a sequence-parallel MLP instead of replicating it. @@ -140,7 +141,7 @@ def shard_sequence_parallel_mlp( the trade-off and :mod:`vllm.envs` for when it is worth enabling. """ enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT - if not (use_sequence_parallel and enabled): + if not (use_sequence_parallel and eligible and enabled): return False tp_size = get_tensor_model_parallel_world_size() return ( @@ -148,6 +149,44 @@ def shard_sequence_parallel_mlp( ) +def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool: + if not envs.VLLM_KIMI_K3_GEMM_RS: + return False + + parallel_config = vllm_config.parallel_config + tp_size = parallel_config.tensor_parallel_size + if not use_sequence_parallel: + reason = "sequence parallelism is disabled" + elif parallel_config.use_ubatching: + reason = "ubatching is enabled" + elif vllm_config.model_config.dtype != torch.bfloat16: + reason = "the model dtype is not BF16" + elif not current_platform.is_cuda(): + reason = "the device is not CUDA" + elif not current_platform.is_device_capability_family(100): + reason = "the device is not SM100-family" + elif not 1 < tp_size <= 16: + reason = "TP size is not in the supported range 2-16" + elif 128 % tp_size != 0: + reason = "TP size does not divide 128" + else: + reason = None + + if reason is not None: + logger.warning_once("GEMM-RS was requested but is disabled because %s.", reason) + return False + + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import init_gemm_rs + + config = vllm_config.model_config.hf_text_config + init_gemm_rs( + max_M=vllm_config.scheduler_config.max_num_batched_tokens, + N=config.hidden_size, + ) + logger.info_once("GEMM-RS is enabled.") + return True + + class KimiMLP(nn.Module): """Dense / shared-expert MLP, optionally TP-sharded under sequence parallel. @@ -172,6 +211,8 @@ class KimiMLP(nn.Module): quant_config: QuantizationConfig | None = None, reduce_results: bool = True, use_sequence_parallel: bool = False, + can_shard_sequence_parallel: bool = False, + run_gemm_rs: bool = False, prefix: str = "", activation_situ_beta: float | None = None, activation_situ_linear_beta: float | None = None, @@ -182,7 +223,9 @@ class KimiMLP(nn.Module): hidden_size, intermediate_size, use_sequence_parallel, + can_shard_sequence_parallel, ) + self.run_gemm_rs = self.shard_sequence_parallel and run_gemm_rs replicate = use_sequence_parallel and not self.shard_sequence_parallel self.gate_up_proj = MergedColumnParallelLinear( @@ -204,6 +247,15 @@ class KimiMLP(nn.Module): disable_tp=replicate, prefix=f"{prefix}.down_proj", ) + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + + self.run_gemm_rs = get_gemm_rs().can_run(self.down_proj) + if not self.run_gemm_rs: + logger.warning_once( + "GEMM-RS is disabled for %s due to an incompatible projection.", + prefix, + ) if hidden_act == "silu": self.act_fn = SiluAndMul() elif hidden_act == "situ": @@ -226,6 +278,14 @@ class KimiMLP(nn.Module): x = sp_all_gather(x) gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) + + if self.run_gemm_rs: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + + gemm_rs = get_gemm_rs() + if gemm_rs.should_run(x): + return gemm_rs(x, self.down_proj.weight) + x, _ = self.down_proj(x) if self.shard_sequence_parallel: x = sp_reduce_scatter(x) @@ -462,6 +522,7 @@ class KimiMoE(nn.Module): prefix: str = "", layer_idx: int = 0, use_sequence_parallel: bool = False, + run_gemm_rs: bool = False, ): super().__init__() hidden_size = config.hidden_size @@ -546,6 +607,11 @@ class KimiMoE(nn.Module): quant_config=quant_config, reduce_results=False, use_sequence_parallel=use_sequence_parallel, + # Only the MegaMoE path calls the shared experts directly; the + # FusedMoE path below hands them to the runner, which fuses + # their reduction and assumes the replicated layout. + can_shard_sequence_parallel=self.use_mega_moe, + run_gemm_rs=run_gemm_rs, prefix=f"{prefix}.shared_experts", activation_situ_beta=activation_situ_beta, activation_situ_linear_beta=activation_situ_linear_beta, @@ -767,6 +833,7 @@ class KimiDecoderLayer(nn.Module): vllm_config: VllmConfig, prefix: str = "", aux_stream: torch.cuda.Stream | None = None, + run_gemm_rs: bool = False, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -801,6 +868,7 @@ class KimiDecoderLayer(nn.Module): config, vllm_config, prefix=f"{prefix}.self_attn", + run_gemm_rs=run_gemm_rs, ) self._self_attn_writes_output = False else: @@ -837,6 +905,7 @@ class KimiDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", aux_stream=aux_stream, + run_gemm_rs=run_gemm_rs, ) self._self_attn_writes_output = False @@ -851,6 +920,7 @@ class KimiDecoderLayer(nn.Module): prefix=f"{prefix}.block_sparse_moe", layer_idx=layer_idx, use_sequence_parallel=self.use_sequence_parallel, + run_gemm_rs=run_gemm_rs, ) self.mlp = self.block_sparse_moe else: @@ -861,6 +931,8 @@ class KimiDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.mlp", use_sequence_parallel=self.use_sequence_parallel, + can_shard_sequence_parallel=True, + run_gemm_rs=run_gemm_rs, activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, ) @@ -991,15 +1063,18 @@ class KimiDecoderLayer(nn.Module): ) assert hidden_states is not None + M = None if self.use_sequence_parallel: hidden_states = sp_all_gather(hidden_states) # Remove SP padding before attention. hidden_states = hidden_states[: positions.shape[0]] + M = hidden_states.shape[0] # Attention. hidden_states = self._run_self_attn(positions, hidden_states) - if self.use_sequence_parallel: + # GEMM-RS returns the local sequence shard; standard O-proj preserves M. + if self.use_sequence_parallel and hidden_states.shape[0] == M: # Add SP padding if needed, and then perform reduce scatter. hidden_states = sp_reduce_scatter(hidden_states) @@ -1038,6 +1113,10 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): self.vocab_size = config.vocab_size + # GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP + # ranks to belong to one NVLink domain. + self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel) + if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -1058,6 +1137,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): vllm_config, prefix, aux_stream=aux_stream, + run_gemm_rs=self.run_gemm_rs, ) self.start_layer, self.end_layer, self.layers = make_layers( diff --git a/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py b/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py new file mode 100644 index 0000000000..a775cd7b12 --- /dev/null +++ b/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py @@ -0,0 +1,790 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""SM100 BF16 GEMM with a fused tensor-parallel reduce-scatter.""" + +# Based on CUTLASS's Blackwell distributed GEMM-RS example at dcf215a. +# See https://github.com/NVIDIA/cutlass/issues/3117 for memory semantics. + +from functools import cache + +import cutlass +import torch +import torch.distributed._symmetric_memory as symm_mem +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Int32, Int64, Uint16, cute, utils +from cutlass._mlir import ir +from cutlass._mlir.dialects import llvm, nvvm, vector +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import make_fake_stream, make_fake_tensor, make_ptr, nullptr +from cutlass.cutlass_dsl import dsl_user_op +from cutlass.utils import get_smem_capacity_in_bytes + +from vllm.cute_utils import _tcgen05, mbarrier, simple_tma_copy, to_cta0_smem +from vllm.distributed import get_tp_group +from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod + + +@dsl_user_op +def nanosleep(ns: int, *, loc=None, ip=None) -> None: + nvvm.nanosleep(Int32(ns).ir_value(loc=loc, ip=ip), loc=loc, ip=ip) + + +@dsl_user_op +def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: + # NOTE: assume x is contiguous + assert x.element_type == BFloat16 + vec_type = ".v4.bf16x2" + + ptr = x.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip) + asm = ( + "multimem.ld_reduce.relaxed.gpu.global.add.acc::f32" + f"{vec_type} {{$0, $1, $2, $3}}, [$4];" + ) + struct = llvm.inline_asm( + llvm.StructType.get_literal([Int32.mlir_type] * 4), + [ptr], + asm, + "=r,=r,=r,=r,l", + has_side_effects=True, + loc=loc, + ip=ip, + ) + vec = vector.from_elements( + ir.VectorType.get([4], Int32.mlir_type, loc=loc), + [ + llvm.extractvalue(Int32.mlir_type, struct, [i], loc=loc, ip=ip) + for i in range(4) + ], + loc=loc, + ip=ip, + ) + ssa = cute.TensorSSA(vec, 4, Int32) + + y = cute.make_rmem_tensor(4, Int32) + y.store(ssa) + return cute.recast_tensor(y, x.element_type) + + +class Sm100GemmRsBF16: + def __init__( + self, rank: int, num_ranks: int, BN: int = 128, cta_group: int = 1 + ) -> None: + self.rank = rank + self.num_ranks = num_ranks + BM, BK = 128, 64 + self.cta_tile = (BM, BN, BK) + self.cta_group = cta_group + + smem_bytes = get_smem_capacity_in_bytes() + self.stage_size = (BM + (BN // cta_group)) * BK * 2 + self.num_stages = smem_bytes // self.stage_size + + @cute.jit + def prepare_tma( + self, tensor: cute.Tensor, BM: cutlass.Constexpr, BK: cutlass.Constexpr + ) -> cpasync.TmaInfo: + tma_group = ( + tcgen05.CtaGroup.TWO if self.cta_group == 2 else tcgen05.CtaGroup.ONE + ) + tma_op = cpasync.CopyBulkTensorTileG2SOp(cta_group=tma_group) + swizzle_128b = cute.make_swizzle(3, 4, 3) + layout = cute.make_layout( + (BM, BK, self.num_stages), + stride=(BK, 1, BM * BK), + ) + layout = cute.make_composed_layout(swizzle_128b, 0, layout) + return cpasync.make_tiled_tma_atom(tma_op, tensor, layout, (BM, BK)) + + @cute.jit + def __call__( + self, + A: cute.Tensor, + B: cute.Tensor, + partial_uc: cute.Tensor, + partial_mc_ptr: cute.Pointer, + output: cute.Tensor, + flags_uc: cute.Tensor, + flags_mc_ptr: cute.Pointer, + peer_flag_ptr: cute.Pointer, + grid_size: Int32, + stream: CUstream, + ) -> None: + N = B.shape[0] + BM, BN, BK = self.cta_tile + A_tma = self.prepare_tma(A, BM, BK) + B_tma = self.prepare_tma(B, BN // self.cta_group, BK) + padded_M = partial_uc.shape[0] + partial_mc = cute.make_tensor( + partial_mc_ptr, + cute.make_layout((padded_M, N), stride=(N, 1)), + ) + peer_flags = cute.make_tensor( + peer_flag_ptr, + cute.make_layout(self.num_ranks), + ) + + grid = (grid_size, 1, 1) + block = (10 * 32, 1, 1) + cluster = (self.cta_group, 1, 1) + self.kernel( + A_tma, + B_tma, + partial_uc, + partial_mc, + output, + flags_uc.iterator, + flags_mc_ptr, + peer_flags, + ).launch(grid=grid, block=block, cluster=cluster, stream=stream) + + @cute.kernel + def kernel( + self, + A_tma: cpasync.TmaInfo, + B_tma: cpasync.TmaInfo, + partial_uc: cute.Tensor, + partial_mc: cute.Tensor, + output: cute.Tensor, + flags_uc_ptr: cute.Pointer, + flags_mc_ptr: cute.Pointer, + peer_flags: cute.Tensor, + ) -> None: + tid, _, _ = cute.arch.thread_idx() + raw_bid, _, _ = cute.arch.block_idx() + num_bids, _, _ = cute.arch.grid_dim() + warp_id = cute.arch.make_warp_uniform(tid // 32) + + BM, BN, BK = self.cta_tile + cta_group = self.cta_group + num_stages = self.num_stages + num_ranks = self.num_ranks + + is_2cta = cta_group == 2 + cta_rank = raw_bid % self.cta_group + num_tmem_stages = 512 // BN + + smem = utils.SmemAllocator() + sA = smem.allocate_tensor( + BFloat16, + A_tma.smem_layout.outer, + byte_alignment=128, + swizzle=A_tma.smem_layout.inner, + ) + sB = smem.allocate_tensor( + BFloat16, + B_tma.smem_layout.outer, + byte_alignment=128, + swizzle=B_tma.smem_layout.inner, + ) + tma_full_mbar = smem.allocate_array(Int64, num_stages) + tma_empty_mbar = smem.allocate_array(Int64, num_stages) + tmem_full_mbar = smem.allocate_array(Int64, num_tmem_stages) + tmem_empty_mbar = smem.allocate_array(Int64, num_tmem_stages) + taddr = smem.allocate(Int32, 4) + + # Named barriers + BAR_TMEM_ALLOC = 1 + BAR_EPILOGUE = 2 + BAR_COMM = 3 + + M, K = A_tma.tma_tensor.shape + N, _ = B_tma.tma_tensor.shape + grid_m = cute.ceil_div(M, BM) + # Keep 2-CTA clusters within a single N tile. + grid_m = cute.ceil_div(grid_m, cta_group) * cta_group + grid_n = cute.ceil_div(N, BN) + + if warp_id == 0: + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_init(tma_full_mbar + i, cta_group) + cute.arch.mbarrier_init(tma_empty_mbar + i, 1) + for i in cutlass.range_constexpr(num_tmem_stages): + cute.arch.mbarrier_init(tmem_full_mbar + i, 1) + cute.arch.mbarrier_init(tmem_empty_mbar + i, 128 * cta_group) + cute.arch.mbarrier_init_fence() + elif warp_id == 1: + cpasync.prefetch_descriptor(A_tma.atom) + cpasync.prefetch_descriptor(B_tma.atom) + + if cutlass.const_expr(is_2cta): + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + else: + cute.arch.sync_threads() + + total_tiles = grid_m * grid_n + + if warp_id == 9: + # TMA warp + tma_stage = 0 + parity = 1 + + if cutlass.const_expr(is_2cta): + tma_full_mbar_ = to_cta0_smem(tma_full_mbar) + else: + tma_full_mbar_ = tma_full_mbar + + # Select global-memory tiles. + # [(BM, BK), (M/BM, K/BK)] + gA_tiles = cute.zipped_divide(A_tma.tma_tensor, (BM, BK)) + gB_tiles = cute.zipped_divide(B_tma.tma_tensor, (BN // cta_group, BK)) + + for bid in range(raw_bid, total_tiles, num_bids): + bid_m = bid % grid_m + bid_n = bid // grid_m + if cutlass.const_expr(cta_group == 2): + bid_n = bid_n * cta_group + cta_rank + + for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1): + mbar = tma_full_mbar_ + tma_stage + cute.arch.mbarrier_wait(tma_empty_mbar + tma_stage, parity) + + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar, self.stage_size, "cluster") + simple_tma_copy( + A_tma.atom, + gA_tiles[None, (bid_m, iter_k)], + sA[None, None, tma_stage], + mbar, + ) + simple_tma_copy( + B_tma.atom, + gB_tiles[None, (bid_n, iter_k)], + sB[None, None, tma_stage], + mbar, + ) + + tma_stage = (tma_stage + 1) % num_stages + if tma_stage == 0: + parity ^= 1 + + elif warp_id == 8: + # MMA warp + cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32) + + if cta_rank == 0: + tma_stage = 0 + tma_full_parity = 0 + tmem_stage = 0 + tmem_empty_parity = 1 + + MMA_M = BM * cta_group + MMA_N = BN + idesc = _tcgen05.make_bf16_idesc(MMA_M, MMA_N) + sdesc = _tcgen05.make_sdesc_128B_swizzle(0) + multicast_mask = Uint16((1 << self.cta_group) - 1) + + for bid in range(raw_bid, total_tiles, num_bids): + cute.arch.mbarrier_wait( + tmem_empty_mbar + tmem_stage, tmem_empty_parity + ) + _tcgen05.fence_after_thread_sync() + + for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1): + d_tmem = BN * tmem_stage + a_addr = sA[None, None, tma_stage].iterator.toint() + b_addr = sB[None, None, tma_stage].iterator.toint() + a_desc = sdesc | (a_addr >> 4) + b_desc = sdesc | (b_addr >> 4) + + cute.arch.mbarrier_wait( + tma_full_mbar + tma_stage, tma_full_parity + ) + _tcgen05.fence_after_thread_sync() + + for mma_k in cutlass.range_constexpr(BK // 16): + enable_d = iter_k > 0 or mma_k > 0 + _tcgen05.mma_f16( + d_tmem, a_desc, b_desc, idesc, enable_d, cta_group + ) + a_desc += 32 >> 4 + b_desc += 32 >> 4 + _tcgen05.commit( + tma_empty_mbar + tma_stage, multicast_mask, cta_group + ) + + tma_stage = (tma_stage + 1) % num_stages + if tma_stage == 0: + tma_full_parity ^= 1 + + _tcgen05.commit( + tmem_full_mbar + tmem_stage, multicast_mask, cta_group + ) + + tmem_stage = (tmem_stage + 1) % num_tmem_stages + if tmem_stage == 0: + tmem_empty_parity ^= 1 + + elif warp_id >= 4: + # Communication warps + # Keep epilogue in warps 0-3 and communication in warps 4-7. + # Swapping the warpgroups can hang due to warp scheduling. + tid_ = tid % 128 + local_M = output.shape[0] + + # Offset in the [M, N] GEMM result. + rank_start = self.rank * local_M + rank_end = min(rank_start + local_M, M) + + st_atom = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=128 + ) + + # Each thread issues one BF16x8 multimem reduction per vector. + vec_width = 8 + vec_cols = BN // vec_width + max_tile_rows = BM // num_ranks + vecs_per_tile = max_tile_rows * vec_cols + partial_vecs = cute.zipped_divide(partial_mc, (1, vec_width)) + output_vecs = cute.zipped_divide(output, (1, vec_width)) + + for tile_id in range(raw_bid, total_tiles, num_bids): + rs_bid_m = tile_id % grid_m + bid_n = tile_id // grid_m + + local_row_start = rs_bid_m * local_M // grid_m + local_row_end = (rs_bid_m + 1) * local_M // grid_m + global_row_start = rank_start + local_row_start + global_row_end = min(rank_start + local_row_end, M) + + if global_row_start < M: + # Map the current RS tile to the required GEMM tiles. + # Since an RS tile is smaller than a GEMM tile, each RS tile + # can overlap at most 2 GEMM tiles. + gemm_bid_m0 = global_row_start // BM + gemm_bid_m1 = max(global_row_end - 1, global_row_start) // BM + + # Poll local L2 with relaxed GPU-scope loads. The following + # multimem reduction reads every rank's L2 directly, so no + # acquire fence or L1 invalidation is needed. + if tid_ == 0: + # Poll the 1st GEMM tile. + flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m0 + arrivals = cute.arch.load( + flag_ptr, Int32, sem="relaxed", scope="gpu" + ) + while arrivals < num_ranks: + nanosleep(64) + arrivals = cute.arch.load( + flag_ptr, Int32, sem="relaxed", scope="gpu" + ) + elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0: + # Poll the 2nd GEMM tile with another warp if needed. + flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m1 + arrivals = cute.arch.load( + flag_ptr, Int32, sem="relaxed", scope="gpu" + ) + while arrivals < num_ranks: + nanosleep(64) + arrivals = cute.arch.load( + flag_ptr, Int32, sem="relaxed", scope="gpu" + ) + cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) + + # Issue multimem.ld_reduce before storing any result. + reduced_vecs = [] + for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128): + vec_idx = tid_ + vec_iter * 128 + + local_row = local_row_start + vec_idx // vec_cols + global_row = rank_start + local_row + col = bid_n * vec_cols + vec_idx % vec_cols + + reduced_vec = cute.make_rmem_tensor(vec_width, BFloat16) + if local_row < local_row_end and global_row < M: + tmp = multimem_ld_reduce_16B( + partial_vecs[None, (global_row, col)] + ) + reduced_vec.store(tmp.load()) + reduced_vecs.append(reduced_vec) + + # Store the result to local L2. + for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128): + vec_idx = tid_ + vec_iter * 128 + + local_row = local_row_start + vec_idx // vec_cols + global_row = rank_start + local_row + col = bid_n * vec_cols + vec_idx % vec_cols + + if local_row < local_row_end and global_row < M: + cute.copy( + st_atom, + reduced_vecs[vec_iter], + output_vecs[None, (local_row, col)], + ) + cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) + + # Release each GEMM tile consumed by this RS tile. The last + # consumer resets its producer flag for the next launch. + def release_gemm_tile( + gemm_bid_m, + tile_M, + logical_M, + rank_start, + rank_end, + local_M, + grid_m, + tile_N, + flags, + num_ranks, + ): + gemm_start = gemm_bid_m * tile_M + gemm_end = min(gemm_start + tile_M, logical_M) + local_start = max(gemm_start, rank_start) - rank_start + local_end = min(gemm_end, rank_end) - rank_start + + # Compute the number of consumers to identify the last + # arrival, which resets the producer flag. + first_rs_tile = ( + cute.ceil_div((local_start + 1) * grid_m, local_M) - 1 + ) + last_rs_tile = cute.ceil_div(local_end * grid_m, local_M) - 1 + num_consumers = last_rs_tile - first_rs_tile + 1 + + flag_ptr = flags + tile_N * grid_m + gemm_bid_m + old_count = cute.arch.atomic_add( + flag_ptr, Int32(1), sem="relaxed", scope="gpu" + ) + if old_count == num_ranks + num_consumers - 1: + cute.arch.store( + flag_ptr, Int32(0), sem="relaxed", scope="gpu" + ) + + if tid_ == 0: + # Arrive at the 1st GEMM tile. + release_gemm_tile( + gemm_bid_m0, + BM, + M, + rank_start, + rank_end, + local_M, + grid_m, + bid_n, + flags_uc_ptr, + num_ranks, + ) + elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0: + # Arrive at the 2nd GEMM tile if necessary. + release_gemm_tile( + gemm_bid_m1, + BM, + M, + rank_start, + rank_end, + local_M, + grid_m, + bid_n, + flags_uc_ptr, + num_ranks, + ) + + # Exit barrier. GPU scope is sufficient because the kernel writes + # local global memory and only needs to flush it to local L2. + cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) + if tid_ == 0: + exit_flag = total_tiles + raw_bid + utils.distributed.multimem_red_add1( + flags_mc_ptr + exit_flag, order="release", scope="gpu" + ) + utils.distributed.spin_lock_atom_cas_acquire_wait( + flags_uc_ptr + exit_flag, + expected_val=num_ranks, + reset_val=0, + scope="gpu", + ) + + else: + # Epilogue warps + warp_id_ = warp_id % 4 + tid_ = tid % 128 + + peer_flag_bases = cute.make_rmem_tensor(num_ranks, Int64) + if tid_ == 0: + for rank in cutlass.range_constexpr(num_ranks): + peer_flag_bases[rank] = cute.arch.load( + (peer_flags.iterator + rank).llvm_ptr, + Int64, + ) + + if warp_id_ == 0: + _tcgen05.alloc(taddr, cta_group) + cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32) + + WIDTH = cutlass.const_expr(16) + partial_vecs = cute.zipped_divide(partial_uc, (1, WIDTH)) + + bf16x16_atom = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), + BFloat16, + num_bits_per_copy=256, + l1c_evict_priority=cute.nvgpu.CacheEvictionPriority.NO_ALLOCATE, + ) + + tmem_stage = 0 + parity = 0 + + if cutlass.const_expr(is_2cta): + tmem_empty_mbar_ = to_cta0_smem(tmem_empty_mbar) + else: + tmem_empty_mbar_ = tmem_empty_mbar + + for bid in range(raw_bid, total_tiles, num_bids): + bid_m = bid % grid_m + bid_n = bid // grid_m + + if warp_id_ == 0: + cute.arch.mbarrier_wait(tmem_full_mbar + tmem_stage, parity) + cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + for i in cutlass.range_constexpr(BN // WIDTH): + tcol = tmem_stage * BN + i * WIDTH + regs = _tcgen05.ld(warp_id_ * 32, tcol, "32x32b", WIDTH) + _tcgen05.wait_ld() + + if cutlass.const_expr(i == BN // WIDTH - 1): + _tcgen05.fence_before_thread_sync() + mbarrier.arrive(tmem_empty_mbar_ + tmem_stage, "cluster") + + tmp = cute.make_rmem_tensor(WIDTH, BFloat16) + tmp.store(regs.to(BFloat16)) + + global_row = bid_m * BM + tid_ + if global_row < M: + coord = (global_row, bid_n * (BN // WIDTH) + i) + cute.copy(bf16x16_atom, tmp, partial_vecs[None, coord]) + + cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) + + # Signal GEMM completion. GPU scope is sufficient because the + # partial output only needs to be flushed to local L2. + if tid_ == 0 and bid_m * BM < M: + gemm_start = bid_m * BM + gemm_end = min(gemm_start + BM, M) + first_owner = gemm_start // output.shape[0] + last_owner = (gemm_end - 1) // output.shape[0] + + # signal to all consuming ranks (can be more than 1) + for rank in cutlass.range_constexpr(num_ranks): + if first_owner <= rank and rank <= last_owner: + ptr = cute.make_ptr( + Int32, + peer_flag_bases[rank], + cute.AddressSpace.gmem, + assumed_align=16, + ) + ptr += bid_m + bid_n * grid_m + utils.distributed.red_add1( + ptr, order="release", scope="gpu" + ) + + tmem_stage = (tmem_stage + 1) % num_tmem_stages + if tmem_stage == 0: + parity ^= 1 + + if cutlass.const_expr(is_2cta): + cute.arch.cluster_arrive_relaxed() + cute.arch.cluster_wait() + else: + cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128) + if warp_id_ == 0: + _tcgen05.dealloc(cta_group) + + @cache + @staticmethod + def compile(rank: int, num_ranks: int, BN: int, cta_group: int): + M = cute.sym_int() + padded_M = cute.sym_int() + N = cute.sym_int() + K = cute.sym_int() + local_M = cute.sym_int() + num_flags = cute.sym_int() + + A = make_fake_tensor( + BFloat16, (M, K), (cute.sym_int64(divisibility=8), 1), assumed_align=16 + ) + B = make_fake_tensor( + BFloat16, (N, K), (cute.sym_int64(divisibility=8), 1), assumed_align=16 + ) + partial = make_fake_tensor( + BFloat16, + (padded_M, N), + (cute.sym_int64(divisibility=16), 1), + assumed_align=32, + ) + partial_mc_ptr = nullptr(BFloat16, cute.AddressSpace.gmem, assumed_align=32) + output = make_fake_tensor( + BFloat16, + (local_M, N), + (cute.sym_int64(divisibility=16), 1), + assumed_align=32, + ) + flags = make_fake_tensor(Int32, (num_flags,), (1,), assumed_align=16) + flags_mc_ptr = nullptr(Int32, cute.AddressSpace.gmem, assumed_align=16) + peer_flag_ptr = nullptr(Int64, cute.AddressSpace.gmem, assumed_align=8) + + stream = make_fake_stream(use_tvm_ffi_env_stream=True) + kernel = Sm100GemmRsBF16(rank, num_ranks, BN, cta_group) + return cute.compile( + kernel, + A, + B, + partial, + partial_mc_ptr, + output, + flags, + flags_mc_ptr, + peer_flag_ptr, + 128, + stream, + options="--enable-tvm-ffi", + ) + + +class GemmRS: + """Own the max-sized symmetric workspace for Kimi-K3 GEMM-RS launches. + + All TP ranks must belong to one NVLink domain for multimem instructions. + """ + + def __init__(self, *, max_M: int, N: int) -> None: + tp_group = get_tp_group() + group = tp_group.device_group + rank = tp_group.rank_in_group + world_size = tp_group.world_size + device = torch.device("cuda", torch.accelerator.current_device_index()) + + assert 1 < world_size <= 16 + assert 128 % world_size == 0 + assert max_M >= 128 and N % 128 == 0 + + max_M = (max_M + world_size - 1) // world_size * world_size + self.rank = rank + self.world_size = world_size + self.max_M = max_M + self.N = N + self.device = device + + self.partial = symm_mem.empty((max_M, N), dtype=torch.bfloat16, device=device) + self.partial_handle = symm_mem.rendezvous(self.partial, group) + self.partial_mc_ptr = make_ptr( + BFloat16, + self.partial_handle.multicast_ptr, + cute.AddressSpace.gmem, + assumed_align=32, + ) + + grid_m = (max_M + 127) // 128 + cta_group = 2 if max_M >= 1024 or grid_m % 2 == 0 else 1 + grid_m = (grid_m + cta_group - 1) // cta_group * cta_group + self.num_sms = torch.cuda.get_device_properties(device).multi_processor_count + max_flags = grid_m * (N // 128) + self.num_sms + self.flags = symm_mem.empty(max_flags, dtype=torch.int32, device=device) + self.flags_handle = symm_mem.rendezvous(self.flags, group) + self.flags.zero_() + self.flags_mc_ptr = make_ptr( + Int32, + self.flags_handle.multicast_ptr, + cute.AddressSpace.gmem, + assumed_align=16, + ) + self.peer_flag_ptr = make_ptr( + Int64, + self.flags_handle.buffer_ptrs_dev, + cute.AddressSpace.gmem, + assumed_align=8, + ) + + assert self.partial_handle.multicast_ptr != 0 + assert self.flags_handle.multicast_ptr != 0 + torch.accelerator.synchronize(device) + tp_group.barrier() + + def can_run(self, linear: LinearBase) -> bool: + # Validate projection-invariant requirements once during model init. + # only supports BF16 for now + if not isinstance(linear.quant_method, UnquantizedLinearMethod): + return False + w = linear.weight + if w.ndim != 2: + return False + K = w.shape[1] + return ( + w.shape == (self.N, K) + and K % 64 == 0 + and w.dtype == torch.bfloat16 + and w.device == self.device + and w.is_contiguous() + ) + + def should_run(self, x: torch.Tensor) -> bool: + # Small-M shapes are supported but faster on the existing LL path. + return x.shape[0] >= 128 + + def __call__(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: + assert x.ndim == 2 + M, K = x.shape + assert 0 < M <= self.max_M + assert w.shape == (self.N, K) and K % 64 == 0 + assert w.dtype == torch.bfloat16 + assert w.device == self.device + assert w.is_contiguous() + assert x.dtype == torch.bfloat16 + assert x.device == self.device + assert x.is_contiguous() + N = w.shape[0] + padded_M = (M + self.world_size - 1) // self.world_size + padded_M *= self.world_size + local_M = padded_M // self.world_size + + grid_m = (M + 127) // 128 + # Avoid padding small odd grids; 2-CTA wins consistently for M >= 1024. + cta_group = 2 if M >= 1024 or grid_m % 2 == 0 else 1 + grid_m = (grid_m + cta_group - 1) // cta_group * cta_group + BN = 256 if M * K >= 24 * 1024 * 1024 else 128 + assert N % BN == 0 + + num_tiles = grid_m * (N // BN) + num_ctas = min(num_tiles, self.num_sms) + num_ctas = num_ctas // cta_group * cta_group + assert self.flags.numel() >= num_tiles + num_ctas + + output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device) + compiled = Sm100GemmRsBF16.compile( + self.rank, + self.world_size, + BN, + cta_group, + ) + compiled( + x, + w, + self.partial[:padded_M], + self.partial_mc_ptr, + output, + self.flags, + self.flags_mc_ptr, + self.peer_flag_ptr, + num_ctas, + ) + return output + + +_gemm_rs: GemmRS | None = None + + +def init_gemm_rs(max_M: int, N: int) -> None: + """Collectively initialize the process-wide GEMM-RS state.""" + global _gemm_rs + if _gemm_rs is not None: + assert _gemm_rs.max_M >= max_M and _gemm_rs.N == N + return + _gemm_rs = GemmRS(max_M=max_M, N=N) + + +def get_gemm_rs() -> GemmRS: + assert _gemm_rs is not None, "GEMM-RS is not initialized" + return _gemm_rs