From 2785c72a1497dcd4db53d7c3bc557e379a518e0c Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Fri, 21 Aug 2026 11:44:15 +0800 Subject: [PATCH] [Kimi-K3] Extend GEMM-RS to GEMM-AR (#53053) Signed-off-by: Thien Tran Co-authored-by: OpenAI Codex Co-authored-by: wzhao18 --- .buildkite/test_areas/distributed.yaml | 13 +- ..._rs.py => benchmark_kimi_k3_gemm_rs_ar.py} | 192 +++++++++++------- ..._gemm_rs.py => test_kimi_k3_gemm_rs_ar.py} | 175 +++++++++++----- vllm/envs.py | 4 + vllm/model_executor/warmup/kernel_warmup.py | 15 ++ vllm/models/kimi_k3/nvidia/kda.py | 26 +-- vllm/models/kimi_k3/nvidia/mla.py | 26 +-- vllm/models/kimi_k3/nvidia/model.py | 96 +++++---- .../cute_dsl/{gemm_rs.py => gemm_rs_ar.py} | 180 ++++++++++++---- 9 files changed, 488 insertions(+), 239 deletions(-) rename benchmarks/kernels/{benchmark_kimi_k3_gemm_rs.py => benchmark_kimi_k3_gemm_rs_ar.py} (70%) rename tests/kernels/{test_kimi_k3_gemm_rs.py => test_kimi_k3_gemm_rs_ar.py} (63%) rename vllm/models/kimi_k3/nvidia/ops/cute_dsl/{gemm_rs.py => gemm_rs_ar.py} (82%) diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index ad8e05343b..ed23e9ec4c 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -244,11 +244,20 @@ steps: commands: - pytest -v -s tests/distributed/test_context_parallel.py - pytest -v -s tests/distributed/test_nccl_symm_mem.py - - pytest -v -s tests/kernels/test_kimi_k3_gemm_rs.py - pytest -v -s tests/v1/distributed/test_dbo.py - pytest -v -s tests/distributed/test_mnnvl_alltoall.py - +- label: ":nvidia: (B200) Kimi-K3 GEMM-RS/AR" + key: kimi-k3-gemm-rs-ar-2xb200 + timeout_in_minutes: 20 + device: b200-k8s + working_dir: "/vllm-workspace/" + num_devices: 2 + source_file_dependencies: + - vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs_ar.py + - tests/kernels/test_kimi_k3_gemm_rs_ar.py + commands: + - pytest -v -s tests/kernels/test_kimi_k3_gemm_rs_ar.py - label: ":nvidia: (L4) Distributed 2-Node" key: 2-node-test-4-gpus diff --git a/benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py b/benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py similarity index 70% rename from benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py rename to benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py index cb04bf86e3..e4daea26bb 100644 --- a/benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py +++ b/benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Benchmark the SM100 Kimi-K3 GEMM-RS kernel. +"""Benchmark the SM100 Kimi-K3 GEMM-RS/AR kernel. All ranks must belong to one NVLink domain. For example, run a TP8 sweep with: torchrun --nproc-per-node=8 \ - benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py + benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py """ import argparse @@ -26,7 +26,7 @@ from vllm.distributed.parallel_state import ( init_distributed_environment, initialize_model_parallel, ) -from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS +from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr # Shared-expert down-proj and attention O-proj. _KIMI_K3_PROJECTION_K = (6144, 12288) @@ -41,6 +41,12 @@ class Candidate: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", + choices=("rs", "ar"), + default="rs", + help="Collective mode to benchmark.", + ) parser.add_argument( "--m", type=int, @@ -138,7 +144,8 @@ def valid_rows(M: int, local_M: int, rank: int) -> int: def benchmark_shape( - gemm_rs: GemmRS, + gemm_rs_ar: GemmRsAr, + mode: str, M: int, N: int, K: int, @@ -148,7 +155,8 @@ def benchmark_shape( device_group: dist.ProcessGroup, cpu_group: dist.ProcessGroup, device_barrier: Callable[[], None], -) -> dict[str, float | int]: +) -> dict[str, float | int | str]: + all_reduce = mode == "ar" world_size = dist.get_world_size(device_group) rank = dist.get_rank(device_group) device = torch.device("cuda", torch.accelerator.current_device_index()) @@ -169,17 +177,19 @@ def benchmark_shape( partial = torch.empty((padded_M, N), dtype=torch.bfloat16, device=device) symm_partial = symm_mem.empty((padded_M, N), dtype=torch.bfloat16, device=device) symm_partial_handle = symm_mem.rendezvous(symm_partial, device_group) - rs_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)] - symm_rs_inputs = [] - symm_rs_handles = [] + collective_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)] + symm_collective_inputs = [] + symm_collective_handles = [] for _ in range(num_workspaces): - rs_input = symm_mem.empty( + collective_input = symm_mem.empty( (padded_M, N), dtype=torch.bfloat16, device=device, ) - symm_rs_inputs.append(rs_input) - symm_rs_handles.append(symm_mem.rendezvous(rs_input, device_group)) + symm_collective_inputs.append(collective_input) + symm_collective_handles.append( + symm_mem.rendezvous(collective_input, device_group) + ) torch_output = torch.empty((local_M, N), dtype=torch.bfloat16, device=device) symm_output = torch.empty_like(torch_output) gemm_output = torch.empty((M, N), dtype=torch.bfloat16, device=device) @@ -188,21 +198,27 @@ def benchmark_shape( partial[M:].zero_() symm_partial[M:].zero_() - def make_torch_ring_ll_gemm_rs( + def make_torch_ring_ll_gemm_collective( x: torch.Tensor, weight: torch.Tensor ) -> Callable[[], torch.Tensor]: def run() -> torch.Tensor: torch.mm(x, weight.T, out=partial[:M]) + if all_reduce: + dist.all_reduce(partial, group=device_group) + return partial[:M] dist.reduce_scatter_single(torch_output, partial, group=device_group) return torch_output return run - def make_torch_ldmc_gemm_rs( + def make_torch_ldmc_gemm_collective( x: torch.Tensor, weight: torch.Tensor ) -> Callable[[], torch.Tensor]: def run() -> torch.Tensor: torch.mm(x, weight.T, out=symm_partial[:M]) + if all_reduce: + dist.all_reduce(symm_partial, group=device_group) + return symm_partial[:M] dist.reduce_scatter_single( symm_output, symm_partial, @@ -212,11 +228,11 @@ def benchmark_shape( return run - def make_fused_gemm_rs( + def make_fused_gemm_collective( x: torch.Tensor, weight: torch.Tensor ) -> Callable[[], torch.Tensor]: def run() -> torch.Tensor: - return gemm_rs(x, weight) + return gemm_rs_ar(x, weight) return run @@ -229,18 +245,30 @@ def benchmark_shape( return run - def make_ring_ll_rs(rs_input: torch.Tensor) -> Callable[[], torch.Tensor]: + def make_ring_ll_collective( + collective_input: torch.Tensor, + ) -> Callable[[], torch.Tensor]: def run() -> torch.Tensor: - dist.reduce_scatter_single(torch_output, rs_input, group=device_group) + if all_reduce: + dist.all_reduce(collective_input, group=device_group) + return collective_input + dist.reduce_scatter_single( + torch_output, collective_input, group=device_group + ) return torch_output return run - def make_ldmc_rs( - rs_input: torch.Tensor, + def make_ldmc_collective( + collective_input: torch.Tensor, ) -> Callable[[], torch.Tensor]: def run() -> torch.Tensor: - dist.reduce_scatter_single(symm_output, rs_input, group=device_group) + if all_reduce: + dist.all_reduce(collective_input, group=device_group) + return collective_input + dist.reduce_scatter_single( + symm_output, collective_input, group=device_group + ) return symm_output return run @@ -248,15 +276,15 @@ def benchmark_shape( candidates = ( Candidate( "ring_ll_us", - [make_torch_ring_ll_gemm_rs(x, w) for x, w in zip(inputs, weights)], + [make_torch_ring_ll_gemm_collective(x, w) for x, w in zip(inputs, weights)], ), Candidate( "ldmc_us", - [make_torch_ldmc_gemm_rs(x, w) for x, w in zip(inputs, weights)], + [make_torch_ldmc_gemm_collective(x, w) for x, w in zip(inputs, weights)], ), Candidate( - "gemm_rs_us", - [make_fused_gemm_rs(x, w) for x, w in zip(inputs, weights)], + "gemm_rs_ar_us", + [make_fused_gemm_collective(x, w) for x, w in zip(inputs, weights)], ), Candidate( "torch_gemm_us", @@ -264,19 +292,19 @@ def benchmark_shape( check_correctness=False, ), Candidate( - "ring_ll_rs_us", - [make_ring_ll_rs(x) for x in rs_inputs], + "ring_ll_collective_us", + [make_ring_ll_collective(x) for x in collective_inputs], check_correctness=False, ), Candidate( - "ldmc_rs_us", - [make_ldmc_rs(x) for x in symm_rs_inputs], + "ldmc_collective_us", + [make_ldmc_collective(x) for x in symm_collective_inputs], check_correctness=False, ), ) expected = candidates[0].runs[0]() - rows = valid_rows(M, local_M, rank) + rows = M if all_reduce else valid_rows(M, local_M, rank) for candidate in candidates[1:]: if not candidate.check_correctness: continue @@ -290,7 +318,7 @@ def benchmark_shape( ) candidate_graphs = {} - graph_keepalive: list[object] = [symm_partial_handle, *symm_rs_handles] + graph_keepalive: list[object] = [symm_partial_handle, *symm_collective_handles] for candidate in candidates: stream = torch.cuda.Stream() bundles = [capture_graph(run, stream, cpu_group) for run in candidate.runs] @@ -306,70 +334,71 @@ def benchmark_shape( device_barrier, ) - best_nccl_rs_us = min(times["ring_ll_rs_us"], times["ldmc_rs_us"]) + best_nccl_us = min(times["ring_ll_collective_us"], times["ldmc_collective_us"]) return { + "mode": mode.upper(), "M": M, "N": N, "K": K, **times, - "best_nccl_rs_us": best_nccl_rs_us, - "speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_us"], - "speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_us"], + "best_nccl_us": best_nccl_us, + "speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_ar_us"], + "speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_ar_us"], } -def print_results(results: list[dict[str, float | int]]) -> None: +def print_results(results: list[dict[str, float | int | str]]) -> None: results_df = pd.DataFrame(results) - end_to_end = results_df[ - [ - "M", - "N", - "K", - "ring_ll_us", - "ldmc_us", - "gemm_rs_us", - "speedup_vs_ring_ll", - "speedup_vs_ldmc", + collective = str(results_df["mode"].iloc[0]) + end_to_end = ( + results_df[ + [ + "M", + "N", + "K", + "ring_ll_us", + "ldmc_us", + "gemm_rs_ar_us", + "speedup_vs_ring_ll", + "speedup_vs_ldmc", + ] ] - ].rename( - columns={ - "ring_ll_us": "Torch GEMM + NCCL RS (RING_LL) (us)", - "ldmc_us": "Torch GEMM + NCCL RS (LDMC) (us)", - "gemm_rs_us": "GEMM-RS (us)", - "speedup_vs_ring_ll": "Speedup vs RING_LL", - "speedup_vs_ldmc": "Speedup vs LDMC", - } + .rename( + columns={ + "ring_ll_us": f"Torch GEMM + NCCL {collective} (RING_LL) (us)", + "ldmc_us": f"Torch GEMM + NCCL {collective} (LDMC) (us)", + "gemm_rs_ar_us": f"GEMM-{collective} (us)", + "speedup_vs_ring_ll": "Speedup vs RING_LL", + "speedup_vs_ldmc": "Speedup vs LDMC", + } + ) + .round(3) ) - end_to_end = end_to_end.round( - { - "Torch GEMM + NCCL RS (RING_LL) (us)": 2, - "Torch GEMM + NCCL RS (LDMC) (us)": 2, - "GEMM-RS (us)": 2, - "Speedup vs RING_LL": 3, - "Speedup vs LDMC": 3, - } + components = ( + results_df[["M", "N", "K", "torch_gemm_us", "best_nccl_us", "gemm_rs_ar_us"]] + .rename( + columns={ + "torch_gemm_us": "Torch GEMM (us)", + "best_nccl_us": f"NCCL {collective} (best) (us)", + "gemm_rs_ar_us": f"GEMM-{collective} (us)", + } + ) + .round(2) ) - components = results_df[ - ["M", "N", "K", "torch_gemm_us", "best_nccl_rs_us", "gemm_rs_us"] - ].rename( - columns={ - "torch_gemm_us": "Torch GEMM (us)", - "best_nccl_rs_us": "NCCL RS (best) (us)", - "gemm_rs_us": "GEMM-RS (us)", - } - ) - components = components.round(2) - print("### End-to-end latency") + print(f"### {collective} end-to-end latency") print(end_to_end.to_markdown(index=False)) - print("\n### Component latency") + print(f"\n### {collective} component latency") print(components.to_markdown(index=False)) - print("\nNCCL RS (best) is the faster of RING_LL and LDMC for each shape.") + print( + f"\nNCCL {collective} (best) is the faster of RING_LL and LDMC " + "for each shape.\n" + ) def main() -> None: args = parse_args() - assert args.m and min(args.m) >= 128 + assert args.m and min(args.m) > 0 assert args.n % 256 == 0 assert args.num_workspaces > 0 assert args.warmup_replays >= 0 @@ -401,12 +430,19 @@ def main() -> None: sync_output = torch.empty_like(sync_input) def device_barrier() -> None: + # Order the timed launch after a device-side rank rendezvous without + # including the rendezvous itself in the measured event interval. pynccl_comm.all_reduce(sync_input, sync_output) - gemm_rs = GemmRS(max_M=max(args.m), N=args.n) + gemm_rs_ar = GemmRsAr( + max_M=max(args.m), + N=args.n, + all_reduce=args.mode == "ar", + ) results = [ benchmark_shape( - gemm_rs, + gemm_rs_ar, + args.mode, M, args.n, K, @@ -420,12 +456,12 @@ def main() -> None: for K in K_values for M in args.m ] + del gemm_rs_ar if tp_group.rank_in_group == 0: print_results(results) dist.barrier(group=tp_group.cpu_group) - del gemm_rs cleanup_dist_env_and_memory() diff --git a/tests/kernels/test_kimi_k3_gemm_rs.py b/tests/kernels/test_kimi_k3_gemm_rs_ar.py similarity index 63% rename from tests/kernels/test_kimi_k3_gemm_rs.py rename to tests/kernels/test_kimi_k3_gemm_rs_ar.py index d4d6347875..7a887c501f 100644 --- a/tests/kernels/test_kimi_k3_gemm_rs.py +++ b/tests/kernels/test_kimi_k3_gemm_rs_ar.py @@ -31,6 +31,7 @@ def _reference( weight: torch.Tensor, world_size: int, group: dist.ProcessGroup, + all_reduce: bool, ) -> torch.Tensor: M = x.shape[0] padded_M = (M + world_size - 1) // world_size * world_size @@ -43,6 +44,10 @@ def _reference( if padded_M > M: partial[M:].zero_() + if all_reduce: + dist.all_reduce(partial, group=group) + return partial[:M] + output = torch.empty( (padded_M // world_size, weight.shape[0]), dtype=x.dtype, @@ -57,7 +62,11 @@ def _assert_valid_rows_close( expected: torch.Tensor, M: int, rank: int, + all_reduce: bool, ) -> None: + if all_reduce: + torch.testing.assert_close(actual, expected, rtol=5e-2, atol=4.0) + return rank_start = rank * actual.shape[0] valid_rows = min(max(M - rank_start, 0), actual.shape[0]) torch.testing.assert_close( @@ -68,11 +77,110 @@ def _assert_valid_rows_close( ) +def _run_mode( + *, + all_reduce: bool, + device: torch.device, + group: dist.ProcessGroup, + rank: int, + world_size: int, + weights: dict[int, torch.Tensor], +) -> None: + # cute_dsl is unavailable off CUDA, so import it only inside the GPU worker. + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr + + gemm_rs_ar = GemmRsAr( + max_M=max(M for M, _ in _SHAPES), + N=_N, + all_reduce=all_reduce, + ) + input_generator = torch.Generator(device=device) + + # Alternating shapes exercise producer-flag reuse across different grids, + # CTA-group choices, and both BN=128 and BN=256 dispatches. + for M, K in (*_SHAPES, *_SHAPES[::-1]): + input_generator.manual_seed(2000 + M + K) + x = torch.randn( + M, + K, + dtype=torch.bfloat16, + device=device, + generator=input_generator, + ) + expected = _reference(x, weights[K], world_size, group, all_reduce) + actual = gemm_rs_ar(x, weights[K]) + torch.accelerator.synchronize(device) + _assert_valid_rows_close(actual, expected, M, rank, all_reduce) + + if all_reduce: + # AR outputs must remain valid after the symmetric workspace is reused. + lifetime_M, lifetime_K = 257, 768 + input_generator.manual_seed(2500) + lifetime_x = torch.randn( + lifetime_M, + lifetime_K, + dtype=torch.bfloat16, + device=device, + generator=input_generator, + ) + first_output = gemm_rs_ar(lifetime_x, weights[lifetime_K]) + first_snapshot = first_output.clone() + second_output = gemm_rs_ar(-lifetime_x, weights[lifetime_K]) + torch.accelerator.synchronize(device) + assert first_output.data_ptr() != second_output.data_ptr() + torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0) + + graph_M, graph_K = 1025, 4224 + input_generator.manual_seed(3000) + graph_x = torch.randn( + graph_M, + graph_K, + dtype=torch.bfloat16, + device=device, + generator=input_generator, + ) + graph_expected = _reference( + graph_x, + weights[graph_K], + world_size, + group, + all_reduce, + ) + graph_output = torch.empty_like(graph_expected) + + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(capture_stream): + for _ in range(3): + graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K])) + capture_stream.synchronize() + dist.barrier(group=group) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K])) + torch.cuda.current_stream().wait_stream(capture_stream) + dist.barrier(group=group) + + for _ in range(3): + graph.replay() + torch.accelerator.synchronize(device) + _assert_valid_rows_close( + graph_output, + graph_expected, + graph_M, + rank, + all_reduce, + ) + + dist.barrier(group=group) + del gemm_rs_ar, capture_stream, graph, graph_output + + def _worker(local_rank: int, world_size: int, master_port: int) -> None: # This module pulls in cute_dsl, which is unavailable off CUDA, so importing # it at module level would fail collection. Import it where it is used, as # `kda.py` does. - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS device = torch.device("cuda", local_rank) torch.accelerator.set_device_index(device) @@ -92,10 +200,8 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None: group = dist.group.WORLD rank = dist.get_rank(group) - gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N) weight_generator = torch.Generator(device=device) - input_generator = torch.Generator(device=device) weights = {} for K in {K for _, K in _SHAPES}: weight_generator.manual_seed(1000 + rank * 10 + K) @@ -107,68 +213,29 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None: generator=weight_generator, ) - # Alternating shapes exercise producer-flag reuse across different grids, - # CTA-group choices, and both BN=128 and BN=256 dispatches. - for M, K in (*_SHAPES, *_SHAPES[::-1]): - input_generator.manual_seed(2000 + M + K) - x = torch.randn( - M, - K, - dtype=torch.bfloat16, + # Production binds one mode per worker; exercise both mode-bound instances + # sequentially in this test without implying that both are initialized. + for all_reduce in (False, True): + _run_mode( + all_reduce=all_reduce, device=device, - generator=input_generator, + group=group, + rank=rank, + world_size=world_size, + weights=weights, ) - expected = _reference(x, weights[K], world_size, group) - actual = gemm_rs(x, weights[K]) - torch.accelerator.synchronize(device) - _assert_valid_rows_close(actual, expected, M, rank) - - # Exercise GEMM-RS under CUDA graph capture and replay. - graph_M, graph_K = 1025, 4224 - input_generator.manual_seed(3000) - graph_x = torch.randn( - graph_M, - graph_K, - dtype=torch.bfloat16, - device=device, - generator=input_generator, - ) - graph_expected = _reference(graph_x, weights[graph_K], world_size, group) - graph_output = torch.empty_like(graph_expected) - - capture_stream = torch.cuda.Stream() - capture_stream.wait_stream(torch.cuda.current_stream()) - with torch.cuda.stream(capture_stream): - for _ in range(3): - graph_output.copy_(gemm_rs(graph_x, weights[graph_K])) - capture_stream.synchronize() - dist.barrier(group=group) - - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph, stream=capture_stream): - graph_output.copy_(gemm_rs(graph_x, weights[graph_K])) - torch.cuda.current_stream().wait_stream(capture_stream) - dist.barrier(group=group) - - for _ in range(3): - graph.replay() - torch.accelerator.synchronize(device) - _assert_valid_rows_close(graph_output, graph_expected, graph_M, rank) - - dist.barrier(group=group) - del gemm_rs cleanup_dist_env_and_memory() @pytest.mark.distributed(num_gpus=2) @pytest.mark.skipif( not current_platform.is_device_capability_family(100), - reason="Kimi-K3 GEMM-RS requires SM100", + reason="Kimi-K3 GEMM-RS/AR requires SM100", ) -def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None: +def test_kimi_k3_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch) -> None: world_size = 2 if torch.accelerator.device_count() < world_size: - pytest.skip("GEMM-RS requires two GPUs") + pytest.skip("GEMM-RS/AR requires two GPUs") monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1") monkeypatch.setenv("NCCL_NVLS_ENABLE", "1") diff --git a/vllm/envs.py b/vllm/envs.py index 90398ee4e0..e410dd7f54 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -210,6 +210,7 @@ if TYPE_CHECKING: VLLM_MOE_SKIP_PADDING: bool = True VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False + VLLM_KIMI_K3_GEMM_AR: bool = True VLLM_KIMI_K3_GEMM_RS: bool = False VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_USE_FLASHINFER_MOE_INT4: bool = False @@ -1608,6 +1609,9 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool( int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0")) ), + # Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel + # attention projections. All TP ranks must belong to one NVLink domain. + "VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))), # Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel # row-parallel projections. All TP ranks must belong to one NVLink domain. "VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))), diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index e25a60c844..e20922d4e0 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -6,6 +6,7 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to happen during model execution. """ +import sys import time from typing import TYPE_CHECKING @@ -95,6 +96,18 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None: ) +def _warmup_kimi_k3_gemm_rs_ar() -> None: + # Kimi-K3 model construction imports this module only when GEMM-RS/AR is + # enabled and initializes its singleton before kernel_warmup runs. Avoid + # importing it here so other models do not compile the RS/AR variants. + module = sys.modules.get("vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar") + if module is None: + return + compiled = module.warmup_gemm_rs_ar() + if compiled: + logger.info_once("Warmed up %d Kimi-K3 GEMM-RS/AR variants.", compiled) + + def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): from vllm.model_executor.warmup.minimax_m3_msa_warmup import ( minimax_m3_msa_warmup, @@ -146,6 +159,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): if current_platform.has_device_capability(90): _warmup_ll_bf16_router_gemm(worker.get_model()) + _warmup_kimi_k3_gemm_rs_ar() + if worker.vllm_config.kernel_config.enable_cutedsl_warmup: # TODO(roberto): Remove after registered CuTeDSL warmups are migrated # to the shared JIT warmup infrastructure. diff --git a/vllm/models/kimi_k3/nvidia/kda.py b/vllm/models/kimi_k3/nvidia/kda.py index e680180f26..fe49610094 100644 --- a/vllm/models/kimi_k3/nvidia/kda.py +++ b/vllm/models/kimi_k3/nvidia/kda.py @@ -319,7 +319,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): config: KimiLinearConfig, vllm_config: VllmConfig, prefix: str = "", - run_gemm_rs: bool = False, + run_gemm_rs_ar: bool = False, ) -> None: super().__init__(config, vllm_config, prefix) self.use_recoverssm = self.cache_config.use_kda_recoverssm @@ -491,14 +491,18 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): quant_config=self.quant_config, prefix=f"{prefix}.o_proj", ) - self.run_gemm_rs = run_gemm_rs - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + self.gemm_rs_ar = None + if run_gemm_rs_ar: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import ( + get_gemm_rs_ar, + ) - self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj) - if not self.run_gemm_rs: + gemm_rs_ar = get_gemm_rs_ar() + if gemm_rs_ar.can_run(self.o_proj): + self.gemm_rs_ar = gemm_rs_ar + else: logger.warning_once( - "GEMM-RS is disabled for %s due to an incompatible projection.", + "GEMM-RS/AR is disabled for %s due to an incompatible projection.", prefix, ) compilation_config = vllm_config.compilation_config @@ -541,12 +545,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): core_attn_out=core_attn_out, ) core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)") - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs - - gemm_rs = get_gemm_rs() - if gemm_rs.should_run(core_attn_out): - return gemm_rs(core_attn_out, self.o_proj.weight) + if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(core_attn_out): + return self.gemm_rs_ar(core_attn_out, self.o_proj.weight) return self.o_proj(core_attn_out)[0] @eager_break_during_capture diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 07f51a9250..9eb89028bb 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -134,7 +134,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): aux_stream: torch.cuda.Stream | None = None, use_rope: bool = False, non_causal_multi_token_decode: bool = False, - run_gemm_rs: bool = False, + run_gemm_rs_ar: bool = False, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -271,14 +271,18 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): quant_config=quant_config, prefix=f"{prefix}.o_proj", ) - self.run_gemm_rs = run_gemm_rs - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + self.gemm_rs_ar = None + if run_gemm_rs_ar: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import ( + get_gemm_rs_ar, + ) - self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj) - if not self.run_gemm_rs: + gemm_rs_ar = get_gemm_rs_ar() + if gemm_rs_ar.can_run(self.o_proj): + self.gemm_rs_ar = gemm_rs_ar + else: logger.warning_once( - "GEMM-RS is disabled for %s due to an incompatible projection.", + "GEMM-RS/AR is disabled for %s due to an incompatible projection.", prefix, ) @@ -549,12 +553,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): if gate is not None: attn_out = _gate_sigmoid_mul(attn_out, gate) - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs - - gemm_rs = get_gemm_rs() - if gemm_rs.should_run(attn_out): - return gemm_rs(attn_out, self.o_proj.weight) + if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(attn_out): + return self.gemm_rs_ar(attn_out, self.o_proj.weight) return self.o_proj(attn_out)[0] diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index d2ceafd514..15ac51cd41 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -154,15 +154,18 @@ def shard_sequence_parallel_mlp( ) -def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool: - if not envs.VLLM_KIMI_K3_GEMM_RS: +def maybe_init_gemm_rs_ar(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool: + # Both feature flags may be enabled; the worker's static SP topology binds + # its singleton to exactly one mode. + all_reduce = not use_sequence_parallel + mode = "GEMM-AR" if all_reduce else "GEMM-RS" + enabled = envs.VLLM_KIMI_K3_GEMM_AR if all_reduce else envs.VLLM_KIMI_K3_GEMM_RS + if not enabled: return False parallel_config = vllm_config.parallel_config tp_size = parallel_config.tensor_parallel_size - if not use_sequence_parallel: - reason = "sequence parallelism is disabled" - elif parallel_config.use_ubatching: + if parallel_config.use_ubatching: reason = "ubatching is enabled" elif vllm_config.model_config.dtype != torch.bfloat16: reason = "the model dtype is not BF16" @@ -178,17 +181,32 @@ def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> reason = None if reason is not None: - logger.warning_once("GEMM-RS was requested but is disabled because %s.", reason) + logger.warning_once("%s is disabled because %s.", mode, reason) return False - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import init_gemm_rs + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import init_gemm_rs_ar config = vllm_config.model_config.hf_text_config - init_gemm_rs( - max_M=vllm_config.scheduler_config.max_num_batched_tokens, - N=config.hidden_size, - ) - logger.info_once("GEMM-RS is enabled.") + try: + init_gemm_rs_ar( + max_M=vllm_config.scheduler_config.max_num_batched_tokens, + N=config.hidden_size, + all_reduce=all_reduce, + ) + except RuntimeError as e: + logger.warning_once( + "%s is disabled because initialization failed: %s. This may mean " + "the TP ranks do not share one NVLink domain.", + mode, + e, + ) + return False + if all_reduce: + logger.info_once( + "GEMM-AR is enabled. To disable it, set VLLM_KIMI_K3_GEMM_AR=0." + ) + else: + logger.info_once("GEMM-RS is enabled.") return True @@ -217,7 +235,7 @@ class KimiMLP(nn.Module): reduce_results: bool = True, use_sequence_parallel: bool = False, can_shard_sequence_parallel: bool = False, - run_gemm_rs: bool = False, + run_gemm_rs_ar: bool = False, prefix: str = "", activation_situ_beta: float | None = None, activation_situ_linear_beta: float | None = None, @@ -230,7 +248,6 @@ class KimiMLP(nn.Module): use_sequence_parallel, can_shard_sequence_parallel, ) - self.run_gemm_rs = self.shard_sequence_parallel and run_gemm_rs replicate = use_sequence_parallel and not self.shard_sequence_parallel self.gate_up_proj = MergedColumnParallelLinear( @@ -252,13 +269,22 @@ class KimiMLP(nn.Module): disable_tp=replicate, prefix=f"{prefix}.down_proj", ) - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs + self.gemm_rs_ar = None + # RS requires sequence sharding; AR operates on replicated tokens. + use_gemm_rs_ar = self.shard_sequence_parallel or ( + not use_sequence_parallel and reduce_results + ) + if use_gemm_rs_ar and run_gemm_rs_ar: + from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import ( + get_gemm_rs_ar, + ) - self.run_gemm_rs = get_gemm_rs().can_run(self.down_proj) - if not self.run_gemm_rs: + gemm_rs_ar = get_gemm_rs_ar() + if gemm_rs_ar.can_run(self.down_proj): + self.gemm_rs_ar = gemm_rs_ar + else: logger.warning_once( - "GEMM-RS is disabled for %s due to an incompatible projection.", + "GEMM-RS/AR is disabled for %s due to an incompatible projection.", prefix, ) if hidden_act == "silu": @@ -284,12 +310,8 @@ class KimiMLP(nn.Module): gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) - if self.run_gemm_rs: - from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs - - gemm_rs = get_gemm_rs() - if gemm_rs.should_run(x): - return gemm_rs(x, self.down_proj.weight) + if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(x): + return self.gemm_rs_ar(x, self.down_proj.weight) x, _ = self.down_proj(x) if self.shard_sequence_parallel: @@ -527,7 +549,7 @@ class KimiMoE(nn.Module): prefix: str = "", layer_idx: int = 0, use_sequence_parallel: bool = False, - run_gemm_rs: bool = False, + run_gemm_rs_ar: bool = False, ): super().__init__() hidden_size = config.hidden_size @@ -616,7 +638,7 @@ class KimiMoE(nn.Module): # FusedMoE path below hands them to the runner, which fuses # their reduction and assumes the replicated layout. can_shard_sequence_parallel=self.use_mega_moe, - run_gemm_rs=run_gemm_rs, + run_gemm_rs_ar=run_gemm_rs_ar, prefix=f"{prefix}.shared_experts", activation_situ_beta=activation_situ_beta, activation_situ_linear_beta=activation_situ_linear_beta, @@ -838,7 +860,7 @@ class KimiDecoderLayer(nn.Module): vllm_config: VllmConfig, prefix: str = "", aux_stream: torch.cuda.Stream | None = None, - run_gemm_rs: bool = False, + run_gemm_rs_ar: bool = False, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -873,7 +895,7 @@ class KimiDecoderLayer(nn.Module): config, vllm_config, prefix=f"{prefix}.self_attn", - run_gemm_rs=run_gemm_rs, + run_gemm_rs_ar=run_gemm_rs_ar, ) self._self_attn_writes_output = False else: @@ -910,7 +932,7 @@ class KimiDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", aux_stream=aux_stream, - run_gemm_rs=run_gemm_rs, + run_gemm_rs_ar=run_gemm_rs_ar, ) self._self_attn_writes_output = False @@ -925,7 +947,7 @@ class KimiDecoderLayer(nn.Module): prefix=f"{prefix}.block_sparse_moe", layer_idx=layer_idx, use_sequence_parallel=self.use_sequence_parallel, - run_gemm_rs=run_gemm_rs, + run_gemm_rs_ar=run_gemm_rs_ar, ) self.mlp = self.block_sparse_moe else: @@ -937,7 +959,7 @@ class KimiDecoderLayer(nn.Module): prefix=f"{prefix}.mlp", use_sequence_parallel=self.use_sequence_parallel, can_shard_sequence_parallel=True, - run_gemm_rs=run_gemm_rs, + run_gemm_rs_ar=run_gemm_rs_ar, activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, ) @@ -1118,9 +1140,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): self.vocab_size = config.vocab_size - # GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP - # ranks to belong to one NVLink domain. - self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel) + # GEMM-RS/AR uses NCCL symmetric-memory multicast, which requires all + # TP ranks to belong to one NVLink domain. + self.run_gemm_rs_ar = maybe_init_gemm_rs_ar( + vllm_config, self.use_sequence_parallel + ) if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( @@ -1142,7 +1166,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): vllm_config, prefix, aux_stream=aux_stream, - run_gemm_rs=self.run_gemm_rs, + run_gemm_rs_ar=self.run_gemm_rs_ar, ) self.start_layer, self.end_layer, self.layers = make_layers( diff --git a/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py b/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs_ar.py similarity index 82% rename from vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py rename to vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs_ar.py index a775cd7b12..923f76b6c9 100644 --- a/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs.py +++ b/vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs_ar.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""SM100 BF16 GEMM with a fused tensor-parallel reduce-scatter.""" +"""SM100 BF16 GEMM with fused tensor-parallel reduce-scatter/all-reduce.""" # Based on CUTLASS's Blackwell distributed GEMM-RS example at dcf215a. # See https://github.com/NVIDIA/cutlass/issues/3117 for memory semantics. @@ -32,7 +32,7 @@ def nanosleep(ns: int, *, loc=None, ip=None) -> None: @dsl_user_op def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: - # NOTE: assume x is contiguous + # The vector instruction assumes x is contiguous and 16-byte aligned. assert x.element_type == BFloat16 vec_type = ".v4.bf16x2" @@ -66,15 +66,40 @@ def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor: return cute.recast_tensor(y, x.element_type) -class Sm100GemmRsBF16: +@dsl_user_op +def multimem_st_16B(dst: cute.Tensor, value: cute.Tensor, *, loc=None, ip=None) -> None: + # The vector instruction assumes dst and value are contiguous BF16x8 vectors + # and dst is 16-byte aligned. + assert dst.element_type == BFloat16 + assert value.element_type == BFloat16 + ptr = dst.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip) + regs = cute.recast_tensor(value, Int32, loc=loc, ip=ip) + llvm.inline_asm( + None, + [ptr] + [regs[i].ir_value(loc=loc, ip=ip) for i in range(4)], + "multimem.st.relaxed.sys.global.v4.f32 [$0], {$1, $2, $3, $4};", + "l,r,r,r,r", + has_side_effects=True, + loc=loc, + ip=ip, + ) + + +class Sm100GemmRsArBF16: def __init__( - self, rank: int, num_ranks: int, BN: int = 128, cta_group: int = 1 + self, + rank: int, + num_ranks: int, + BN: int = 128, + cta_group: int = 1, + all_reduce: bool = False, ) -> None: self.rank = rank self.num_ranks = num_ranks BM, BK = 128, 64 self.cta_tile = (BM, BN, BK) self.cta_group = cta_group + self.all_reduce = all_reduce smem_bytes = get_smem_capacity_in_bytes() self.stage_size = (BM + (BN // cta_group)) * BK * 2 @@ -190,6 +215,9 @@ class Sm100GemmRsBF16: M, K = A_tma.tma_tensor.shape N, _ = B_tma.tma_tensor.shape + local_M = ( + partial_uc.shape[0] // num_ranks if self.all_reduce else output.shape[0] + ) grid_m = cute.ceil_div(M, BM) # Keep 2-CTA clusters within a single N tile. grid_m = cute.ceil_div(grid_m, cta_group) * cta_group @@ -322,7 +350,6 @@ class Sm100GemmRsBF16: # Keep epilogue in warps 0-3 and communication in warps 4-7. # Swapping the warpgroups can hang due to warp scheduling. tid_ = tid % 128 - local_M = output.shape[0] # Offset in the [M, N] GEMM result. rank_start = self.rank * local_M @@ -338,7 +365,8 @@ class Sm100GemmRsBF16: max_tile_rows = BM // num_ranks vecs_per_tile = max_tile_rows * vec_cols partial_vecs = cute.zipped_divide(partial_mc, (1, vec_width)) - output_vecs = cute.zipped_divide(output, (1, vec_width)) + if cutlass.const_expr(not self.all_reduce): + output_vecs = cute.zipped_divide(output, (1, vec_width)) for tile_id in range(raw_bid, total_tiles, num_bids): rs_bid_m = tile_id % grid_m @@ -400,7 +428,6 @@ class Sm100GemmRsBF16: reduced_vec.store(tmp.load()) reduced_vecs.append(reduced_vec) - # Store the result to local L2. for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128): vec_idx = tid_ + vec_iter * 128 @@ -409,11 +436,19 @@ class Sm100GemmRsBF16: col = bid_n * vec_cols + vec_idx % vec_cols if local_row < local_row_end and global_row < M: - cute.copy( - st_atom, - reduced_vecs[vec_iter], - output_vecs[None, (local_row, col)], - ) + if cutlass.const_expr(self.all_reduce): + # Broadcast the result to all ranks. + multimem_st_16B( + partial_vecs[None, (global_row, col)], + reduced_vecs[vec_iter], + ) + else: + # Store the result to local L2. + cute.copy( + st_atom, + reduced_vecs[vec_iter], + output_vecs[None, (local_row, col)], + ) cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) # Release each GEMM tile consumed by this RS tile. The last @@ -481,19 +516,21 @@ class Sm100GemmRsBF16: num_ranks, ) - # Exit barrier. GPU scope is sufficient because the kernel writes - # local global memory and only needs to flush it to local L2. + # Exit barrier. GPU scope is sufficient for RS because the kernel + # writes local global memory and only needs to flush it to local L2. + # AR uses system scope to flush the multimem stores to remote L2. cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128) if tid_ == 0: exit_flag = total_tiles + raw_bid + scope = "sys" if self.all_reduce else "gpu" utils.distributed.multimem_red_add1( - flags_mc_ptr + exit_flag, order="release", scope="gpu" + flags_mc_ptr + exit_flag, order="release", scope=scope ) utils.distributed.spin_lock_atom_cas_acquire_wait( flags_uc_ptr + exit_flag, expected_val=num_ranks, reset_val=0, - scope="gpu", + scope=scope, ) else: @@ -564,8 +601,8 @@ class Sm100GemmRsBF16: if tid_ == 0 and bid_m * BM < M: gemm_start = bid_m * BM gemm_end = min(gemm_start + BM, M) - first_owner = gemm_start // output.shape[0] - last_owner = (gemm_end - 1) // output.shape[0] + first_owner = gemm_start // local_M + last_owner = (gemm_end - 1) // local_M # signal to all consuming ranks (can be more than 1) for rank in cutlass.range_constexpr(num_ranks): @@ -595,7 +632,13 @@ class Sm100GemmRsBF16: @cache @staticmethod - def compile(rank: int, num_ranks: int, BN: int, cta_group: int): + def compile( + rank: int, + num_ranks: int, + BN: int, + cta_group: int, + all_reduce: bool = False, + ): M = cute.sym_int() padded_M = cute.sym_int() N = cute.sym_int() @@ -616,18 +659,24 @@ class Sm100GemmRsBF16: assumed_align=32, ) partial_mc_ptr = nullptr(BFloat16, cute.AddressSpace.gmem, assumed_align=32) - output = make_fake_tensor( - BFloat16, - (local_M, N), - (cute.sym_int64(divisibility=16), 1), - assumed_align=32, + output = ( + None + if all_reduce + else make_fake_tensor( + BFloat16, + (local_M, N), + (cute.sym_int64(divisibility=16), 1), + assumed_align=32, + ) ) flags = make_fake_tensor(Int32, (num_flags,), (1,), assumed_align=16) flags_mc_ptr = nullptr(Int32, cute.AddressSpace.gmem, assumed_align=16) peer_flag_ptr = nullptr(Int64, cute.AddressSpace.gmem, assumed_align=8) stream = make_fake_stream(use_tvm_ffi_env_stream=True) - kernel = Sm100GemmRsBF16(rank, num_ranks, BN, cta_group) + kernel = Sm100GemmRsArBF16( + rank, num_ranks, BN, cta_group, all_reduce=all_reduce + ) return cute.compile( kernel, A, @@ -644,13 +693,20 @@ class Sm100GemmRsBF16: ) -class GemmRS: - """Own the max-sized symmetric workspace for Kimi-K3 GEMM-RS launches. +class GemmRsAr: + """Own the symmetric workspace for Kimi-K3 GEMM-RS/AR launches. All TP ranks must belong to one NVLink domain for multimem instructions. + + Each instance is bound to either RS or AR. A vLLM worker has one static + sequence-parallel topology, so the process-wide singleton only needs one + mode. Two independent mode-specific singletons would lift that restriction + but duplicate the large symmetric workspace. A future mixed-mode design + should instead use lightweight RS/AR frontends over one shared multicast + workspace; that is outside this integration's current scope. """ - def __init__(self, *, max_M: int, N: int) -> None: + def __init__(self, *, max_M: int, N: int, all_reduce: bool = False) -> None: tp_group = get_tp_group() group = tp_group.device_group rank = tp_group.rank_in_group @@ -667,9 +723,12 @@ class GemmRS: self.max_M = max_M self.N = N self.device = device + self.all_reduce = all_reduce self.partial = symm_mem.empty((max_M, N), dtype=torch.bfloat16, device=device) self.partial_handle = symm_mem.rendezvous(self.partial, group) + if self.partial_handle.multicast_ptr == 0: + raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory") self.partial_mc_ptr = make_ptr( BFloat16, self.partial_handle.multicast_ptr, @@ -684,6 +743,8 @@ class GemmRS: max_flags = grid_m * (N // 128) + self.num_sms self.flags = symm_mem.empty(max_flags, dtype=torch.int32, device=device) self.flags_handle = symm_mem.rendezvous(self.flags, group) + if self.flags_handle.multicast_ptr == 0: + raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory") self.flags.zero_() self.flags_mc_ptr = make_ptr( Int32, @@ -698,8 +759,6 @@ class GemmRS: assumed_align=8, ) - assert self.partial_handle.multicast_ptr != 0 - assert self.flags_handle.multicast_ptr != 0 torch.accelerator.synchronize(device) tp_group.barrier() @@ -721,7 +780,8 @@ class GemmRS: ) def should_run(self, x: torch.Tensor) -> bool: - # Small-M shapes are supported but faster on the existing LL path. + # Use the same threshold for RS and AR for now. Small-M shapes are + # supported but faster on the existing LL path. return x.shape[0] >= 128 def __call__(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: @@ -752,12 +812,15 @@ class GemmRS: num_ctas = num_ctas // cta_group * cta_group assert self.flags.numel() >= num_tiles + num_ctas - output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device) - compiled = Sm100GemmRsBF16.compile( + output = None + if not self.all_reduce: + output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device) + compiled = Sm100GemmRsArBF16.compile( self.rank, self.world_size, BN, cta_group, + self.all_reduce, ) compiled( x, @@ -770,21 +833,52 @@ class GemmRS: self.peer_flag_ptr, num_ctas, ) + if self.all_reduce: + # AttnRes may retain output past the next workspace reuse. + # A future kernel could overlap this copy using an extra warp or + # the communication warp. + return self.partial[:M].clone() + assert output is not None return output -_gemm_rs: GemmRS | None = None +_gemm_rs_ar: GemmRsAr | None = None -def init_gemm_rs(max_M: int, N: int) -> None: - """Collectively initialize the process-wide GEMM-RS state.""" - global _gemm_rs - if _gemm_rs is not None: - assert _gemm_rs.max_M >= max_M and _gemm_rs.N == N +def init_gemm_rs_ar(max_M: int, N: int, *, all_reduce: bool = False) -> None: + """Collectively initialize the process-wide, mode-bound GEMM-RS/AR state.""" + global _gemm_rs_ar + if _gemm_rs_ar is not None: + if _gemm_rs_ar.all_reduce != all_reduce: + current = "AR" if _gemm_rs_ar.all_reduce else "RS" + requested = "AR" if all_reduce else "RS" + raise RuntimeError( + f"GEMM-RS/AR is already initialized for {current}; " + f"a worker cannot reinitialize it for {requested}" + ) + assert _gemm_rs_ar.max_M >= max_M and _gemm_rs_ar.N == N return - _gemm_rs = GemmRS(max_M=max_M, N=N) + _gemm_rs_ar = GemmRsAr(max_M=max_M, N=N, all_reduce=all_reduce) -def get_gemm_rs() -> GemmRS: - assert _gemm_rs is not None, "GEMM-RS is not initialized" - return _gemm_rs +def warmup_gemm_rs_ar() -> int: + """Compile every reachable dispatch for the initialized GEMM-RS/AR mode.""" + # Initialization can be disabled or fail when multicast is unavailable. + if _gemm_rs_ar is None: + return 0 + # Keep these profiles in sync with the dispatch in GemmRsAr.__call__. + profiles = ((128, 1), (128, 2), (256, 2)) + for BN, cta_group in profiles: + Sm100GemmRsArBF16.compile( + _gemm_rs_ar.rank, + _gemm_rs_ar.world_size, + BN, + cta_group, + _gemm_rs_ar.all_reduce, + ) + return len(profiles) + + +def get_gemm_rs_ar() -> GemmRsAr: + assert _gemm_rs_ar is not None, "GEMM-RS/AR is not initialized" + return _gemm_rs_ar