[Kernel][DSV4.1] Fuse MXFP8 wo_b GEMM with sequence-parallel reduce-scatter (#57428)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Canlin Guo
2026-09-22 01:07:17 -07:00
committed by GitHub
co-authored by Thien Tran OpenAI Codex Claude Fable 5.1 Jee Jee Li
parent a6c47fbbf4
commit f92b78f6ef
18 changed files with 1037 additions and 624 deletions
+407
View File
@@ -0,0 +1,407 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the SM100 GEMM-RS/AR kernel against the model's unfused path.
The baseline is what a model runs without the fusion: the projection's own
linear kernel followed by vLLM's collective (``sp_reduce_scatter`` for RS,
``tensor_model_parallel_all_reduce`` for AR), with the collective backend
chosen by vLLM's dispatch exactly as in serving. The same two kernels are also
timed bare (the GEMM alone, the collective alone on a precomputed partial);
their max is the floor a perfectly overlapped fused kernel could reach.
All ranks must belong to one NVLink domain. Run a TP8 sweep over the Kimi-K3
projections with:
torchrun --nproc-per-node=8 -- benchmarks/kernels/benchmark_gemm_rs_ar.py
or the DeepSeek-V4.1 MXFP8 ``wo_b`` projection at TP4 with:
torchrun --nproc-per-node=4 -- benchmarks/kernels/benchmark_gemm_rs_ar.py \
--model deepseek_v41 --backend flashinfer_cutedsl
"""
import argparse
import json
import os
import statistics
from collections.abc import Callable
import torch
import torch.distributed as dist
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
tensor_model_parallel_all_reduce,
)
from vllm.distributed.parallel_state import (
get_tp_group,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
# Per model: (hidden size N, global input widths K). Kimi-K3 lists the
# shared-expert down-proj and attention O-proj; DeepSeek-V4.1 lists the
# ``wo_b`` output projection (o_groups * o_lora_rank).
_MODEL_PROJECTIONS = {
"kimi_k3": (7168, (6144, 12288)),
"deepseek_v41": (5120, (8192,)),
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--mode",
choices=("rs", "ar"),
default="rs",
help="Collective mode to benchmark.",
)
parser.add_argument(
"--m",
type=int,
nargs="+",
default=[128, 512, 2048, 8192, 32768],
help="Global token counts to benchmark.",
)
parser.add_argument(
"--model",
choices=tuple(_MODEL_PROJECTIONS),
default="kimi_k3",
help="Model whose projection shapes set the default --n and --k.",
)
parser.add_argument(
"--backend",
choices=("bf16", "flashinfer_cutlass", "flashinfer_cutedsl"),
default="bf16",
help="Weight dtype/layout: BF16 or an online-MXFP8 FlashInfer kernel.",
)
parser.add_argument(
"--k",
type=int,
nargs="+",
help=(
"Per-rank input dimensions. By default, derive the selected "
"model's projection dimensions from the TP world size."
),
)
parser.add_argument(
"--n",
type=int,
help="Output width. Defaults to the selected model's hidden size.",
)
parser.add_argument(
"--num-workspaces",
type=int,
default=10,
help="Pointer-distinct inputs and CUDA graphs to rotate.",
)
parser.add_argument("--warmup-replays", type=int, default=5)
parser.add_argument("--samples", type=int, default=20)
parser.add_argument("--json", type=str, help="Also write the results here.")
return parser.parse_args()
def make_projection(
gemm_rs_ar: GemmRsAr,
weight: torch.Tensor,
world_size: int,
backend: str,
) -> RowParallelLinear:
"""Wrap a BF16 [N, K] weight the way the model would hand it to GEMM-RS.
For the MXFP8 backends this quantizes online through the FlashInfer
kernel's post-load step, so the fused kernel and the unfused baseline see
the production weight and scale layouts.
"""
N, K = weight.shape
quant_config = None
if backend != "bf16":
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
quant_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
with torch.device(weight.device):
linear = RowParallelLinear(
K * world_size,
N,
bias=False,
params_dtype=weight.dtype,
quant_config=quant_config,
# The baseline reduces the local partial itself, like the models.
reduce_results=False,
return_bias=False,
)
# Select the fused path before online quantization replaces the weights.
assert gemm_rs_ar.can_run(linear)
linear.weight = torch.nn.Parameter(weight, requires_grad=False)
linear.quant_method.process_weights_after_loading(linear)
return linear
def capture_graph(
op: Callable[[], torch.Tensor],
stream: torch.cuda.Stream,
cpu_group: dist.ProcessGroup,
) -> torch.cuda.CUDAGraph:
stream.wait_stream(torch.cuda.current_stream())
dist.barrier(group=cpu_group)
with torch.cuda.stream(stream):
for _ in range(3):
op()
stream.synchronize()
dist.barrier(group=cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
op()
torch.cuda.current_stream().wait_stream(stream)
dist.barrier(group=cpu_group)
return graph
def benchmark_graphs(
candidate_graphs: dict[str, list[torch.cuda.CUDAGraph]],
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
) -> dict[str, float]:
"""Median over samples of the slowest rank's replay time, in us."""
names = list(candidate_graphs)
for round_index in range(warmup_replays):
for offset in range(len(names)):
name = names[(round_index + offset) % len(names)]
graphs = candidate_graphs[name]
device_barrier()
graphs[round_index % len(graphs)].replay()
torch.accelerator.synchronize()
timings: dict[str, list[float]] = {name: [] for name in names}
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
for sample_index in range(samples):
for offset in range(len(names)):
name = names[(sample_index + offset) % len(names)]
graphs = candidate_graphs[name]
device_barrier()
start.record()
graphs[sample_index % len(graphs)].replay()
end.record()
end.synchronize()
elapsed = torch.tensor(
start.elapsed_time(end) * 1000,
dtype=torch.float64,
device=torch.accelerator.current_device_index(),
)
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
timings[name].append(elapsed.item())
return {name: statistics.median(values) for name, values in timings.items()}
def benchmark_shape(
gemm_rs_ar: GemmRsAr,
mode: str,
backend: str,
M: int,
N: int,
K: int,
num_workspaces: int,
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
) -> dict[str, float | int | str]:
all_reduce = mode == "ar"
world_size = dist.get_world_size(device_group)
rank = dist.get_rank(device_group)
device = torch.device("cuda", torch.accelerator.current_device_index())
local_M = (M + world_size - 1) // world_size
rng = torch.Generator(device=device)
rng.manual_seed(1000 + rank * 10 + M + K)
inputs = [
torch.randn(M, K, dtype=torch.bfloat16, device=device, generator=rng)
for _ in range(num_workspaces)
]
projections = [
make_projection(
gemm_rs_ar,
torch.randn(N, K, dtype=torch.bfloat16, device=device, generator=rng),
world_size,
backend,
)
for _ in range(num_workspaces)
]
def collective(partial: torch.Tensor) -> torch.Tensor:
if all_reduce:
return tensor_model_parallel_all_reduce(partial)
return sp_reduce_scatter(partial)
def gemm(x: torch.Tensor, linear: RowParallelLinear) -> torch.Tensor:
return linear(x)
def unfused(x: torch.Tensor, linear: RowParallelLinear) -> torch.Tensor:
return collective(linear(x))
def fused(x: torch.Tensor, linear: RowParallelLinear) -> torch.Tensor:
return gemm_rs_ar.apply(x, linear)
# The bare collective runs on the GEMM's own output so it sees the same
# message size and layout the unfused path hands to the collective.
partials = [linear(x) for x, linear in zip(inputs, projections)]
expected = unfused(inputs[0], projections[0])
actual = fused(inputs[0], projections[0])
torch.accelerator.synchronize(device)
rows = M if all_reduce else min(max(M - rank * local_M, 0), local_M)
torch.testing.assert_close(
actual[:rows], expected[:rows], rtol=5e-2, atol=4.0, msg=f"M={M}, K={K}"
)
candidates: dict[str, list[Callable[[], torch.Tensor]]] = {
"gemm": [lambda x=x, w=w: gemm(x, w) for x, w in zip(inputs, projections)],
"collective": [lambda p=p: collective(p) for p in partials],
"vllm": [lambda x=x, w=w: unfused(x, w) for x, w in zip(inputs, projections)],
"gemm_rs_ar": [
lambda x=x, w=w: fused(x, w) for x, w in zip(inputs, projections)
],
}
candidate_graphs = {}
keepalive: list[object] = []
for name, ops in candidates.items():
stream = torch.cuda.Stream()
candidate_graphs[name] = [capture_graph(op, stream, cpu_group) for op in ops]
keepalive.append(stream)
times = benchmark_graphs(
candidate_graphs, warmup_replays, samples, device_group, device_barrier
)
floor = max(times["gemm"], times["collective"])
return {
"mode": mode.upper(),
"backend": backend,
"M": M,
"N": N,
"K": K,
"gemm_us": times["gemm"],
"collective_us": times["collective"],
"floor_us": floor,
"vllm_us": times["vllm"],
"gemm_rs_ar_us": times["gemm_rs_ar"],
"latency_change": (times["gemm_rs_ar"] - times["vllm"]) / times["vllm"],
"floor_gap": (times["gemm_rs_ar"] - floor) / floor,
}
def print_results(results: list[dict[str, float | int | str]]) -> None:
collective = results[0]["mode"]
backend = results[0]["backend"]
print(f"### GEMM-{collective} vs vLLM unfused path ({backend} weights)")
print(
f"| M | N | K | GEMM (us) | {collective} (us) | floor (us) | vLLM (us) | "
f"GEMM-{collective} (us) | vs vLLM | vs floor |"
)
print("|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|")
for r in results:
print(
f"| {r['M']} | {r['N']} | {r['K']} | {r['gemm_us']:.1f} | "
f"{r['collective_us']:.1f} | {r['floor_us']:.1f} | {r['vllm_us']:.1f} | "
f"{r['gemm_rs_ar_us']:.1f} | {r['latency_change']:+.1%} | "
f"{r['floor_gap']:+.1%} |"
)
def main() -> None:
args = parse_args()
default_n, default_k = _MODEL_PROJECTIONS[args.model]
if args.n is None:
args.n = default_n
assert args.m and min(args.m) > 0
assert args.n % 128 == 0
assert args.num_workspaces > 0
assert args.warmup_replays >= 0
assert args.samples > 0
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
init_distributed_environment()
world_size = dist.get_world_size()
if args.k is None:
assert all(K % world_size == 0 for K in default_k)
K_values = [K // world_size for K in default_k]
else:
K_values = args.k
k_alignment = 64 if args.backend == "bf16" else 128
assert all(K % k_alignment == 0 for K in K_values)
config = VllmConfig()
config.model_config = ModelConfig(dtype="bfloat16")
# Pin the online-MXFP8 kernel the projections quantize through.
config.kernel_config.linear_backend = (
"auto" if args.backend == "bf16" else args.backend
)
with set_current_vllm_config(config):
# Builds the TP communicator as in serving, so the baseline's
# collective is whatever vLLM's dispatch selects (FlashInfer all-reduce
# first, then NCCL symmetric memory, custom all-reduce, pynccl).
initialize_model_parallel(tensor_model_parallel_size=world_size)
tp_group = get_tp_group()
pynccl_comm = tp_group.device_communicator.pynccl_comm
assert pynccl_comm is not None
sync_input = torch.zeros(1, device=torch.accelerator.current_device_index())
sync_output = torch.empty_like(sync_input)
def device_barrier() -> None:
# Order the timed launch after a device-side rank rendezvous without
# including the rendezvous itself in the measured event interval.
pynccl_comm.all_reduce(sync_input, sync_output)
gemm_rs_ar = GemmRsAr(
max_M=max(args.m),
N=args.n,
all_reduce=args.mode == "ar",
)
with set_current_vllm_config(config):
results = [
benchmark_shape(
gemm_rs_ar,
args.mode,
args.backend,
M,
args.n,
K,
args.num_workspaces,
args.warmup_replays,
args.samples,
tp_group.device_group,
tp_group.cpu_group,
device_barrier,
)
for K in K_values
for M in args.m
]
del gemm_rs_ar
if tp_group.rank_in_group == 0:
print_results(results)
if args.json:
with open(args.json, "w") as f:
json.dump(results, f, indent=2)
dist.barrier(group=tp_group.cpu_group)
cleanup_dist_env_and_memory()
if __name__ == "__main__":
main()
@@ -1,469 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the SM100 Kimi-K3 GEMM-RS/AR kernel.
All ranks must belong to one NVLink domain. For example, run a TP8 sweep with:
torchrun --nproc-per-node=8 \
benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py
"""
import argparse
import os
import statistics
from collections.abc import Callable
from dataclasses import dataclass
import pandas as pd
import torch
import torch.distributed as dist
import torch.distributed._symmetric_memory as symm_mem
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.parallel_state import (
get_tp_group,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr
# Shared-expert down-proj and attention O-proj.
_KIMI_K3_PROJECTION_K = (6144, 12288)
@dataclass
class Candidate:
name: str
runs: list[Callable[[], torch.Tensor]]
check_correctness: bool = True
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--mode",
choices=("rs", "ar"),
default="rs",
help="Collective mode to benchmark.",
)
parser.add_argument(
"--m",
type=int,
nargs="+",
default=[128, 512, 2048, 8192, 32768],
help="Global token counts to benchmark.",
)
parser.add_argument(
"--k",
type=int,
nargs="+",
help=(
"Per-rank input dimensions. By default, derive the Kimi-K3 "
"shared-expert down-proj and O-proj dimensions from the TP "
"world size."
),
)
parser.add_argument("--n", type=int, default=7168)
parser.add_argument(
"--num-workspaces",
type=int,
default=10,
help="Pointer-distinct inputs and CUDA graphs to rotate.",
)
parser.add_argument("--warmup-replays", type=int, default=5)
parser.add_argument("--samples", type=int, default=20)
return parser.parse_args()
def capture_graph(
op: Callable[[], torch.Tensor],
stream: torch.cuda.Stream,
cpu_group: dist.ProcessGroup,
) -> tuple[torch.cuda.CUDAGraph, list[torch.Tensor | None]]:
result: list[torch.Tensor | None] = [None]
stream.wait_stream(torch.cuda.current_stream())
dist.barrier(group=cpu_group)
with torch.cuda.stream(stream):
for _ in range(3):
result[0] = op()
stream.synchronize()
dist.barrier(group=cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
result[0] = op()
torch.cuda.current_stream().wait_stream(stream)
dist.barrier(group=cpu_group)
return graph, result
def benchmark_graphs(
candidate_graphs: dict[str, list[torch.cuda.CUDAGraph]],
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
) -> dict[str, float]:
candidate_names = list(candidate_graphs)
for round_index in range(warmup_replays):
for candidate_index in range(len(candidate_names)):
candidate_id = (round_index + candidate_index) % len(candidate_names)
name = candidate_names[candidate_id]
graphs = candidate_graphs[name]
device_barrier()
graphs[round_index % len(graphs)].replay()
torch.accelerator.synchronize()
timings: dict[str, list[float]] = {name: [] for name in candidate_names}
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
for sample_index in range(samples):
for candidate_index in range(len(candidate_names)):
candidate_id = (sample_index + candidate_index) % len(candidate_names)
name = candidate_names[candidate_id]
graphs = candidate_graphs[name]
device_barrier()
start.record()
graphs[sample_index % len(graphs)].replay()
end.record()
end.synchronize()
elapsed = torch.tensor(
start.elapsed_time(end) * 1000,
dtype=torch.float64,
device=torch.accelerator.current_device_index(),
)
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
timings[name].append(elapsed.item())
return {name: statistics.median(values) for name, values in timings.items()}
def valid_rows(M: int, local_M: int, rank: int) -> int:
return min(max(M - rank * local_M, 0), local_M)
def benchmark_shape(
gemm_rs_ar: GemmRsAr,
mode: str,
M: int,
N: int,
K: int,
num_workspaces: int,
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
device_barrier: Callable[[], None],
) -> dict[str, float | int | str]:
all_reduce = mode == "ar"
world_size = dist.get_world_size(device_group)
rank = dist.get_rank(device_group)
device = torch.device("cuda", torch.accelerator.current_device_index())
padded_M = (M + world_size - 1) // world_size * world_size
local_M = padded_M // world_size
rng = torch.Generator(device=device)
rng.manual_seed(1000 + rank * 10 + M + K)
inputs = [
torch.randn(M, K, dtype=torch.bfloat16, device=device, generator=rng)
for _ in range(num_workspaces)
]
weights = [
torch.randn(N, K, dtype=torch.bfloat16, device=device, generator=rng)
for _ in range(num_workspaces)
]
partial = torch.empty((padded_M, N), dtype=torch.bfloat16, device=device)
symm_partial = symm_mem.empty((padded_M, N), dtype=torch.bfloat16, device=device)
symm_partial_handle = symm_mem.rendezvous(symm_partial, device_group)
collective_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)]
symm_collective_inputs = []
symm_collective_handles = []
for _ in range(num_workspaces):
collective_input = symm_mem.empty(
(padded_M, N),
dtype=torch.bfloat16,
device=device,
)
symm_collective_inputs.append(collective_input)
symm_collective_handles.append(
symm_mem.rendezvous(collective_input, device_group)
)
torch_output = torch.empty((local_M, N), dtype=torch.bfloat16, device=device)
symm_output = torch.empty_like(torch_output)
gemm_output = torch.empty((M, N), dtype=torch.bfloat16, device=device)
if padded_M > M:
partial[M:].zero_()
symm_partial[M:].zero_()
def make_torch_ring_ll_gemm_collective(
x: torch.Tensor, weight: torch.Tensor
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
torch.mm(x, weight.T, out=partial[:M])
if all_reduce:
dist.all_reduce(partial, group=device_group)
return partial[:M]
dist.reduce_scatter_single(torch_output, partial, group=device_group)
return torch_output
return run
def make_torch_ldmc_gemm_collective(
x: torch.Tensor, weight: torch.Tensor
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
torch.mm(x, weight.T, out=symm_partial[:M])
if all_reduce:
dist.all_reduce(symm_partial, group=device_group)
return symm_partial[:M]
dist.reduce_scatter_single(
symm_output,
symm_partial,
group=device_group,
)
return symm_output
return run
def make_fused_gemm_collective(
x: torch.Tensor, weight: torch.Tensor
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
return gemm_rs_ar(x, weight)
return run
def make_torch_gemm(
x: torch.Tensor,
weight: torch.Tensor,
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
return torch.mm(x, weight.T, out=gemm_output)
return run
def make_ring_ll_collective(
collective_input: torch.Tensor,
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
if all_reduce:
dist.all_reduce(collective_input, group=device_group)
return collective_input
dist.reduce_scatter_single(
torch_output, collective_input, group=device_group
)
return torch_output
return run
def make_ldmc_collective(
collective_input: torch.Tensor,
) -> Callable[[], torch.Tensor]:
def run() -> torch.Tensor:
if all_reduce:
dist.all_reduce(collective_input, group=device_group)
return collective_input
dist.reduce_scatter_single(
symm_output, collective_input, group=device_group
)
return symm_output
return run
candidates = (
Candidate(
"ring_ll_us",
[make_torch_ring_ll_gemm_collective(x, w) for x, w in zip(inputs, weights)],
),
Candidate(
"ldmc_us",
[make_torch_ldmc_gemm_collective(x, w) for x, w in zip(inputs, weights)],
),
Candidate(
"gemm_rs_ar_us",
[make_fused_gemm_collective(x, w) for x, w in zip(inputs, weights)],
),
Candidate(
"torch_gemm_us",
[make_torch_gemm(x, w) for x, w in zip(inputs, weights)],
check_correctness=False,
),
Candidate(
"ring_ll_collective_us",
[make_ring_ll_collective(x) for x in collective_inputs],
check_correctness=False,
),
Candidate(
"ldmc_collective_us",
[make_ldmc_collective(x) for x in symm_collective_inputs],
check_correctness=False,
),
)
expected = candidates[0].runs[0]()
rows = M if all_reduce else valid_rows(M, local_M, rank)
for candidate in candidates[1:]:
if not candidate.check_correctness:
continue
actual = candidate.runs[0]()
torch.accelerator.synchronize(device)
torch.testing.assert_close(
actual[:rows],
expected[:rows],
rtol=5e-2,
atol=4.0,
)
candidate_graphs = {}
graph_keepalive: list[object] = [symm_partial_handle, *symm_collective_handles]
for candidate in candidates:
stream = torch.cuda.Stream()
bundles = [capture_graph(run, stream, cpu_group) for run in candidate.runs]
candidate_graphs[candidate.name] = [graph for graph, _ in bundles]
graph_keepalive.extend(bundles)
graph_keepalive.append(stream)
times = benchmark_graphs(
candidate_graphs,
warmup_replays,
samples,
device_group,
device_barrier,
)
best_nccl_us = min(times["ring_ll_collective_us"], times["ldmc_collective_us"])
return {
"mode": mode.upper(),
"M": M,
"N": N,
"K": K,
**times,
"best_nccl_us": best_nccl_us,
"speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_ar_us"],
"speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_ar_us"],
}
def print_results(results: list[dict[str, float | int | str]]) -> None:
results_df = pd.DataFrame(results)
collective = str(results_df["mode"].iloc[0])
end_to_end = (
results_df[
[
"M",
"N",
"K",
"ring_ll_us",
"ldmc_us",
"gemm_rs_ar_us",
"speedup_vs_ring_ll",
"speedup_vs_ldmc",
]
]
.rename(
columns={
"ring_ll_us": f"Torch GEMM + NCCL {collective} (RING_LL) (us)",
"ldmc_us": f"Torch GEMM + NCCL {collective} (LDMC) (us)",
"gemm_rs_ar_us": f"GEMM-{collective} (us)",
"speedup_vs_ring_ll": "Speedup vs RING_LL",
"speedup_vs_ldmc": "Speedup vs LDMC",
}
)
.round(3)
)
components = (
results_df[["M", "N", "K", "torch_gemm_us", "best_nccl_us", "gemm_rs_ar_us"]]
.rename(
columns={
"torch_gemm_us": "Torch GEMM (us)",
"best_nccl_us": f"NCCL {collective} (best) (us)",
"gemm_rs_ar_us": f"GEMM-{collective} (us)",
}
)
.round(2)
)
print(f"### {collective} end-to-end latency")
print(end_to_end.to_markdown(index=False))
print(f"\n### {collective} component latency")
print(components.to_markdown(index=False))
print(
f"\nNCCL {collective} (best) is the faster of RING_LL and LDMC "
"for each shape.\n"
)
def main() -> None:
args = parse_args()
assert args.m and min(args.m) > 0
assert args.n % 256 == 0
assert args.num_workspaces > 0
assert args.warmup_replays >= 0
assert args.samples > 0
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
init_distributed_environment()
world_size = dist.get_world_size()
if args.k is None:
assert all(K % world_size == 0 for K in _KIMI_K3_PROJECTION_K)
K_values = [K // world_size for K in _KIMI_K3_PROJECTION_K]
else:
K_values = args.k
assert all(K % 64 == 0 for K in K_values)
# Reserve symmetric memory for the NCCL-managed benchmark allocations.
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
# NCCL-managed symmetric allocations select the NVLS/LDMC collective path.
symm_mem.set_backend("NCCL")
with set_current_vllm_config(VllmConfig()):
initialize_model_parallel(tensor_model_parallel_size=world_size)
tp_group = get_tp_group()
group_warmup = torch.zeros(1, device=torch.accelerator.current_device_index())
dist.all_reduce(group_warmup, group=tp_group.device_group)
pynccl_comm = tp_group.device_communicator.pynccl_comm
assert pynccl_comm is not None
sync_input = torch.zeros(1, device=torch.accelerator.current_device_index())
sync_output = torch.empty_like(sync_input)
def device_barrier() -> None:
# Order the timed launch after a device-side rank rendezvous without
# including the rendezvous itself in the measured event interval.
pynccl_comm.all_reduce(sync_input, sync_output)
gemm_rs_ar = GemmRsAr(
max_M=max(args.m),
N=args.n,
all_reduce=args.mode == "ar",
)
results = [
benchmark_shape(
gemm_rs_ar,
args.mode,
M,
args.n,
K,
args.num_workspaces,
args.warmup_replays,
args.samples,
tp_group.device_group,
tp_group.cpu_group,
device_barrier,
)
for K in K_values
for M in args.m
]
del gemm_rs_ar
if tp_group.rank_in_group == 0:
print_results(results)
dist.barrier(group=tp_group.cpu_group)
cleanup_dist_env_and_memory()
if __name__ == "__main__":
main()