[Kimi-K3] Add GEMM-RS for sequence parallelism (#52079)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -242,6 +242,7 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||
- pytest -v -s tests/distributed/test_nccl_symm_mem.py
|
||||
- pytest -v -s tests/kernels/test_kimi_k3_gemm_rs.py
|
||||
- pytest -v -s tests/v1/distributed/test_dbo.py
|
||||
- pytest -v -s tests/distributed/test_mnnvl_alltoall.py
|
||||
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the SM100 Kimi-K3 GEMM-RS kernel.
|
||||
|
||||
All ranks must belong to one NVLink domain. For example, run a TP8 sweep with:
|
||||
|
||||
torchrun --nproc-per-node=8 \
|
||||
benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import statistics
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
|
||||
|
||||
# Shared-expert down-proj and attention O-proj.
|
||||
_KIMI_K3_PROJECTION_K = (6144, 12288)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Candidate:
|
||||
name: str
|
||||
runs: list[Callable[[], torch.Tensor]]
|
||||
check_correctness: bool = True
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--m",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[128, 512, 2048, 8192, 32768],
|
||||
help="Global token counts to benchmark.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--k",
|
||||
type=int,
|
||||
nargs="+",
|
||||
help=(
|
||||
"Per-rank input dimensions. By default, derive the Kimi-K3 "
|
||||
"shared-expert down-proj and O-proj dimensions from the TP "
|
||||
"world size."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--n", type=int, default=7168)
|
||||
parser.add_argument(
|
||||
"--num-workspaces",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Pointer-distinct inputs and CUDA graphs to rotate.",
|
||||
)
|
||||
parser.add_argument("--warmup-replays", type=int, default=5)
|
||||
parser.add_argument("--samples", type=int, default=20)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def capture_graph(
|
||||
op: Callable[[], torch.Tensor],
|
||||
stream: torch.cuda.Stream,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> tuple[torch.cuda.CUDAGraph, list[torch.Tensor | None]]:
|
||||
result: list[torch.Tensor | None] = [None]
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
dist.barrier(group=cpu_group)
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
result[0] = op()
|
||||
stream.synchronize()
|
||||
dist.barrier(group=cpu_group)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
result[0] = op()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
dist.barrier(group=cpu_group)
|
||||
return graph, result
|
||||
|
||||
|
||||
def benchmark_graphs(
|
||||
candidate_graphs: dict[str, list[torch.cuda.CUDAGraph]],
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
device_group: dist.ProcessGroup,
|
||||
device_barrier: Callable[[], None],
|
||||
) -> dict[str, float]:
|
||||
candidate_names = list(candidate_graphs)
|
||||
for round_index in range(warmup_replays):
|
||||
for candidate_index in range(len(candidate_names)):
|
||||
candidate_id = (round_index + candidate_index) % len(candidate_names)
|
||||
name = candidate_names[candidate_id]
|
||||
graphs = candidate_graphs[name]
|
||||
device_barrier()
|
||||
graphs[round_index % len(graphs)].replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
timings: dict[str, list[float]] = {name: [] for name in candidate_names}
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
for sample_index in range(samples):
|
||||
for candidate_index in range(len(candidate_names)):
|
||||
candidate_id = (sample_index + candidate_index) % len(candidate_names)
|
||||
name = candidate_names[candidate_id]
|
||||
graphs = candidate_graphs[name]
|
||||
device_barrier()
|
||||
start.record()
|
||||
graphs[sample_index % len(graphs)].replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
|
||||
elapsed = torch.tensor(
|
||||
start.elapsed_time(end) * 1000,
|
||||
dtype=torch.float64,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
|
||||
timings[name].append(elapsed.item())
|
||||
return {name: statistics.median(values) for name, values in timings.items()}
|
||||
|
||||
|
||||
def valid_rows(M: int, local_M: int, rank: int) -> int:
|
||||
return min(max(M - rank * local_M, 0), local_M)
|
||||
|
||||
|
||||
def benchmark_shape(
|
||||
gemm_rs: GemmRS,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
num_workspaces: int,
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
device_group: dist.ProcessGroup,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
device_barrier: Callable[[], None],
|
||||
) -> dict[str, float | int]:
|
||||
world_size = dist.get_world_size(device_group)
|
||||
rank = dist.get_rank(device_group)
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
padded_M = (M + world_size - 1) // world_size * world_size
|
||||
local_M = padded_M // world_size
|
||||
|
||||
rng = torch.Generator(device=device)
|
||||
rng.manual_seed(1000 + rank * 10 + M + K)
|
||||
inputs = [
|
||||
torch.randn(M, K, dtype=torch.bfloat16, device=device, generator=rng)
|
||||
for _ in range(num_workspaces)
|
||||
]
|
||||
weights = [
|
||||
torch.randn(N, K, dtype=torch.bfloat16, device=device, generator=rng)
|
||||
for _ in range(num_workspaces)
|
||||
]
|
||||
|
||||
partial = torch.empty((padded_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_partial = symm_mem.empty((padded_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_partial_handle = symm_mem.rendezvous(symm_partial, device_group)
|
||||
rs_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)]
|
||||
symm_rs_inputs = []
|
||||
symm_rs_handles = []
|
||||
for _ in range(num_workspaces):
|
||||
rs_input = symm_mem.empty(
|
||||
(padded_M, N),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
symm_rs_inputs.append(rs_input)
|
||||
symm_rs_handles.append(symm_mem.rendezvous(rs_input, device_group))
|
||||
torch_output = torch.empty((local_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_output = torch.empty_like(torch_output)
|
||||
gemm_output = torch.empty((M, N), dtype=torch.bfloat16, device=device)
|
||||
|
||||
if padded_M > M:
|
||||
partial[M:].zero_()
|
||||
symm_partial[M:].zero_()
|
||||
|
||||
def make_torch_ring_ll_gemm_rs(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
torch.mm(x, weight.T, out=partial[:M])
|
||||
dist.reduce_scatter_single(torch_output, partial, group=device_group)
|
||||
return torch_output
|
||||
|
||||
return run
|
||||
|
||||
def make_torch_ldmc_gemm_rs(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
torch.mm(x, weight.T, out=symm_partial[:M])
|
||||
dist.reduce_scatter_single(
|
||||
symm_output,
|
||||
symm_partial,
|
||||
group=device_group,
|
||||
)
|
||||
return symm_output
|
||||
|
||||
return run
|
||||
|
||||
def make_fused_gemm_rs(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
return gemm_rs(x, weight)
|
||||
|
||||
return run
|
||||
|
||||
def make_torch_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
return torch.mm(x, weight.T, out=gemm_output)
|
||||
|
||||
return run
|
||||
|
||||
def make_ring_ll_rs(rs_input: torch.Tensor) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
dist.reduce_scatter_single(torch_output, rs_input, group=device_group)
|
||||
return torch_output
|
||||
|
||||
return run
|
||||
|
||||
def make_ldmc_rs(
|
||||
rs_input: torch.Tensor,
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
dist.reduce_scatter_single(symm_output, rs_input, group=device_group)
|
||||
return symm_output
|
||||
|
||||
return run
|
||||
|
||||
candidates = (
|
||||
Candidate(
|
||||
"ring_ll_us",
|
||||
[make_torch_ring_ll_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"ldmc_us",
|
||||
[make_torch_ldmc_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"gemm_rs_us",
|
||||
[make_fused_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"torch_gemm_us",
|
||||
[make_torch_gemm(x, w) for x, w in zip(inputs, weights)],
|
||||
check_correctness=False,
|
||||
),
|
||||
Candidate(
|
||||
"ring_ll_rs_us",
|
||||
[make_ring_ll_rs(x) for x in rs_inputs],
|
||||
check_correctness=False,
|
||||
),
|
||||
Candidate(
|
||||
"ldmc_rs_us",
|
||||
[make_ldmc_rs(x) for x in symm_rs_inputs],
|
||||
check_correctness=False,
|
||||
),
|
||||
)
|
||||
|
||||
expected = candidates[0].runs[0]()
|
||||
rows = valid_rows(M, local_M, rank)
|
||||
for candidate in candidates[1:]:
|
||||
if not candidate.check_correctness:
|
||||
continue
|
||||
actual = candidate.runs[0]()
|
||||
torch.accelerator.synchronize(device)
|
||||
torch.testing.assert_close(
|
||||
actual[:rows],
|
||||
expected[:rows],
|
||||
rtol=5e-2,
|
||||
atol=4.0,
|
||||
)
|
||||
|
||||
candidate_graphs = {}
|
||||
graph_keepalive: list[object] = [symm_partial_handle, *symm_rs_handles]
|
||||
for candidate in candidates:
|
||||
stream = torch.cuda.Stream()
|
||||
bundles = [capture_graph(run, stream, cpu_group) for run in candidate.runs]
|
||||
candidate_graphs[candidate.name] = [graph for graph, _ in bundles]
|
||||
graph_keepalive.extend(bundles)
|
||||
graph_keepalive.append(stream)
|
||||
|
||||
times = benchmark_graphs(
|
||||
candidate_graphs,
|
||||
warmup_replays,
|
||||
samples,
|
||||
device_group,
|
||||
device_barrier,
|
||||
)
|
||||
|
||||
best_nccl_rs_us = min(times["ring_ll_rs_us"], times["ldmc_rs_us"])
|
||||
return {
|
||||
"M": M,
|
||||
"N": N,
|
||||
"K": K,
|
||||
**times,
|
||||
"best_nccl_rs_us": best_nccl_rs_us,
|
||||
"speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_us"],
|
||||
"speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_us"],
|
||||
}
|
||||
|
||||
|
||||
def print_results(results: list[dict[str, float | int]]) -> None:
|
||||
results_df = pd.DataFrame(results)
|
||||
end_to_end = results_df[
|
||||
[
|
||||
"M",
|
||||
"N",
|
||||
"K",
|
||||
"ring_ll_us",
|
||||
"ldmc_us",
|
||||
"gemm_rs_us",
|
||||
"speedup_vs_ring_ll",
|
||||
"speedup_vs_ldmc",
|
||||
]
|
||||
].rename(
|
||||
columns={
|
||||
"ring_ll_us": "Torch GEMM + NCCL RS (RING_LL) (us)",
|
||||
"ldmc_us": "Torch GEMM + NCCL RS (LDMC) (us)",
|
||||
"gemm_rs_us": "GEMM-RS (us)",
|
||||
"speedup_vs_ring_ll": "Speedup vs RING_LL",
|
||||
"speedup_vs_ldmc": "Speedup vs LDMC",
|
||||
}
|
||||
)
|
||||
end_to_end = end_to_end.round(
|
||||
{
|
||||
"Torch GEMM + NCCL RS (RING_LL) (us)": 2,
|
||||
"Torch GEMM + NCCL RS (LDMC) (us)": 2,
|
||||
"GEMM-RS (us)": 2,
|
||||
"Speedup vs RING_LL": 3,
|
||||
"Speedup vs LDMC": 3,
|
||||
}
|
||||
)
|
||||
components = results_df[
|
||||
["M", "N", "K", "torch_gemm_us", "best_nccl_rs_us", "gemm_rs_us"]
|
||||
].rename(
|
||||
columns={
|
||||
"torch_gemm_us": "Torch GEMM (us)",
|
||||
"best_nccl_rs_us": "NCCL RS (best) (us)",
|
||||
"gemm_rs_us": "GEMM-RS (us)",
|
||||
}
|
||||
)
|
||||
components = components.round(2)
|
||||
|
||||
print("### End-to-end latency")
|
||||
print(end_to_end.to_markdown(index=False))
|
||||
print("\n### Component latency")
|
||||
print(components.to_markdown(index=False))
|
||||
print("\nNCCL RS (best) is the faster of RING_LL and LDMC for each shape.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
assert args.m and min(args.m) >= 128
|
||||
assert args.n % 256 == 0
|
||||
assert args.num_workspaces > 0
|
||||
assert args.warmup_replays >= 0
|
||||
assert args.samples > 0
|
||||
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
init_distributed_environment()
|
||||
world_size = dist.get_world_size()
|
||||
if args.k is None:
|
||||
assert all(K % world_size == 0 for K in _KIMI_K3_PROJECTION_K)
|
||||
K_values = [K // world_size for K in _KIMI_K3_PROJECTION_K]
|
||||
else:
|
||||
K_values = args.k
|
||||
assert all(K % 64 == 0 for K in K_values)
|
||||
# Reserve symmetric memory for the NCCL-managed benchmark allocations.
|
||||
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
|
||||
# NCCL-managed symmetric allocations select the NVLS/LDMC collective path.
|
||||
symm_mem.set_backend("NCCL")
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
tp_group = get_tp_group()
|
||||
group_warmup = torch.zeros(1, device=torch.accelerator.current_device_index())
|
||||
dist.all_reduce(group_warmup, group=tp_group.device_group)
|
||||
pynccl_comm = tp_group.device_communicator.pynccl_comm
|
||||
assert pynccl_comm is not None
|
||||
sync_input = torch.zeros(1, device=torch.accelerator.current_device_index())
|
||||
sync_output = torch.empty_like(sync_input)
|
||||
|
||||
def device_barrier() -> None:
|
||||
pynccl_comm.all_reduce(sync_input, sync_output)
|
||||
|
||||
gemm_rs = GemmRS(max_M=max(args.m), N=args.n)
|
||||
results = [
|
||||
benchmark_shape(
|
||||
gemm_rs,
|
||||
M,
|
||||
args.n,
|
||||
K,
|
||||
args.num_workspaces,
|
||||
args.warmup_replays,
|
||||
args.samples,
|
||||
tp_group.device_group,
|
||||
tp_group.cpu_group,
|
||||
device_barrier,
|
||||
)
|
||||
for K in K_values
|
||||
for M in args.m
|
||||
]
|
||||
|
||||
if tp_group.rank_in_group == 0:
|
||||
print_results(results)
|
||||
|
||||
dist.barrier(group=tp_group.cpu_group)
|
||||
del gemm_rs
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,178 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
_SHAPES = (
|
||||
(129, 768),
|
||||
(257, 768),
|
||||
(1023, 4224),
|
||||
(1024, 4224),
|
||||
(8191, 4224),
|
||||
)
|
||||
_N = 512
|
||||
|
||||
|
||||
def _reference(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
world_size: int,
|
||||
group: dist.ProcessGroup,
|
||||
) -> torch.Tensor:
|
||||
M = x.shape[0]
|
||||
padded_M = (M + world_size - 1) // world_size * world_size
|
||||
partial = torch.empty(
|
||||
(padded_M, weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
torch.mm(x, weight.T, out=partial[:M])
|
||||
if padded_M > M:
|
||||
partial[M:].zero_()
|
||||
|
||||
output = torch.empty(
|
||||
(padded_M // world_size, weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
dist.reduce_scatter_single(output, partial, group=group)
|
||||
return output
|
||||
|
||||
|
||||
def _assert_valid_rows_close(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
M: int,
|
||||
rank: int,
|
||||
) -> None:
|
||||
rank_start = rank * actual.shape[0]
|
||||
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
|
||||
torch.testing.assert_close(
|
||||
actual[:valid_rows],
|
||||
expected[:valid_rows],
|
||||
rtol=5e-2,
|
||||
atol=4.0,
|
||||
)
|
||||
|
||||
|
||||
def _worker(local_rank: int, world_size: int, master_port: int) -> None:
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(master_port),
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
group = dist.group.WORLD
|
||||
rank = dist.get_rank(group)
|
||||
gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N)
|
||||
|
||||
weight_generator = torch.Generator(device=device)
|
||||
input_generator = torch.Generator(device=device)
|
||||
weights = {}
|
||||
for K in {K for _, K in _SHAPES}:
|
||||
weight_generator.manual_seed(1000 + rank * 10 + K)
|
||||
weights[K] = torch.randn(
|
||||
_N,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=weight_generator,
|
||||
)
|
||||
|
||||
# Alternating shapes exercise producer-flag reuse across different grids,
|
||||
# CTA-group choices, and both BN=128 and BN=256 dispatches.
|
||||
for M, K in (*_SHAPES, *_SHAPES[::-1]):
|
||||
input_generator.manual_seed(2000 + M + K)
|
||||
x = torch.randn(
|
||||
M,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
expected = _reference(x, weights[K], world_size, group)
|
||||
actual = gemm_rs(x, weights[K])
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(actual, expected, M, rank)
|
||||
|
||||
# Exercise GEMM-RS under CUDA graph capture and replay.
|
||||
graph_M, graph_K = 1025, 4224
|
||||
input_generator.manual_seed(3000)
|
||||
graph_x = torch.randn(
|
||||
graph_M,
|
||||
graph_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
graph_expected = _reference(graph_x, weights[graph_K], world_size, group)
|
||||
graph_output = torch.empty_like(graph_expected)
|
||||
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(3):
|
||||
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
|
||||
capture_stream.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
dist.barrier(group=group)
|
||||
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(graph_output, graph_expected, graph_M, rank)
|
||||
|
||||
dist.barrier(group=group)
|
||||
del gemm_rs
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Kimi-K3 GEMM-RS requires SM100",
|
||||
)
|
||||
def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
world_size = 2
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip("GEMM-RS requires two GPUs")
|
||||
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
try:
|
||||
mp.spawn(
|
||||
_worker,
|
||||
args=(world_size, get_open_port()),
|
||||
nprocs=world_size,
|
||||
)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -267,19 +267,21 @@ def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "use_sequence_parallel", "tp_size", "expected"),
|
||||
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
|
||||
[
|
||||
(True, True, 8, True),
|
||||
(False, True, 8, False), # opt-in only
|
||||
(True, False, 8, False), # replication only exists under SP
|
||||
(True, True, 1, False), # nothing to shard
|
||||
(True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
(True, True, True, 8, True),
|
||||
(False, True, True, 8, False), # opt-in only
|
||||
(True, False, True, 8, False), # replication only exists under SP
|
||||
(True, True, False, 8, False), # FusedMoE path owns the reduction
|
||||
(True, True, True, 1, False), # nothing to shard
|
||||
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
],
|
||||
)
|
||||
def test_shard_sequence_parallel_mlp_gating(
|
||||
monkeypatch,
|
||||
enabled: bool,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
tp_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
@@ -293,6 +295,7 @@ def test_shard_sequence_parallel_mlp_gating(
|
||||
hidden_size=7168,
|
||||
intermediate_size=6144,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
eligible=eligible,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
@@ -40,6 +40,18 @@ def recast_val(x, dtype, *, loc=None, ip=None):
|
||||
return dtype(llvm.bitcast(dtype.mlir_type, x.ir_value(loc=loc, ip=ip)))
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def to_cta0_smem(ptr: cute.Pointer, *, loc=None, ip=None):
|
||||
return cute.make_ptr(
|
||||
ptr.dtype,
|
||||
ptr.toint(loc=loc, ip=ip) & 0xFEFF_FFFF,
|
||||
cute.AddressSpace.smem,
|
||||
assumed_align=8,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
def simple_tma_copy(atom, src, dst, mbar=None, cache_policy=None):
|
||||
"""A simple helper that wraps group_modes() and tma_partition()
|
||||
NOTE: this should be called WITHOUT cute.elect_one()
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from cutlass import Int32
|
||||
from cutlass._mlir.dialects import nvvm
|
||||
from cutlass.cutlass_dsl import dsl_user_op
|
||||
|
||||
_SPACE_MAP = {
|
||||
"cta": nvvm.MBarrierSpaceKind.CTA,
|
||||
"cluster": nvvm.MBarrierSpaceKind.CLUSTER,
|
||||
}
|
||||
|
||||
_MEMORY_ORDER_MAP = {
|
||||
"weak": nvvm.MemOrderKind.WEAK,
|
||||
"relaxed": nvvm.MemOrderKind.RELAXED,
|
||||
"acquire": nvvm.MemOrderKind.ACQUIRE,
|
||||
"release": nvvm.MemOrderKind.RELEASE,
|
||||
"acq_rel": nvvm.MemOrderKind.ACQ_REL,
|
||||
}
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def arrive(mbar, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None):
|
||||
nvvm.mbarrier_txn(
|
||||
mbar.to_llvm_ptr(loc=loc, ip=ip),
|
||||
Int32(1).ir_value(loc=loc, ip=ip),
|
||||
kind=nvvm.MBarrierTxnKind.ARRIVE,
|
||||
space=_SPACE_MAP[space],
|
||||
order=_MEMORY_ORDER_MAP[order],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def arrive_expect_tx(
|
||||
mbar, size, space: str = "cta", order: str = "relaxed", *, loc=None, ip=None
|
||||
):
|
||||
nvvm.mbarrier_txn(
|
||||
mbar.to_llvm_ptr(loc=loc, ip=ip),
|
||||
Int32(size).ir_value(loc=loc, ip=ip),
|
||||
kind=nvvm.MBarrierTxnKind.ARRIVE_EXPECT_TX,
|
||||
space=_SPACE_MAP[space],
|
||||
order=_MEMORY_ORDER_MAP[order],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
@@ -206,6 +206,7 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -1594,6 +1595,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
|
||||
),
|
||||
# Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel
|
||||
# row-parallel projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
|
||||
@@ -305,6 +305,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
run_gemm_rs: bool = False,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
@@ -470,7 +471,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.run_gemm_rs = run_gemm_rs
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj)
|
||||
if not self.run_gemm_rs:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -511,6 +521,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(core_attn_out):
|
||||
return gemm_rs(core_attn_out, self.o_proj.weight)
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
|
||||
@@ -134,6 +134,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
use_rope: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -270,6 +271,16 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.run_gemm_rs = run_gemm_rs
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj)
|
||||
if not self.run_gemm_rs:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
|
||||
# ---- Attention backend / impl / KV cache ----
|
||||
self.quant_config = quant_config
|
||||
@@ -541,10 +552,13 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
if gate is not None:
|
||||
attn_out = _gate_sigmoid_mul(attn_out, gate)
|
||||
|
||||
# ``o_proj`` (RowParallelLinear + out-of-place all-reduce) returns a
|
||||
# fresh private tensor, so return it directly rather than copying into a
|
||||
# caller buffer -- the previous ``output[:] = ...`` convention forced an
|
||||
# extra [num_tokens, hidden] copy per layer.
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(attn_out):
|
||||
return gemm_rs(attn_out, self.o_proj.weight)
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
|
||||
@@ -133,6 +133,7 @@ def shard_sequence_parallel_mlp(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
) -> bool:
|
||||
"""Whether to TP-shard a sequence-parallel MLP instead of replicating it.
|
||||
|
||||
@@ -140,7 +141,7 @@ def shard_sequence_parallel_mlp(
|
||||
the trade-off and :mod:`vllm.envs` for when it is worth enabling.
|
||||
"""
|
||||
enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT
|
||||
if not (use_sequence_parallel and enabled):
|
||||
if not (use_sequence_parallel and eligible and enabled):
|
||||
return False
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
return (
|
||||
@@ -148,6 +149,44 @@ def shard_sequence_parallel_mlp(
|
||||
)
|
||||
|
||||
|
||||
def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool:
|
||||
if not envs.VLLM_KIMI_K3_GEMM_RS:
|
||||
return False
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
if not use_sequence_parallel:
|
||||
reason = "sequence parallelism is disabled"
|
||||
elif parallel_config.use_ubatching:
|
||||
reason = "ubatching is enabled"
|
||||
elif vllm_config.model_config.dtype != torch.bfloat16:
|
||||
reason = "the model dtype is not BF16"
|
||||
elif not current_platform.is_cuda():
|
||||
reason = "the device is not CUDA"
|
||||
elif not current_platform.is_device_capability_family(100):
|
||||
reason = "the device is not SM100-family"
|
||||
elif not 1 < tp_size <= 16:
|
||||
reason = "TP size is not in the supported range 2-16"
|
||||
elif 128 % tp_size != 0:
|
||||
reason = "TP size does not divide 128"
|
||||
else:
|
||||
reason = None
|
||||
|
||||
if reason is not None:
|
||||
logger.warning_once("GEMM-RS was requested but is disabled because %s.", reason)
|
||||
return False
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import init_gemm_rs
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
init_gemm_rs(
|
||||
max_M=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
N=config.hidden_size,
|
||||
)
|
||||
logger.info_once("GEMM-RS is enabled.")
|
||||
return True
|
||||
|
||||
|
||||
class KimiMLP(nn.Module):
|
||||
"""Dense / shared-expert MLP, optionally TP-sharded under sequence parallel.
|
||||
|
||||
@@ -172,6 +211,8 @@ class KimiMLP(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
use_sequence_parallel: bool = False,
|
||||
can_shard_sequence_parallel: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
prefix: str = "",
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
@@ -182,7 +223,9 @@ class KimiMLP(nn.Module):
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
use_sequence_parallel,
|
||||
can_shard_sequence_parallel,
|
||||
)
|
||||
self.run_gemm_rs = self.shard_sequence_parallel and run_gemm_rs
|
||||
replicate = use_sequence_parallel and not self.shard_sequence_parallel
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -204,6 +247,15 @@ class KimiMLP(nn.Module):
|
||||
disable_tp=replicate,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.down_proj)
|
||||
if not self.run_gemm_rs:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
if hidden_act == "silu":
|
||||
self.act_fn = SiluAndMul()
|
||||
elif hidden_act == "situ":
|
||||
@@ -226,6 +278,14 @@ class KimiMLP(nn.Module):
|
||||
x = sp_all_gather(x)
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(x):
|
||||
return gemm_rs(x, self.down_proj.weight)
|
||||
|
||||
x, _ = self.down_proj(x)
|
||||
if self.shard_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
@@ -462,6 +522,7 @@ class KimiMoE(nn.Module):
|
||||
prefix: str = "",
|
||||
layer_idx: int = 0,
|
||||
use_sequence_parallel: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
@@ -546,6 +607,11 @@ class KimiMoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
# Only the MegaMoE path calls the shared experts directly; the
|
||||
# FusedMoE path below hands them to the runner, which fuses
|
||||
# their reduction and assumes the replicated layout.
|
||||
can_shard_sequence_parallel=self.use_mega_moe,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
@@ -767,6 +833,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
run_gemm_rs: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -801,6 +868,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
)
|
||||
self._self_attn_writes_output = False
|
||||
else:
|
||||
@@ -837,6 +905,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
aux_stream=aux_stream,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
)
|
||||
self._self_attn_writes_output = False
|
||||
|
||||
@@ -851,6 +920,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
layer_idx=layer_idx,
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
)
|
||||
self.mlp = self.block_sparse_moe
|
||||
else:
|
||||
@@ -861,6 +931,8 @@ class KimiDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
can_shard_sequence_parallel=True,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
activation_situ_beta=config.activation_situ_beta,
|
||||
activation_situ_linear_beta=config.activation_situ_linear_beta,
|
||||
)
|
||||
@@ -991,15 +1063,18 @@ class KimiDecoderLayer(nn.Module):
|
||||
)
|
||||
assert hidden_states is not None
|
||||
|
||||
M = None
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)
|
||||
# Remove SP padding before attention.
|
||||
hidden_states = hidden_states[: positions.shape[0]]
|
||||
M = hidden_states.shape[0]
|
||||
|
||||
# Attention.
|
||||
hidden_states = self._run_self_attn(positions, hidden_states)
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
# GEMM-RS returns the local sequence shard; standard O-proj preserves M.
|
||||
if self.use_sequence_parallel and hidden_states.shape[0] == M:
|
||||
# Add SP padding if needed, and then perform reduce scatter.
|
||||
hidden_states = sp_reduce_scatter(hidden_states)
|
||||
|
||||
@@ -1038,6 +1113,10 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
# GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP
|
||||
# ranks to belong to one NVLink domain.
|
||||
self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -1058,6 +1137,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
vllm_config,
|
||||
prefix,
|
||||
aux_stream=aux_stream,
|
||||
run_gemm_rs=self.run_gemm_rs,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
|
||||
@@ -0,0 +1,790 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SM100 BF16 GEMM with a fused tensor-parallel reduce-scatter."""
|
||||
|
||||
# Based on CUTLASS's Blackwell distributed GEMM-RS example at dcf215a.
|
||||
# See https://github.com/NVIDIA/cutlass/issues/3117 for memory semantics.
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Int32, Int64, Uint16, cute, utils
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, nvvm, vector
|
||||
from cutlass.cute.nvgpu import cpasync, tcgen05
|
||||
from cutlass.cute.runtime import make_fake_stream, make_fake_tensor, make_ptr, nullptr
|
||||
from cutlass.cutlass_dsl import dsl_user_op
|
||||
from cutlass.utils import get_smem_capacity_in_bytes
|
||||
|
||||
from vllm.cute_utils import _tcgen05, mbarrier, simple_tma_copy, to_cta0_smem
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def nanosleep(ns: int, *, loc=None, ip=None) -> None:
|
||||
nvvm.nanosleep(Int32(ns).ir_value(loc=loc, ip=ip), loc=loc, ip=ip)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:
|
||||
# NOTE: assume x is contiguous
|
||||
assert x.element_type == BFloat16
|
||||
vec_type = ".v4.bf16x2"
|
||||
|
||||
ptr = x.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
|
||||
asm = (
|
||||
"multimem.ld_reduce.relaxed.gpu.global.add.acc::f32"
|
||||
f"{vec_type} {{$0, $1, $2, $3}}, [$4];"
|
||||
)
|
||||
struct = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([Int32.mlir_type] * 4),
|
||||
[ptr],
|
||||
asm,
|
||||
"=r,=r,=r,=r,l",
|
||||
has_side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
vec = vector.from_elements(
|
||||
ir.VectorType.get([4], Int32.mlir_type, loc=loc),
|
||||
[
|
||||
llvm.extractvalue(Int32.mlir_type, struct, [i], loc=loc, ip=ip)
|
||||
for i in range(4)
|
||||
],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
ssa = cute.TensorSSA(vec, 4, Int32)
|
||||
|
||||
y = cute.make_rmem_tensor(4, Int32)
|
||||
y.store(ssa)
|
||||
return cute.recast_tensor(y, x.element_type)
|
||||
|
||||
|
||||
class Sm100GemmRsBF16:
|
||||
def __init__(
|
||||
self, rank: int, num_ranks: int, BN: int = 128, cta_group: int = 1
|
||||
) -> None:
|
||||
self.rank = rank
|
||||
self.num_ranks = num_ranks
|
||||
BM, BK = 128, 64
|
||||
self.cta_tile = (BM, BN, BK)
|
||||
self.cta_group = cta_group
|
||||
|
||||
smem_bytes = get_smem_capacity_in_bytes()
|
||||
self.stage_size = (BM + (BN // cta_group)) * BK * 2
|
||||
self.num_stages = smem_bytes // self.stage_size
|
||||
|
||||
@cute.jit
|
||||
def prepare_tma(
|
||||
self, tensor: cute.Tensor, BM: cutlass.Constexpr, BK: cutlass.Constexpr
|
||||
) -> cpasync.TmaInfo:
|
||||
tma_group = (
|
||||
tcgen05.CtaGroup.TWO if self.cta_group == 2 else tcgen05.CtaGroup.ONE
|
||||
)
|
||||
tma_op = cpasync.CopyBulkTensorTileG2SOp(cta_group=tma_group)
|
||||
swizzle_128b = cute.make_swizzle(3, 4, 3)
|
||||
layout = cute.make_layout(
|
||||
(BM, BK, self.num_stages),
|
||||
stride=(BK, 1, BM * BK),
|
||||
)
|
||||
layout = cute.make_composed_layout(swizzle_128b, 0, layout)
|
||||
return cpasync.make_tiled_tma_atom(tma_op, tensor, layout, (BM, BK))
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
A: cute.Tensor,
|
||||
B: cute.Tensor,
|
||||
partial_uc: cute.Tensor,
|
||||
partial_mc_ptr: cute.Pointer,
|
||||
output: cute.Tensor,
|
||||
flags_uc: cute.Tensor,
|
||||
flags_mc_ptr: cute.Pointer,
|
||||
peer_flag_ptr: cute.Pointer,
|
||||
grid_size: Int32,
|
||||
stream: CUstream,
|
||||
) -> None:
|
||||
N = B.shape[0]
|
||||
BM, BN, BK = self.cta_tile
|
||||
A_tma = self.prepare_tma(A, BM, BK)
|
||||
B_tma = self.prepare_tma(B, BN // self.cta_group, BK)
|
||||
padded_M = partial_uc.shape[0]
|
||||
partial_mc = cute.make_tensor(
|
||||
partial_mc_ptr,
|
||||
cute.make_layout((padded_M, N), stride=(N, 1)),
|
||||
)
|
||||
peer_flags = cute.make_tensor(
|
||||
peer_flag_ptr,
|
||||
cute.make_layout(self.num_ranks),
|
||||
)
|
||||
|
||||
grid = (grid_size, 1, 1)
|
||||
block = (10 * 32, 1, 1)
|
||||
cluster = (self.cta_group, 1, 1)
|
||||
self.kernel(
|
||||
A_tma,
|
||||
B_tma,
|
||||
partial_uc,
|
||||
partial_mc,
|
||||
output,
|
||||
flags_uc.iterator,
|
||||
flags_mc_ptr,
|
||||
peer_flags,
|
||||
).launch(grid=grid, block=block, cluster=cluster, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
A_tma: cpasync.TmaInfo,
|
||||
B_tma: cpasync.TmaInfo,
|
||||
partial_uc: cute.Tensor,
|
||||
partial_mc: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
flags_uc_ptr: cute.Pointer,
|
||||
flags_mc_ptr: cute.Pointer,
|
||||
peer_flags: cute.Tensor,
|
||||
) -> None:
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
raw_bid, _, _ = cute.arch.block_idx()
|
||||
num_bids, _, _ = cute.arch.grid_dim()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
|
||||
BM, BN, BK = self.cta_tile
|
||||
cta_group = self.cta_group
|
||||
num_stages = self.num_stages
|
||||
num_ranks = self.num_ranks
|
||||
|
||||
is_2cta = cta_group == 2
|
||||
cta_rank = raw_bid % self.cta_group
|
||||
num_tmem_stages = 512 // BN
|
||||
|
||||
smem = utils.SmemAllocator()
|
||||
sA = smem.allocate_tensor(
|
||||
BFloat16,
|
||||
A_tma.smem_layout.outer,
|
||||
byte_alignment=128,
|
||||
swizzle=A_tma.smem_layout.inner,
|
||||
)
|
||||
sB = smem.allocate_tensor(
|
||||
BFloat16,
|
||||
B_tma.smem_layout.outer,
|
||||
byte_alignment=128,
|
||||
swizzle=B_tma.smem_layout.inner,
|
||||
)
|
||||
tma_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||
tmem_full_mbar = smem.allocate_array(Int64, num_tmem_stages)
|
||||
tmem_empty_mbar = smem.allocate_array(Int64, num_tmem_stages)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
# Named barriers
|
||||
BAR_TMEM_ALLOC = 1
|
||||
BAR_EPILOGUE = 2
|
||||
BAR_COMM = 3
|
||||
|
||||
M, K = A_tma.tma_tensor.shape
|
||||
N, _ = B_tma.tma_tensor.shape
|
||||
grid_m = cute.ceil_div(M, BM)
|
||||
# Keep 2-CTA clusters within a single N tile.
|
||||
grid_m = cute.ceil_div(grid_m, cta_group) * cta_group
|
||||
grid_n = cute.ceil_div(N, BN)
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_full_mbar + i, cta_group)
|
||||
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
|
||||
for i in cutlass.range_constexpr(num_tmem_stages):
|
||||
cute.arch.mbarrier_init(tmem_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(tmem_empty_mbar + i, 128 * cta_group)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(A_tma.atom)
|
||||
cpasync.prefetch_descriptor(B_tma.atom)
|
||||
|
||||
if cutlass.const_expr(is_2cta):
|
||||
cute.arch.cluster_arrive_relaxed()
|
||||
cute.arch.cluster_wait()
|
||||
else:
|
||||
cute.arch.sync_threads()
|
||||
|
||||
total_tiles = grid_m * grid_n
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
tma_stage = 0
|
||||
parity = 1
|
||||
|
||||
if cutlass.const_expr(is_2cta):
|
||||
tma_full_mbar_ = to_cta0_smem(tma_full_mbar)
|
||||
else:
|
||||
tma_full_mbar_ = tma_full_mbar
|
||||
|
||||
# Select global-memory tiles.
|
||||
# [(BM, BK), (M/BM, K/BK)]
|
||||
gA_tiles = cute.zipped_divide(A_tma.tma_tensor, (BM, BK))
|
||||
gB_tiles = cute.zipped_divide(B_tma.tma_tensor, (BN // cta_group, BK))
|
||||
|
||||
for bid in range(raw_bid, total_tiles, num_bids):
|
||||
bid_m = bid % grid_m
|
||||
bid_n = bid // grid_m
|
||||
if cutlass.const_expr(cta_group == 2):
|
||||
bid_n = bid_n * cta_group + cta_rank
|
||||
|
||||
for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1):
|
||||
mbar = tma_full_mbar_ + tma_stage
|
||||
cute.arch.mbarrier_wait(tma_empty_mbar + tma_stage, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
mbarrier.arrive_expect_tx(mbar, self.stage_size, "cluster")
|
||||
simple_tma_copy(
|
||||
A_tma.atom,
|
||||
gA_tiles[None, (bid_m, iter_k)],
|
||||
sA[None, None, tma_stage],
|
||||
mbar,
|
||||
)
|
||||
simple_tma_copy(
|
||||
B_tma.atom,
|
||||
gB_tiles[None, (bid_n, iter_k)],
|
||||
sB[None, None, tma_stage],
|
||||
mbar,
|
||||
)
|
||||
|
||||
tma_stage = (tma_stage + 1) % num_stages
|
||||
if tma_stage == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32)
|
||||
|
||||
if cta_rank == 0:
|
||||
tma_stage = 0
|
||||
tma_full_parity = 0
|
||||
tmem_stage = 0
|
||||
tmem_empty_parity = 1
|
||||
|
||||
MMA_M = BM * cta_group
|
||||
MMA_N = BN
|
||||
idesc = _tcgen05.make_bf16_idesc(MMA_M, MMA_N)
|
||||
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
|
||||
multicast_mask = Uint16((1 << self.cta_group) - 1)
|
||||
|
||||
for bid in range(raw_bid, total_tiles, num_bids):
|
||||
cute.arch.mbarrier_wait(
|
||||
tmem_empty_mbar + tmem_stage, tmem_empty_parity
|
||||
)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for iter_k in cutlass.range(cute.ceil_div(K, BK), unroll=1):
|
||||
d_tmem = BN * tmem_stage
|
||||
a_addr = sA[None, None, tma_stage].iterator.toint()
|
||||
b_addr = sB[None, None, tma_stage].iterator.toint()
|
||||
a_desc = sdesc | (a_addr >> 4)
|
||||
b_desc = sdesc | (b_addr >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(
|
||||
tma_full_mbar + tma_stage, tma_full_parity
|
||||
)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for mma_k in cutlass.range_constexpr(BK // 16):
|
||||
enable_d = iter_k > 0 or mma_k > 0
|
||||
_tcgen05.mma_f16(
|
||||
d_tmem, a_desc, b_desc, idesc, enable_d, cta_group
|
||||
)
|
||||
a_desc += 32 >> 4
|
||||
b_desc += 32 >> 4
|
||||
_tcgen05.commit(
|
||||
tma_empty_mbar + tma_stage, multicast_mask, cta_group
|
||||
)
|
||||
|
||||
tma_stage = (tma_stage + 1) % num_stages
|
||||
if tma_stage == 0:
|
||||
tma_full_parity ^= 1
|
||||
|
||||
_tcgen05.commit(
|
||||
tmem_full_mbar + tmem_stage, multicast_mask, cta_group
|
||||
)
|
||||
|
||||
tmem_stage = (tmem_stage + 1) % num_tmem_stages
|
||||
if tmem_stage == 0:
|
||||
tmem_empty_parity ^= 1
|
||||
|
||||
elif warp_id >= 4:
|
||||
# Communication warps
|
||||
# Keep epilogue in warps 0-3 and communication in warps 4-7.
|
||||
# Swapping the warpgroups can hang due to warp scheduling.
|
||||
tid_ = tid % 128
|
||||
local_M = output.shape[0]
|
||||
|
||||
# Offset in the [M, N] GEMM result.
|
||||
rank_start = self.rank * local_M
|
||||
rank_end = min(rank_start + local_M, M)
|
||||
|
||||
st_atom = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=128
|
||||
)
|
||||
|
||||
# Each thread issues one BF16x8 multimem reduction per vector.
|
||||
vec_width = 8
|
||||
vec_cols = BN // vec_width
|
||||
max_tile_rows = BM // num_ranks
|
||||
vecs_per_tile = max_tile_rows * vec_cols
|
||||
partial_vecs = cute.zipped_divide(partial_mc, (1, vec_width))
|
||||
output_vecs = cute.zipped_divide(output, (1, vec_width))
|
||||
|
||||
for tile_id in range(raw_bid, total_tiles, num_bids):
|
||||
rs_bid_m = tile_id % grid_m
|
||||
bid_n = tile_id // grid_m
|
||||
|
||||
local_row_start = rs_bid_m * local_M // grid_m
|
||||
local_row_end = (rs_bid_m + 1) * local_M // grid_m
|
||||
global_row_start = rank_start + local_row_start
|
||||
global_row_end = min(rank_start + local_row_end, M)
|
||||
|
||||
if global_row_start < M:
|
||||
# Map the current RS tile to the required GEMM tiles.
|
||||
# Since an RS tile is smaller than a GEMM tile, each RS tile
|
||||
# can overlap at most 2 GEMM tiles.
|
||||
gemm_bid_m0 = global_row_start // BM
|
||||
gemm_bid_m1 = max(global_row_end - 1, global_row_start) // BM
|
||||
|
||||
# Poll local L2 with relaxed GPU-scope loads. The following
|
||||
# multimem reduction reads every rank's L2 directly, so no
|
||||
# acquire fence or L1 invalidation is needed.
|
||||
if tid_ == 0:
|
||||
# Poll the 1st GEMM tile.
|
||||
flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m0
|
||||
arrivals = cute.arch.load(
|
||||
flag_ptr, Int32, sem="relaxed", scope="gpu"
|
||||
)
|
||||
while arrivals < num_ranks:
|
||||
nanosleep(64)
|
||||
arrivals = cute.arch.load(
|
||||
flag_ptr, Int32, sem="relaxed", scope="gpu"
|
||||
)
|
||||
elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0:
|
||||
# Poll the 2nd GEMM tile with another warp if needed.
|
||||
flag_ptr = flags_uc_ptr + bid_n * grid_m + gemm_bid_m1
|
||||
arrivals = cute.arch.load(
|
||||
flag_ptr, Int32, sem="relaxed", scope="gpu"
|
||||
)
|
||||
while arrivals < num_ranks:
|
||||
nanosleep(64)
|
||||
arrivals = cute.arch.load(
|
||||
flag_ptr, Int32, sem="relaxed", scope="gpu"
|
||||
)
|
||||
cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128)
|
||||
|
||||
# Issue multimem.ld_reduce before storing any result.
|
||||
reduced_vecs = []
|
||||
for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128):
|
||||
vec_idx = tid_ + vec_iter * 128
|
||||
|
||||
local_row = local_row_start + vec_idx // vec_cols
|
||||
global_row = rank_start + local_row
|
||||
col = bid_n * vec_cols + vec_idx % vec_cols
|
||||
|
||||
reduced_vec = cute.make_rmem_tensor(vec_width, BFloat16)
|
||||
if local_row < local_row_end and global_row < M:
|
||||
tmp = multimem_ld_reduce_16B(
|
||||
partial_vecs[None, (global_row, col)]
|
||||
)
|
||||
reduced_vec.store(tmp.load())
|
||||
reduced_vecs.append(reduced_vec)
|
||||
|
||||
# Store the result to local L2.
|
||||
for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128):
|
||||
vec_idx = tid_ + vec_iter * 128
|
||||
|
||||
local_row = local_row_start + vec_idx // vec_cols
|
||||
global_row = rank_start + local_row
|
||||
col = bid_n * vec_cols + vec_idx % vec_cols
|
||||
|
||||
if local_row < local_row_end and global_row < M:
|
||||
cute.copy(
|
||||
st_atom,
|
||||
reduced_vecs[vec_iter],
|
||||
output_vecs[None, (local_row, col)],
|
||||
)
|
||||
cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128)
|
||||
|
||||
# Release each GEMM tile consumed by this RS tile. The last
|
||||
# consumer resets its producer flag for the next launch.
|
||||
def release_gemm_tile(
|
||||
gemm_bid_m,
|
||||
tile_M,
|
||||
logical_M,
|
||||
rank_start,
|
||||
rank_end,
|
||||
local_M,
|
||||
grid_m,
|
||||
tile_N,
|
||||
flags,
|
||||
num_ranks,
|
||||
):
|
||||
gemm_start = gemm_bid_m * tile_M
|
||||
gemm_end = min(gemm_start + tile_M, logical_M)
|
||||
local_start = max(gemm_start, rank_start) - rank_start
|
||||
local_end = min(gemm_end, rank_end) - rank_start
|
||||
|
||||
# Compute the number of consumers to identify the last
|
||||
# arrival, which resets the producer flag.
|
||||
first_rs_tile = (
|
||||
cute.ceil_div((local_start + 1) * grid_m, local_M) - 1
|
||||
)
|
||||
last_rs_tile = cute.ceil_div(local_end * grid_m, local_M) - 1
|
||||
num_consumers = last_rs_tile - first_rs_tile + 1
|
||||
|
||||
flag_ptr = flags + tile_N * grid_m + gemm_bid_m
|
||||
old_count = cute.arch.atomic_add(
|
||||
flag_ptr, Int32(1), sem="relaxed", scope="gpu"
|
||||
)
|
||||
if old_count == num_ranks + num_consumers - 1:
|
||||
cute.arch.store(
|
||||
flag_ptr, Int32(0), sem="relaxed", scope="gpu"
|
||||
)
|
||||
|
||||
if tid_ == 0:
|
||||
# Arrive at the 1st GEMM tile.
|
||||
release_gemm_tile(
|
||||
gemm_bid_m0,
|
||||
BM,
|
||||
M,
|
||||
rank_start,
|
||||
rank_end,
|
||||
local_M,
|
||||
grid_m,
|
||||
bid_n,
|
||||
flags_uc_ptr,
|
||||
num_ranks,
|
||||
)
|
||||
elif tid_ == 32 and gemm_bid_m1 != gemm_bid_m0:
|
||||
# Arrive at the 2nd GEMM tile if necessary.
|
||||
release_gemm_tile(
|
||||
gemm_bid_m1,
|
||||
BM,
|
||||
M,
|
||||
rank_start,
|
||||
rank_end,
|
||||
local_M,
|
||||
grid_m,
|
||||
bid_n,
|
||||
flags_uc_ptr,
|
||||
num_ranks,
|
||||
)
|
||||
|
||||
# Exit barrier. GPU scope is sufficient because the kernel writes
|
||||
# local global memory and only needs to flush it to local L2.
|
||||
cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128)
|
||||
if tid_ == 0:
|
||||
exit_flag = total_tiles + raw_bid
|
||||
utils.distributed.multimem_red_add1(
|
||||
flags_mc_ptr + exit_flag, order="release", scope="gpu"
|
||||
)
|
||||
utils.distributed.spin_lock_atom_cas_acquire_wait(
|
||||
flags_uc_ptr + exit_flag,
|
||||
expected_val=num_ranks,
|
||||
reset_val=0,
|
||||
scope="gpu",
|
||||
)
|
||||
|
||||
else:
|
||||
# Epilogue warps
|
||||
warp_id_ = warp_id % 4
|
||||
tid_ = tid % 128
|
||||
|
||||
peer_flag_bases = cute.make_rmem_tensor(num_ranks, Int64)
|
||||
if tid_ == 0:
|
||||
for rank in cutlass.range_constexpr(num_ranks):
|
||||
peer_flag_bases[rank] = cute.arch.load(
|
||||
(peer_flags.iterator + rank).llvm_ptr,
|
||||
Int64,
|
||||
)
|
||||
|
||||
if warp_id_ == 0:
|
||||
_tcgen05.alloc(taddr, cta_group)
|
||||
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=5 * 32)
|
||||
|
||||
WIDTH = cutlass.const_expr(16)
|
||||
partial_vecs = cute.zipped_divide(partial_uc, (1, WIDTH))
|
||||
|
||||
bf16x16_atom = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(),
|
||||
BFloat16,
|
||||
num_bits_per_copy=256,
|
||||
l1c_evict_priority=cute.nvgpu.CacheEvictionPriority.NO_ALLOCATE,
|
||||
)
|
||||
|
||||
tmem_stage = 0
|
||||
parity = 0
|
||||
|
||||
if cutlass.const_expr(is_2cta):
|
||||
tmem_empty_mbar_ = to_cta0_smem(tmem_empty_mbar)
|
||||
else:
|
||||
tmem_empty_mbar_ = tmem_empty_mbar
|
||||
|
||||
for bid in range(raw_bid, total_tiles, num_bids):
|
||||
bid_m = bid % grid_m
|
||||
bid_n = bid // grid_m
|
||||
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(tmem_full_mbar + tmem_stage, parity)
|
||||
cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = tmem_stage * BN + i * WIDTH
|
||||
regs = _tcgen05.ld(warp_id_ * 32, tcol, "32x32b", WIDTH)
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
if cutlass.const_expr(i == BN // WIDTH - 1):
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
mbarrier.arrive(tmem_empty_mbar_ + tmem_stage, "cluster")
|
||||
|
||||
tmp = cute.make_rmem_tensor(WIDTH, BFloat16)
|
||||
tmp.store(regs.to(BFloat16))
|
||||
|
||||
global_row = bid_m * BM + tid_
|
||||
if global_row < M:
|
||||
coord = (global_row, bid_n * (BN // WIDTH) + i)
|
||||
cute.copy(bf16x16_atom, tmp, partial_vecs[None, coord])
|
||||
|
||||
cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128)
|
||||
|
||||
# Signal GEMM completion. GPU scope is sufficient because the
|
||||
# partial output only needs to be flushed to local L2.
|
||||
if tid_ == 0 and bid_m * BM < M:
|
||||
gemm_start = bid_m * BM
|
||||
gemm_end = min(gemm_start + BM, M)
|
||||
first_owner = gemm_start // output.shape[0]
|
||||
last_owner = (gemm_end - 1) // output.shape[0]
|
||||
|
||||
# signal to all consuming ranks (can be more than 1)
|
||||
for rank in cutlass.range_constexpr(num_ranks):
|
||||
if first_owner <= rank and rank <= last_owner:
|
||||
ptr = cute.make_ptr(
|
||||
Int32,
|
||||
peer_flag_bases[rank],
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=16,
|
||||
)
|
||||
ptr += bid_m + bid_n * grid_m
|
||||
utils.distributed.red_add1(
|
||||
ptr, order="release", scope="gpu"
|
||||
)
|
||||
|
||||
tmem_stage = (tmem_stage + 1) % num_tmem_stages
|
||||
if tmem_stage == 0:
|
||||
parity ^= 1
|
||||
|
||||
if cutlass.const_expr(is_2cta):
|
||||
cute.arch.cluster_arrive_relaxed()
|
||||
cute.arch.cluster_wait()
|
||||
else:
|
||||
cute.arch.barrier(barrier_id=BAR_EPILOGUE, number_of_threads=128)
|
||||
if warp_id_ == 0:
|
||||
_tcgen05.dealloc(cta_group)
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(rank: int, num_ranks: int, BN: int, cta_group: int):
|
||||
M = cute.sym_int()
|
||||
padded_M = cute.sym_int()
|
||||
N = cute.sym_int()
|
||||
K = cute.sym_int()
|
||||
local_M = cute.sym_int()
|
||||
num_flags = cute.sym_int()
|
||||
|
||||
A = make_fake_tensor(
|
||||
BFloat16, (M, K), (cute.sym_int64(divisibility=8), 1), assumed_align=16
|
||||
)
|
||||
B = make_fake_tensor(
|
||||
BFloat16, (N, K), (cute.sym_int64(divisibility=8), 1), assumed_align=16
|
||||
)
|
||||
partial = make_fake_tensor(
|
||||
BFloat16,
|
||||
(padded_M, N),
|
||||
(cute.sym_int64(divisibility=16), 1),
|
||||
assumed_align=32,
|
||||
)
|
||||
partial_mc_ptr = nullptr(BFloat16, cute.AddressSpace.gmem, assumed_align=32)
|
||||
output = make_fake_tensor(
|
||||
BFloat16,
|
||||
(local_M, N),
|
||||
(cute.sym_int64(divisibility=16), 1),
|
||||
assumed_align=32,
|
||||
)
|
||||
flags = make_fake_tensor(Int32, (num_flags,), (1,), assumed_align=16)
|
||||
flags_mc_ptr = nullptr(Int32, cute.AddressSpace.gmem, assumed_align=16)
|
||||
peer_flag_ptr = nullptr(Int64, cute.AddressSpace.gmem, assumed_align=8)
|
||||
|
||||
stream = make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
kernel = Sm100GemmRsBF16(rank, num_ranks, BN, cta_group)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
A,
|
||||
B,
|
||||
partial,
|
||||
partial_mc_ptr,
|
||||
output,
|
||||
flags,
|
||||
flags_mc_ptr,
|
||||
peer_flag_ptr,
|
||||
128,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
class GemmRS:
|
||||
"""Own the max-sized symmetric workspace for Kimi-K3 GEMM-RS launches.
|
||||
|
||||
All TP ranks must belong to one NVLink domain for multimem instructions.
|
||||
"""
|
||||
|
||||
def __init__(self, *, max_M: int, N: int) -> None:
|
||||
tp_group = get_tp_group()
|
||||
group = tp_group.device_group
|
||||
rank = tp_group.rank_in_group
|
||||
world_size = tp_group.world_size
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
|
||||
assert 1 < world_size <= 16
|
||||
assert 128 % world_size == 0
|
||||
assert max_M >= 128 and N % 128 == 0
|
||||
|
||||
max_M = (max_M + world_size - 1) // world_size * world_size
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.max_M = max_M
|
||||
self.N = N
|
||||
self.device = device
|
||||
|
||||
self.partial = symm_mem.empty((max_M, N), dtype=torch.bfloat16, device=device)
|
||||
self.partial_handle = symm_mem.rendezvous(self.partial, group)
|
||||
self.partial_mc_ptr = make_ptr(
|
||||
BFloat16,
|
||||
self.partial_handle.multicast_ptr,
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=32,
|
||||
)
|
||||
|
||||
grid_m = (max_M + 127) // 128
|
||||
cta_group = 2 if max_M >= 1024 or grid_m % 2 == 0 else 1
|
||||
grid_m = (grid_m + cta_group - 1) // cta_group * cta_group
|
||||
self.num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
max_flags = grid_m * (N // 128) + self.num_sms
|
||||
self.flags = symm_mem.empty(max_flags, dtype=torch.int32, device=device)
|
||||
self.flags_handle = symm_mem.rendezvous(self.flags, group)
|
||||
self.flags.zero_()
|
||||
self.flags_mc_ptr = make_ptr(
|
||||
Int32,
|
||||
self.flags_handle.multicast_ptr,
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=16,
|
||||
)
|
||||
self.peer_flag_ptr = make_ptr(
|
||||
Int64,
|
||||
self.flags_handle.buffer_ptrs_dev,
|
||||
cute.AddressSpace.gmem,
|
||||
assumed_align=8,
|
||||
)
|
||||
|
||||
assert self.partial_handle.multicast_ptr != 0
|
||||
assert self.flags_handle.multicast_ptr != 0
|
||||
torch.accelerator.synchronize(device)
|
||||
tp_group.barrier()
|
||||
|
||||
def can_run(self, linear: LinearBase) -> bool:
|
||||
# Validate projection-invariant requirements once during model init.
|
||||
# only supports BF16 for now
|
||||
if not isinstance(linear.quant_method, UnquantizedLinearMethod):
|
||||
return False
|
||||
w = linear.weight
|
||||
if w.ndim != 2:
|
||||
return False
|
||||
K = w.shape[1]
|
||||
return (
|
||||
w.shape == (self.N, K)
|
||||
and K % 64 == 0
|
||||
and w.dtype == torch.bfloat16
|
||||
and w.device == self.device
|
||||
and w.is_contiguous()
|
||||
)
|
||||
|
||||
def should_run(self, x: torch.Tensor) -> bool:
|
||||
# Small-M shapes are supported but faster on the existing LL path.
|
||||
return x.shape[0] >= 128
|
||||
|
||||
def __call__(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
|
||||
assert x.ndim == 2
|
||||
M, K = x.shape
|
||||
assert 0 < M <= self.max_M
|
||||
assert w.shape == (self.N, K) and K % 64 == 0
|
||||
assert w.dtype == torch.bfloat16
|
||||
assert w.device == self.device
|
||||
assert w.is_contiguous()
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert x.device == self.device
|
||||
assert x.is_contiguous()
|
||||
N = w.shape[0]
|
||||
padded_M = (M + self.world_size - 1) // self.world_size
|
||||
padded_M *= self.world_size
|
||||
local_M = padded_M // self.world_size
|
||||
|
||||
grid_m = (M + 127) // 128
|
||||
# Avoid padding small odd grids; 2-CTA wins consistently for M >= 1024.
|
||||
cta_group = 2 if M >= 1024 or grid_m % 2 == 0 else 1
|
||||
grid_m = (grid_m + cta_group - 1) // cta_group * cta_group
|
||||
BN = 256 if M * K >= 24 * 1024 * 1024 else 128
|
||||
assert N % BN == 0
|
||||
|
||||
num_tiles = grid_m * (N // BN)
|
||||
num_ctas = min(num_tiles, self.num_sms)
|
||||
num_ctas = num_ctas // cta_group * cta_group
|
||||
assert self.flags.numel() >= num_tiles + num_ctas
|
||||
|
||||
output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device)
|
||||
compiled = Sm100GemmRsBF16.compile(
|
||||
self.rank,
|
||||
self.world_size,
|
||||
BN,
|
||||
cta_group,
|
||||
)
|
||||
compiled(
|
||||
x,
|
||||
w,
|
||||
self.partial[:padded_M],
|
||||
self.partial_mc_ptr,
|
||||
output,
|
||||
self.flags,
|
||||
self.flags_mc_ptr,
|
||||
self.peer_flag_ptr,
|
||||
num_ctas,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
_gemm_rs: GemmRS | None = None
|
||||
|
||||
|
||||
def init_gemm_rs(max_M: int, N: int) -> None:
|
||||
"""Collectively initialize the process-wide GEMM-RS state."""
|
||||
global _gemm_rs
|
||||
if _gemm_rs is not None:
|
||||
assert _gemm_rs.max_M >= max_M and _gemm_rs.N == N
|
||||
return
|
||||
_gemm_rs = GemmRS(max_M=max_M, N=N)
|
||||
|
||||
|
||||
def get_gemm_rs() -> GemmRS:
|
||||
assert _gemm_rs is not None, "GEMM-RS is not initialized"
|
||||
return _gemm_rs
|
||||
Reference in New Issue
Block a user