[Kimi-K3] Extend GEMM-RS to GEMM-AR (#53053)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
wzhao18
parent
b00f475f09
commit
2785c72a14
@@ -244,11 +244,20 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||
- pytest -v -s tests/distributed/test_nccl_symm_mem.py
|
||||
- pytest -v -s tests/kernels/test_kimi_k3_gemm_rs.py
|
||||
- pytest -v -s tests/v1/distributed/test_dbo.py
|
||||
- pytest -v -s tests/distributed/test_mnnvl_alltoall.py
|
||||
|
||||
|
||||
- label: ":nvidia: (B200) Kimi-K3 GEMM-RS/AR"
|
||||
key: kimi-k3-gemm-rs-ar-2xb200
|
||||
timeout_in_minutes: 20
|
||||
device: b200-k8s
|
||||
working_dir: "/vllm-workspace/"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/models/kimi_k3/nvidia/ops/cute_dsl/gemm_rs_ar.py
|
||||
- tests/kernels/test_kimi_k3_gemm_rs_ar.py
|
||||
commands:
|
||||
- pytest -v -s tests/kernels/test_kimi_k3_gemm_rs_ar.py
|
||||
|
||||
- label: ":nvidia: (L4) Distributed 2-Node"
|
||||
key: 2-node-test-4-gpus
|
||||
|
||||
+114
-78
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the SM100 Kimi-K3 GEMM-RS kernel.
|
||||
"""Benchmark the SM100 Kimi-K3 GEMM-RS/AR kernel.
|
||||
|
||||
All ranks must belong to one NVLink domain. For example, run a TP8 sweep with:
|
||||
|
||||
torchrun --nproc-per-node=8 \
|
||||
benchmarks/kernels/benchmark_kimi_k3_gemm_rs.py
|
||||
benchmarks/kernels/benchmark_kimi_k3_gemm_rs_ar.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -26,7 +26,7 @@ from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr
|
||||
|
||||
# Shared-expert down-proj and attention O-proj.
|
||||
_KIMI_K3_PROJECTION_K = (6144, 12288)
|
||||
@@ -41,6 +41,12 @@ class Candidate:
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("rs", "ar"),
|
||||
default="rs",
|
||||
help="Collective mode to benchmark.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--m",
|
||||
type=int,
|
||||
@@ -138,7 +144,8 @@ def valid_rows(M: int, local_M: int, rank: int) -> int:
|
||||
|
||||
|
||||
def benchmark_shape(
|
||||
gemm_rs: GemmRS,
|
||||
gemm_rs_ar: GemmRsAr,
|
||||
mode: str,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
@@ -148,7 +155,8 @@ def benchmark_shape(
|
||||
device_group: dist.ProcessGroup,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
device_barrier: Callable[[], None],
|
||||
) -> dict[str, float | int]:
|
||||
) -> dict[str, float | int | str]:
|
||||
all_reduce = mode == "ar"
|
||||
world_size = dist.get_world_size(device_group)
|
||||
rank = dist.get_rank(device_group)
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
@@ -169,17 +177,19 @@ def benchmark_shape(
|
||||
partial = torch.empty((padded_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_partial = symm_mem.empty((padded_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_partial_handle = symm_mem.rendezvous(symm_partial, device_group)
|
||||
rs_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)]
|
||||
symm_rs_inputs = []
|
||||
symm_rs_handles = []
|
||||
collective_inputs = [torch.empty_like(partial) for _ in range(num_workspaces)]
|
||||
symm_collective_inputs = []
|
||||
symm_collective_handles = []
|
||||
for _ in range(num_workspaces):
|
||||
rs_input = symm_mem.empty(
|
||||
collective_input = symm_mem.empty(
|
||||
(padded_M, N),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
symm_rs_inputs.append(rs_input)
|
||||
symm_rs_handles.append(symm_mem.rendezvous(rs_input, device_group))
|
||||
symm_collective_inputs.append(collective_input)
|
||||
symm_collective_handles.append(
|
||||
symm_mem.rendezvous(collective_input, device_group)
|
||||
)
|
||||
torch_output = torch.empty((local_M, N), dtype=torch.bfloat16, device=device)
|
||||
symm_output = torch.empty_like(torch_output)
|
||||
gemm_output = torch.empty((M, N), dtype=torch.bfloat16, device=device)
|
||||
@@ -188,21 +198,27 @@ def benchmark_shape(
|
||||
partial[M:].zero_()
|
||||
symm_partial[M:].zero_()
|
||||
|
||||
def make_torch_ring_ll_gemm_rs(
|
||||
def make_torch_ring_ll_gemm_collective(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
torch.mm(x, weight.T, out=partial[:M])
|
||||
if all_reduce:
|
||||
dist.all_reduce(partial, group=device_group)
|
||||
return partial[:M]
|
||||
dist.reduce_scatter_single(torch_output, partial, group=device_group)
|
||||
return torch_output
|
||||
|
||||
return run
|
||||
|
||||
def make_torch_ldmc_gemm_rs(
|
||||
def make_torch_ldmc_gemm_collective(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
torch.mm(x, weight.T, out=symm_partial[:M])
|
||||
if all_reduce:
|
||||
dist.all_reduce(symm_partial, group=device_group)
|
||||
return symm_partial[:M]
|
||||
dist.reduce_scatter_single(
|
||||
symm_output,
|
||||
symm_partial,
|
||||
@@ -212,11 +228,11 @@ def benchmark_shape(
|
||||
|
||||
return run
|
||||
|
||||
def make_fused_gemm_rs(
|
||||
def make_fused_gemm_collective(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
return gemm_rs(x, weight)
|
||||
return gemm_rs_ar(x, weight)
|
||||
|
||||
return run
|
||||
|
||||
@@ -229,18 +245,30 @@ def benchmark_shape(
|
||||
|
||||
return run
|
||||
|
||||
def make_ring_ll_rs(rs_input: torch.Tensor) -> Callable[[], torch.Tensor]:
|
||||
def make_ring_ll_collective(
|
||||
collective_input: torch.Tensor,
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
dist.reduce_scatter_single(torch_output, rs_input, group=device_group)
|
||||
if all_reduce:
|
||||
dist.all_reduce(collective_input, group=device_group)
|
||||
return collective_input
|
||||
dist.reduce_scatter_single(
|
||||
torch_output, collective_input, group=device_group
|
||||
)
|
||||
return torch_output
|
||||
|
||||
return run
|
||||
|
||||
def make_ldmc_rs(
|
||||
rs_input: torch.Tensor,
|
||||
def make_ldmc_collective(
|
||||
collective_input: torch.Tensor,
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
def run() -> torch.Tensor:
|
||||
dist.reduce_scatter_single(symm_output, rs_input, group=device_group)
|
||||
if all_reduce:
|
||||
dist.all_reduce(collective_input, group=device_group)
|
||||
return collective_input
|
||||
dist.reduce_scatter_single(
|
||||
symm_output, collective_input, group=device_group
|
||||
)
|
||||
return symm_output
|
||||
|
||||
return run
|
||||
@@ -248,15 +276,15 @@ def benchmark_shape(
|
||||
candidates = (
|
||||
Candidate(
|
||||
"ring_ll_us",
|
||||
[make_torch_ring_ll_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
[make_torch_ring_ll_gemm_collective(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"ldmc_us",
|
||||
[make_torch_ldmc_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
[make_torch_ldmc_gemm_collective(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"gemm_rs_us",
|
||||
[make_fused_gemm_rs(x, w) for x, w in zip(inputs, weights)],
|
||||
"gemm_rs_ar_us",
|
||||
[make_fused_gemm_collective(x, w) for x, w in zip(inputs, weights)],
|
||||
),
|
||||
Candidate(
|
||||
"torch_gemm_us",
|
||||
@@ -264,19 +292,19 @@ def benchmark_shape(
|
||||
check_correctness=False,
|
||||
),
|
||||
Candidate(
|
||||
"ring_ll_rs_us",
|
||||
[make_ring_ll_rs(x) for x in rs_inputs],
|
||||
"ring_ll_collective_us",
|
||||
[make_ring_ll_collective(x) for x in collective_inputs],
|
||||
check_correctness=False,
|
||||
),
|
||||
Candidate(
|
||||
"ldmc_rs_us",
|
||||
[make_ldmc_rs(x) for x in symm_rs_inputs],
|
||||
"ldmc_collective_us",
|
||||
[make_ldmc_collective(x) for x in symm_collective_inputs],
|
||||
check_correctness=False,
|
||||
),
|
||||
)
|
||||
|
||||
expected = candidates[0].runs[0]()
|
||||
rows = valid_rows(M, local_M, rank)
|
||||
rows = M if all_reduce else valid_rows(M, local_M, rank)
|
||||
for candidate in candidates[1:]:
|
||||
if not candidate.check_correctness:
|
||||
continue
|
||||
@@ -290,7 +318,7 @@ def benchmark_shape(
|
||||
)
|
||||
|
||||
candidate_graphs = {}
|
||||
graph_keepalive: list[object] = [symm_partial_handle, *symm_rs_handles]
|
||||
graph_keepalive: list[object] = [symm_partial_handle, *symm_collective_handles]
|
||||
for candidate in candidates:
|
||||
stream = torch.cuda.Stream()
|
||||
bundles = [capture_graph(run, stream, cpu_group) for run in candidate.runs]
|
||||
@@ -306,70 +334,71 @@ def benchmark_shape(
|
||||
device_barrier,
|
||||
)
|
||||
|
||||
best_nccl_rs_us = min(times["ring_ll_rs_us"], times["ldmc_rs_us"])
|
||||
best_nccl_us = min(times["ring_ll_collective_us"], times["ldmc_collective_us"])
|
||||
return {
|
||||
"mode": mode.upper(),
|
||||
"M": M,
|
||||
"N": N,
|
||||
"K": K,
|
||||
**times,
|
||||
"best_nccl_rs_us": best_nccl_rs_us,
|
||||
"speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_us"],
|
||||
"speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_us"],
|
||||
"best_nccl_us": best_nccl_us,
|
||||
"speedup_vs_ring_ll": times["ring_ll_us"] / times["gemm_rs_ar_us"],
|
||||
"speedup_vs_ldmc": times["ldmc_us"] / times["gemm_rs_ar_us"],
|
||||
}
|
||||
|
||||
|
||||
def print_results(results: list[dict[str, float | int]]) -> None:
|
||||
def print_results(results: list[dict[str, float | int | str]]) -> None:
|
||||
results_df = pd.DataFrame(results)
|
||||
end_to_end = results_df[
|
||||
[
|
||||
"M",
|
||||
"N",
|
||||
"K",
|
||||
"ring_ll_us",
|
||||
"ldmc_us",
|
||||
"gemm_rs_us",
|
||||
"speedup_vs_ring_ll",
|
||||
"speedup_vs_ldmc",
|
||||
collective = str(results_df["mode"].iloc[0])
|
||||
end_to_end = (
|
||||
results_df[
|
||||
[
|
||||
"M",
|
||||
"N",
|
||||
"K",
|
||||
"ring_ll_us",
|
||||
"ldmc_us",
|
||||
"gemm_rs_ar_us",
|
||||
"speedup_vs_ring_ll",
|
||||
"speedup_vs_ldmc",
|
||||
]
|
||||
]
|
||||
].rename(
|
||||
columns={
|
||||
"ring_ll_us": "Torch GEMM + NCCL RS (RING_LL) (us)",
|
||||
"ldmc_us": "Torch GEMM + NCCL RS (LDMC) (us)",
|
||||
"gemm_rs_us": "GEMM-RS (us)",
|
||||
"speedup_vs_ring_ll": "Speedup vs RING_LL",
|
||||
"speedup_vs_ldmc": "Speedup vs LDMC",
|
||||
}
|
||||
.rename(
|
||||
columns={
|
||||
"ring_ll_us": f"Torch GEMM + NCCL {collective} (RING_LL) (us)",
|
||||
"ldmc_us": f"Torch GEMM + NCCL {collective} (LDMC) (us)",
|
||||
"gemm_rs_ar_us": f"GEMM-{collective} (us)",
|
||||
"speedup_vs_ring_ll": "Speedup vs RING_LL",
|
||||
"speedup_vs_ldmc": "Speedup vs LDMC",
|
||||
}
|
||||
)
|
||||
.round(3)
|
||||
)
|
||||
end_to_end = end_to_end.round(
|
||||
{
|
||||
"Torch GEMM + NCCL RS (RING_LL) (us)": 2,
|
||||
"Torch GEMM + NCCL RS (LDMC) (us)": 2,
|
||||
"GEMM-RS (us)": 2,
|
||||
"Speedup vs RING_LL": 3,
|
||||
"Speedup vs LDMC": 3,
|
||||
}
|
||||
components = (
|
||||
results_df[["M", "N", "K", "torch_gemm_us", "best_nccl_us", "gemm_rs_ar_us"]]
|
||||
.rename(
|
||||
columns={
|
||||
"torch_gemm_us": "Torch GEMM (us)",
|
||||
"best_nccl_us": f"NCCL {collective} (best) (us)",
|
||||
"gemm_rs_ar_us": f"GEMM-{collective} (us)",
|
||||
}
|
||||
)
|
||||
.round(2)
|
||||
)
|
||||
components = results_df[
|
||||
["M", "N", "K", "torch_gemm_us", "best_nccl_rs_us", "gemm_rs_us"]
|
||||
].rename(
|
||||
columns={
|
||||
"torch_gemm_us": "Torch GEMM (us)",
|
||||
"best_nccl_rs_us": "NCCL RS (best) (us)",
|
||||
"gemm_rs_us": "GEMM-RS (us)",
|
||||
}
|
||||
)
|
||||
components = components.round(2)
|
||||
|
||||
print("### End-to-end latency")
|
||||
print(f"### {collective} end-to-end latency")
|
||||
print(end_to_end.to_markdown(index=False))
|
||||
print("\n### Component latency")
|
||||
print(f"\n### {collective} component latency")
|
||||
print(components.to_markdown(index=False))
|
||||
print("\nNCCL RS (best) is the faster of RING_LL and LDMC for each shape.")
|
||||
print(
|
||||
f"\nNCCL {collective} (best) is the faster of RING_LL and LDMC "
|
||||
"for each shape.\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
assert args.m and min(args.m) >= 128
|
||||
assert args.m and min(args.m) > 0
|
||||
assert args.n % 256 == 0
|
||||
assert args.num_workspaces > 0
|
||||
assert args.warmup_replays >= 0
|
||||
@@ -401,12 +430,19 @@ def main() -> None:
|
||||
sync_output = torch.empty_like(sync_input)
|
||||
|
||||
def device_barrier() -> None:
|
||||
# Order the timed launch after a device-side rank rendezvous without
|
||||
# including the rendezvous itself in the measured event interval.
|
||||
pynccl_comm.all_reduce(sync_input, sync_output)
|
||||
|
||||
gemm_rs = GemmRS(max_M=max(args.m), N=args.n)
|
||||
gemm_rs_ar = GemmRsAr(
|
||||
max_M=max(args.m),
|
||||
N=args.n,
|
||||
all_reduce=args.mode == "ar",
|
||||
)
|
||||
results = [
|
||||
benchmark_shape(
|
||||
gemm_rs,
|
||||
gemm_rs_ar,
|
||||
args.mode,
|
||||
M,
|
||||
args.n,
|
||||
K,
|
||||
@@ -420,12 +456,12 @@ def main() -> None:
|
||||
for K in K_values
|
||||
for M in args.m
|
||||
]
|
||||
del gemm_rs_ar
|
||||
|
||||
if tp_group.rank_in_group == 0:
|
||||
print_results(results)
|
||||
|
||||
dist.barrier(group=tp_group.cpu_group)
|
||||
del gemm_rs
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ def _reference(
|
||||
weight: torch.Tensor,
|
||||
world_size: int,
|
||||
group: dist.ProcessGroup,
|
||||
all_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
M = x.shape[0]
|
||||
padded_M = (M + world_size - 1) // world_size * world_size
|
||||
@@ -43,6 +44,10 @@ def _reference(
|
||||
if padded_M > M:
|
||||
partial[M:].zero_()
|
||||
|
||||
if all_reduce:
|
||||
dist.all_reduce(partial, group=group)
|
||||
return partial[:M]
|
||||
|
||||
output = torch.empty(
|
||||
(padded_M // world_size, weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
@@ -57,7 +62,11 @@ def _assert_valid_rows_close(
|
||||
expected: torch.Tensor,
|
||||
M: int,
|
||||
rank: int,
|
||||
all_reduce: bool,
|
||||
) -> None:
|
||||
if all_reduce:
|
||||
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=4.0)
|
||||
return
|
||||
rank_start = rank * actual.shape[0]
|
||||
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
|
||||
torch.testing.assert_close(
|
||||
@@ -68,11 +77,110 @@ def _assert_valid_rows_close(
|
||||
)
|
||||
|
||||
|
||||
def _run_mode(
|
||||
*,
|
||||
all_reduce: bool,
|
||||
device: torch.device,
|
||||
group: dist.ProcessGroup,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
weights: dict[int, torch.Tensor],
|
||||
) -> None:
|
||||
# cute_dsl is unavailable off CUDA, so import it only inside the GPU worker.
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr
|
||||
|
||||
gemm_rs_ar = GemmRsAr(
|
||||
max_M=max(M for M, _ in _SHAPES),
|
||||
N=_N,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
input_generator = torch.Generator(device=device)
|
||||
|
||||
# Alternating shapes exercise producer-flag reuse across different grids,
|
||||
# CTA-group choices, and both BN=128 and BN=256 dispatches.
|
||||
for M, K in (*_SHAPES, *_SHAPES[::-1]):
|
||||
input_generator.manual_seed(2000 + M + K)
|
||||
x = torch.randn(
|
||||
M,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
expected = _reference(x, weights[K], world_size, group, all_reduce)
|
||||
actual = gemm_rs_ar(x, weights[K])
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(actual, expected, M, rank, all_reduce)
|
||||
|
||||
if all_reduce:
|
||||
# AR outputs must remain valid after the symmetric workspace is reused.
|
||||
lifetime_M, lifetime_K = 257, 768
|
||||
input_generator.manual_seed(2500)
|
||||
lifetime_x = torch.randn(
|
||||
lifetime_M,
|
||||
lifetime_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
first_output = gemm_rs_ar(lifetime_x, weights[lifetime_K])
|
||||
first_snapshot = first_output.clone()
|
||||
second_output = gemm_rs_ar(-lifetime_x, weights[lifetime_K])
|
||||
torch.accelerator.synchronize(device)
|
||||
assert first_output.data_ptr() != second_output.data_ptr()
|
||||
torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0)
|
||||
|
||||
graph_M, graph_K = 1025, 4224
|
||||
input_generator.manual_seed(3000)
|
||||
graph_x = torch.randn(
|
||||
graph_M,
|
||||
graph_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
graph_expected = _reference(
|
||||
graph_x,
|
||||
weights[graph_K],
|
||||
world_size,
|
||||
group,
|
||||
all_reduce,
|
||||
)
|
||||
graph_output = torch.empty_like(graph_expected)
|
||||
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(3):
|
||||
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
|
||||
capture_stream.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
dist.barrier(group=group)
|
||||
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(
|
||||
graph_output,
|
||||
graph_expected,
|
||||
graph_M,
|
||||
rank,
|
||||
all_reduce,
|
||||
)
|
||||
|
||||
dist.barrier(group=group)
|
||||
del gemm_rs_ar, capture_stream, graph, graph_output
|
||||
|
||||
|
||||
def _worker(local_rank: int, world_size: int, master_port: int) -> None:
|
||||
# This module pulls in cute_dsl, which is unavailable off CUDA, so importing
|
||||
# it at module level would fail collection. Import it where it is used, as
|
||||
# `kda.py` does.
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import GemmRS
|
||||
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
@@ -92,10 +200,8 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
|
||||
|
||||
group = dist.group.WORLD
|
||||
rank = dist.get_rank(group)
|
||||
gemm_rs = GemmRS(max_M=max(M for M, _ in _SHAPES), N=_N)
|
||||
|
||||
weight_generator = torch.Generator(device=device)
|
||||
input_generator = torch.Generator(device=device)
|
||||
weights = {}
|
||||
for K in {K for _, K in _SHAPES}:
|
||||
weight_generator.manual_seed(1000 + rank * 10 + K)
|
||||
@@ -107,68 +213,29 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
|
||||
generator=weight_generator,
|
||||
)
|
||||
|
||||
# Alternating shapes exercise producer-flag reuse across different grids,
|
||||
# CTA-group choices, and both BN=128 and BN=256 dispatches.
|
||||
for M, K in (*_SHAPES, *_SHAPES[::-1]):
|
||||
input_generator.manual_seed(2000 + M + K)
|
||||
x = torch.randn(
|
||||
M,
|
||||
K,
|
||||
dtype=torch.bfloat16,
|
||||
# Production binds one mode per worker; exercise both mode-bound instances
|
||||
# sequentially in this test without implying that both are initialized.
|
||||
for all_reduce in (False, True):
|
||||
_run_mode(
|
||||
all_reduce=all_reduce,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
group=group,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
weights=weights,
|
||||
)
|
||||
expected = _reference(x, weights[K], world_size, group)
|
||||
actual = gemm_rs(x, weights[K])
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(actual, expected, M, rank)
|
||||
|
||||
# Exercise GEMM-RS under CUDA graph capture and replay.
|
||||
graph_M, graph_K = 1025, 4224
|
||||
input_generator.manual_seed(3000)
|
||||
graph_x = torch.randn(
|
||||
graph_M,
|
||||
graph_K,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
generator=input_generator,
|
||||
)
|
||||
graph_expected = _reference(graph_x, weights[graph_K], world_size, group)
|
||||
graph_output = torch.empty_like(graph_expected)
|
||||
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(3):
|
||||
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
|
||||
capture_stream.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
graph_output.copy_(gemm_rs(graph_x, weights[graph_K]))
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
dist.barrier(group=group)
|
||||
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_valid_rows_close(graph_output, graph_expected, graph_M, rank)
|
||||
|
||||
dist.barrier(group=group)
|
||||
del gemm_rs
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Kimi-K3 GEMM-RS requires SM100",
|
||||
reason="Kimi-K3 GEMM-RS/AR requires SM100",
|
||||
)
|
||||
def test_kimi_k3_gemm_rs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_kimi_k3_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
world_size = 2
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip("GEMM-RS requires two GPUs")
|
||||
pytest.skip("GEMM-RS/AR requires two GPUs")
|
||||
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
@@ -210,6 +210,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
|
||||
VLLM_KIMI_K3_GEMM_AR: bool = True
|
||||
VLLM_KIMI_K3_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
@@ -1608,6 +1609,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
|
||||
),
|
||||
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
|
||||
# attention projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
|
||||
# Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel
|
||||
# row-parallel projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))),
|
||||
|
||||
@@ -6,6 +6,7 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
||||
happen during model execution.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -95,6 +96,18 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _warmup_kimi_k3_gemm_rs_ar() -> None:
|
||||
# Kimi-K3 model construction imports this module only when GEMM-RS/AR is
|
||||
# enabled and initializes its singleton before kernel_warmup runs. Avoid
|
||||
# importing it here so other models do not compile the RS/AR variants.
|
||||
module = sys.modules.get("vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar")
|
||||
if module is None:
|
||||
return
|
||||
compiled = module.warmup_gemm_rs_ar()
|
||||
if compiled:
|
||||
logger.info_once("Warmed up %d Kimi-K3 GEMM-RS/AR variants.", compiled)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
@@ -146,6 +159,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
_warmup_kimi_k3_gemm_rs_ar()
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
|
||||
# to the shared JIT warmup infrastructure.
|
||||
|
||||
@@ -319,7 +319,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
run_gemm_rs: bool = False,
|
||||
run_gemm_rs_ar: bool = False,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
self.use_recoverssm = self.cache_config.use_kda_recoverssm
|
||||
@@ -491,14 +491,18 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.run_gemm_rs = run_gemm_rs
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
self.gemm_rs_ar = None
|
||||
if run_gemm_rs_ar:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import (
|
||||
get_gemm_rs_ar,
|
||||
)
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj)
|
||||
if not self.run_gemm_rs:
|
||||
gemm_rs_ar = get_gemm_rs_ar()
|
||||
if gemm_rs_ar.can_run(self.o_proj):
|
||||
self.gemm_rs_ar = gemm_rs_ar
|
||||
else:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
compilation_config = vllm_config.compilation_config
|
||||
@@ -541,12 +545,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(core_attn_out):
|
||||
return gemm_rs(core_attn_out, self.o_proj.weight)
|
||||
if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(core_attn_out):
|
||||
return self.gemm_rs_ar(core_attn_out, self.o_proj.weight)
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
|
||||
@@ -134,7 +134,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
use_rope: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
run_gemm_rs_ar: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -271,14 +271,18 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.run_gemm_rs = run_gemm_rs
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
self.gemm_rs_ar = None
|
||||
if run_gemm_rs_ar:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import (
|
||||
get_gemm_rs_ar,
|
||||
)
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.o_proj)
|
||||
if not self.run_gemm_rs:
|
||||
gemm_rs_ar = get_gemm_rs_ar()
|
||||
if gemm_rs_ar.can_run(self.o_proj):
|
||||
self.gemm_rs_ar = gemm_rs_ar
|
||||
else:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
|
||||
@@ -549,12 +553,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
if gate is not None:
|
||||
attn_out = _gate_sigmoid_mul(attn_out, gate)
|
||||
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(attn_out):
|
||||
return gemm_rs(attn_out, self.o_proj.weight)
|
||||
if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(attn_out):
|
||||
return self.gemm_rs_ar(attn_out, self.o_proj.weight)
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
|
||||
@@ -154,15 +154,18 @@ def shard_sequence_parallel_mlp(
|
||||
)
|
||||
|
||||
|
||||
def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool:
|
||||
if not envs.VLLM_KIMI_K3_GEMM_RS:
|
||||
def maybe_init_gemm_rs_ar(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool:
|
||||
# Both feature flags may be enabled; the worker's static SP topology binds
|
||||
# its singleton to exactly one mode.
|
||||
all_reduce = not use_sequence_parallel
|
||||
mode = "GEMM-AR" if all_reduce else "GEMM-RS"
|
||||
enabled = envs.VLLM_KIMI_K3_GEMM_AR if all_reduce else envs.VLLM_KIMI_K3_GEMM_RS
|
||||
if not enabled:
|
||||
return False
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
if not use_sequence_parallel:
|
||||
reason = "sequence parallelism is disabled"
|
||||
elif parallel_config.use_ubatching:
|
||||
if parallel_config.use_ubatching:
|
||||
reason = "ubatching is enabled"
|
||||
elif vllm_config.model_config.dtype != torch.bfloat16:
|
||||
reason = "the model dtype is not BF16"
|
||||
@@ -178,17 +181,32 @@ def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) ->
|
||||
reason = None
|
||||
|
||||
if reason is not None:
|
||||
logger.warning_once("GEMM-RS was requested but is disabled because %s.", reason)
|
||||
logger.warning_once("%s is disabled because %s.", mode, reason)
|
||||
return False
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import init_gemm_rs
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import init_gemm_rs_ar
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
init_gemm_rs(
|
||||
max_M=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
N=config.hidden_size,
|
||||
)
|
||||
logger.info_once("GEMM-RS is enabled.")
|
||||
try:
|
||||
init_gemm_rs_ar(
|
||||
max_M=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
N=config.hidden_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
logger.warning_once(
|
||||
"%s is disabled because initialization failed: %s. This may mean "
|
||||
"the TP ranks do not share one NVLink domain.",
|
||||
mode,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
if all_reduce:
|
||||
logger.info_once(
|
||||
"GEMM-AR is enabled. To disable it, set VLLM_KIMI_K3_GEMM_AR=0."
|
||||
)
|
||||
else:
|
||||
logger.info_once("GEMM-RS is enabled.")
|
||||
return True
|
||||
|
||||
|
||||
@@ -217,7 +235,7 @@ class KimiMLP(nn.Module):
|
||||
reduce_results: bool = True,
|
||||
use_sequence_parallel: bool = False,
|
||||
can_shard_sequence_parallel: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
run_gemm_rs_ar: bool = False,
|
||||
prefix: str = "",
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
@@ -230,7 +248,6 @@ class KimiMLP(nn.Module):
|
||||
use_sequence_parallel,
|
||||
can_shard_sequence_parallel,
|
||||
)
|
||||
self.run_gemm_rs = self.shard_sequence_parallel and run_gemm_rs
|
||||
replicate = use_sequence_parallel and not self.shard_sequence_parallel
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -252,13 +269,22 @@ class KimiMLP(nn.Module):
|
||||
disable_tp=replicate,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
self.gemm_rs_ar = None
|
||||
# RS requires sequence sharding; AR operates on replicated tokens.
|
||||
use_gemm_rs_ar = self.shard_sequence_parallel or (
|
||||
not use_sequence_parallel and reduce_results
|
||||
)
|
||||
if use_gemm_rs_ar and run_gemm_rs_ar:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import (
|
||||
get_gemm_rs_ar,
|
||||
)
|
||||
|
||||
self.run_gemm_rs = get_gemm_rs().can_run(self.down_proj)
|
||||
if not self.run_gemm_rs:
|
||||
gemm_rs_ar = get_gemm_rs_ar()
|
||||
if gemm_rs_ar.can_run(self.down_proj):
|
||||
self.gemm_rs_ar = gemm_rs_ar
|
||||
else:
|
||||
logger.warning_once(
|
||||
"GEMM-RS is disabled for %s due to an incompatible projection.",
|
||||
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
|
||||
prefix,
|
||||
)
|
||||
if hidden_act == "silu":
|
||||
@@ -284,12 +310,8 @@ class KimiMLP(nn.Module):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
|
||||
if self.run_gemm_rs:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs import get_gemm_rs
|
||||
|
||||
gemm_rs = get_gemm_rs()
|
||||
if gemm_rs.should_run(x):
|
||||
return gemm_rs(x, self.down_proj.weight)
|
||||
if self.gemm_rs_ar is not None and self.gemm_rs_ar.should_run(x):
|
||||
return self.gemm_rs_ar(x, self.down_proj.weight)
|
||||
|
||||
x, _ = self.down_proj(x)
|
||||
if self.shard_sequence_parallel:
|
||||
@@ -527,7 +549,7 @@ class KimiMoE(nn.Module):
|
||||
prefix: str = "",
|
||||
layer_idx: int = 0,
|
||||
use_sequence_parallel: bool = False,
|
||||
run_gemm_rs: bool = False,
|
||||
run_gemm_rs_ar: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
@@ -616,7 +638,7 @@ class KimiMoE(nn.Module):
|
||||
# FusedMoE path below hands them to the runner, which fuses
|
||||
# their reduction and assumes the replicated layout.
|
||||
can_shard_sequence_parallel=self.use_mega_moe,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
run_gemm_rs_ar=run_gemm_rs_ar,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
activation_situ_beta=activation_situ_beta,
|
||||
activation_situ_linear_beta=activation_situ_linear_beta,
|
||||
@@ -838,7 +860,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
run_gemm_rs: bool = False,
|
||||
run_gemm_rs_ar: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -873,7 +895,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
run_gemm_rs_ar=run_gemm_rs_ar,
|
||||
)
|
||||
self._self_attn_writes_output = False
|
||||
else:
|
||||
@@ -910,7 +932,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
aux_stream=aux_stream,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
run_gemm_rs_ar=run_gemm_rs_ar,
|
||||
)
|
||||
self._self_attn_writes_output = False
|
||||
|
||||
@@ -925,7 +947,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
layer_idx=layer_idx,
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
run_gemm_rs_ar=run_gemm_rs_ar,
|
||||
)
|
||||
self.mlp = self.block_sparse_moe
|
||||
else:
|
||||
@@ -937,7 +959,7 @@ class KimiDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.mlp",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
can_shard_sequence_parallel=True,
|
||||
run_gemm_rs=run_gemm_rs,
|
||||
run_gemm_rs_ar=run_gemm_rs_ar,
|
||||
activation_situ_beta=config.activation_situ_beta,
|
||||
activation_situ_linear_beta=config.activation_situ_linear_beta,
|
||||
)
|
||||
@@ -1118,9 +1140,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
# GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP
|
||||
# ranks to belong to one NVLink domain.
|
||||
self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel)
|
||||
# GEMM-RS/AR uses NCCL symmetric-memory multicast, which requires all
|
||||
# TP ranks to belong to one NVLink domain.
|
||||
self.run_gemm_rs_ar = maybe_init_gemm_rs_ar(
|
||||
vllm_config, self.use_sequence_parallel
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -1142,7 +1166,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
vllm_config,
|
||||
prefix,
|
||||
aux_stream=aux_stream,
|
||||
run_gemm_rs=self.run_gemm_rs,
|
||||
run_gemm_rs_ar=self.run_gemm_rs_ar,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
|
||||
+137
-43
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SM100 BF16 GEMM with a fused tensor-parallel reduce-scatter."""
|
||||
"""SM100 BF16 GEMM with fused tensor-parallel reduce-scatter/all-reduce."""
|
||||
|
||||
# Based on CUTLASS's Blackwell distributed GEMM-RS example at dcf215a.
|
||||
# See https://github.com/NVIDIA/cutlass/issues/3117 for memory semantics.
|
||||
@@ -32,7 +32,7 @@ def nanosleep(ns: int, *, loc=None, ip=None) -> None:
|
||||
|
||||
@dsl_user_op
|
||||
def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:
|
||||
# NOTE: assume x is contiguous
|
||||
# The vector instruction assumes x is contiguous and 16-byte aligned.
|
||||
assert x.element_type == BFloat16
|
||||
vec_type = ".v4.bf16x2"
|
||||
|
||||
@@ -66,15 +66,40 @@ def multimem_ld_reduce_16B(x: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:
|
||||
return cute.recast_tensor(y, x.element_type)
|
||||
|
||||
|
||||
class Sm100GemmRsBF16:
|
||||
@dsl_user_op
|
||||
def multimem_st_16B(dst: cute.Tensor, value: cute.Tensor, *, loc=None, ip=None) -> None:
|
||||
# The vector instruction assumes dst and value are contiguous BF16x8 vectors
|
||||
# and dst is 16-byte aligned.
|
||||
assert dst.element_type == BFloat16
|
||||
assert value.element_type == BFloat16
|
||||
ptr = dst.iterator.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
|
||||
regs = cute.recast_tensor(value, Int32, loc=loc, ip=ip)
|
||||
llvm.inline_asm(
|
||||
None,
|
||||
[ptr] + [regs[i].ir_value(loc=loc, ip=ip) for i in range(4)],
|
||||
"multimem.st.relaxed.sys.global.v4.f32 [$0], {$1, $2, $3, $4};",
|
||||
"l,r,r,r,r",
|
||||
has_side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
class Sm100GemmRsArBF16:
|
||||
def __init__(
|
||||
self, rank: int, num_ranks: int, BN: int = 128, cta_group: int = 1
|
||||
self,
|
||||
rank: int,
|
||||
num_ranks: int,
|
||||
BN: int = 128,
|
||||
cta_group: int = 1,
|
||||
all_reduce: bool = False,
|
||||
) -> None:
|
||||
self.rank = rank
|
||||
self.num_ranks = num_ranks
|
||||
BM, BK = 128, 64
|
||||
self.cta_tile = (BM, BN, BK)
|
||||
self.cta_group = cta_group
|
||||
self.all_reduce = all_reduce
|
||||
|
||||
smem_bytes = get_smem_capacity_in_bytes()
|
||||
self.stage_size = (BM + (BN // cta_group)) * BK * 2
|
||||
@@ -190,6 +215,9 @@ class Sm100GemmRsBF16:
|
||||
|
||||
M, K = A_tma.tma_tensor.shape
|
||||
N, _ = B_tma.tma_tensor.shape
|
||||
local_M = (
|
||||
partial_uc.shape[0] // num_ranks if self.all_reduce else output.shape[0]
|
||||
)
|
||||
grid_m = cute.ceil_div(M, BM)
|
||||
# Keep 2-CTA clusters within a single N tile.
|
||||
grid_m = cute.ceil_div(grid_m, cta_group) * cta_group
|
||||
@@ -322,7 +350,6 @@ class Sm100GemmRsBF16:
|
||||
# Keep epilogue in warps 0-3 and communication in warps 4-7.
|
||||
# Swapping the warpgroups can hang due to warp scheduling.
|
||||
tid_ = tid % 128
|
||||
local_M = output.shape[0]
|
||||
|
||||
# Offset in the [M, N] GEMM result.
|
||||
rank_start = self.rank * local_M
|
||||
@@ -338,7 +365,8 @@ class Sm100GemmRsBF16:
|
||||
max_tile_rows = BM // num_ranks
|
||||
vecs_per_tile = max_tile_rows * vec_cols
|
||||
partial_vecs = cute.zipped_divide(partial_mc, (1, vec_width))
|
||||
output_vecs = cute.zipped_divide(output, (1, vec_width))
|
||||
if cutlass.const_expr(not self.all_reduce):
|
||||
output_vecs = cute.zipped_divide(output, (1, vec_width))
|
||||
|
||||
for tile_id in range(raw_bid, total_tiles, num_bids):
|
||||
rs_bid_m = tile_id % grid_m
|
||||
@@ -400,7 +428,6 @@ class Sm100GemmRsBF16:
|
||||
reduced_vec.store(tmp.load())
|
||||
reduced_vecs.append(reduced_vec)
|
||||
|
||||
# Store the result to local L2.
|
||||
for vec_iter in cutlass.range_constexpr(vecs_per_tile // 128):
|
||||
vec_idx = tid_ + vec_iter * 128
|
||||
|
||||
@@ -409,11 +436,19 @@ class Sm100GemmRsBF16:
|
||||
col = bid_n * vec_cols + vec_idx % vec_cols
|
||||
|
||||
if local_row < local_row_end and global_row < M:
|
||||
cute.copy(
|
||||
st_atom,
|
||||
reduced_vecs[vec_iter],
|
||||
output_vecs[None, (local_row, col)],
|
||||
)
|
||||
if cutlass.const_expr(self.all_reduce):
|
||||
# Broadcast the result to all ranks.
|
||||
multimem_st_16B(
|
||||
partial_vecs[None, (global_row, col)],
|
||||
reduced_vecs[vec_iter],
|
||||
)
|
||||
else:
|
||||
# Store the result to local L2.
|
||||
cute.copy(
|
||||
st_atom,
|
||||
reduced_vecs[vec_iter],
|
||||
output_vecs[None, (local_row, col)],
|
||||
)
|
||||
cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128)
|
||||
|
||||
# Release each GEMM tile consumed by this RS tile. The last
|
||||
@@ -481,19 +516,21 @@ class Sm100GemmRsBF16:
|
||||
num_ranks,
|
||||
)
|
||||
|
||||
# Exit barrier. GPU scope is sufficient because the kernel writes
|
||||
# local global memory and only needs to flush it to local L2.
|
||||
# Exit barrier. GPU scope is sufficient for RS because the kernel
|
||||
# writes local global memory and only needs to flush it to local L2.
|
||||
# AR uses system scope to flush the multimem stores to remote L2.
|
||||
cute.arch.barrier(barrier_id=BAR_COMM, number_of_threads=128)
|
||||
if tid_ == 0:
|
||||
exit_flag = total_tiles + raw_bid
|
||||
scope = "sys" if self.all_reduce else "gpu"
|
||||
utils.distributed.multimem_red_add1(
|
||||
flags_mc_ptr + exit_flag, order="release", scope="gpu"
|
||||
flags_mc_ptr + exit_flag, order="release", scope=scope
|
||||
)
|
||||
utils.distributed.spin_lock_atom_cas_acquire_wait(
|
||||
flags_uc_ptr + exit_flag,
|
||||
expected_val=num_ranks,
|
||||
reset_val=0,
|
||||
scope="gpu",
|
||||
scope=scope,
|
||||
)
|
||||
|
||||
else:
|
||||
@@ -564,8 +601,8 @@ class Sm100GemmRsBF16:
|
||||
if tid_ == 0 and bid_m * BM < M:
|
||||
gemm_start = bid_m * BM
|
||||
gemm_end = min(gemm_start + BM, M)
|
||||
first_owner = gemm_start // output.shape[0]
|
||||
last_owner = (gemm_end - 1) // output.shape[0]
|
||||
first_owner = gemm_start // local_M
|
||||
last_owner = (gemm_end - 1) // local_M
|
||||
|
||||
# signal to all consuming ranks (can be more than 1)
|
||||
for rank in cutlass.range_constexpr(num_ranks):
|
||||
@@ -595,7 +632,13 @@ class Sm100GemmRsBF16:
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(rank: int, num_ranks: int, BN: int, cta_group: int):
|
||||
def compile(
|
||||
rank: int,
|
||||
num_ranks: int,
|
||||
BN: int,
|
||||
cta_group: int,
|
||||
all_reduce: bool = False,
|
||||
):
|
||||
M = cute.sym_int()
|
||||
padded_M = cute.sym_int()
|
||||
N = cute.sym_int()
|
||||
@@ -616,18 +659,24 @@ class Sm100GemmRsBF16:
|
||||
assumed_align=32,
|
||||
)
|
||||
partial_mc_ptr = nullptr(BFloat16, cute.AddressSpace.gmem, assumed_align=32)
|
||||
output = make_fake_tensor(
|
||||
BFloat16,
|
||||
(local_M, N),
|
||||
(cute.sym_int64(divisibility=16), 1),
|
||||
assumed_align=32,
|
||||
output = (
|
||||
None
|
||||
if all_reduce
|
||||
else make_fake_tensor(
|
||||
BFloat16,
|
||||
(local_M, N),
|
||||
(cute.sym_int64(divisibility=16), 1),
|
||||
assumed_align=32,
|
||||
)
|
||||
)
|
||||
flags = make_fake_tensor(Int32, (num_flags,), (1,), assumed_align=16)
|
||||
flags_mc_ptr = nullptr(Int32, cute.AddressSpace.gmem, assumed_align=16)
|
||||
peer_flag_ptr = nullptr(Int64, cute.AddressSpace.gmem, assumed_align=8)
|
||||
|
||||
stream = make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
kernel = Sm100GemmRsBF16(rank, num_ranks, BN, cta_group)
|
||||
kernel = Sm100GemmRsArBF16(
|
||||
rank, num_ranks, BN, cta_group, all_reduce=all_reduce
|
||||
)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
A,
|
||||
@@ -644,13 +693,20 @@ class Sm100GemmRsBF16:
|
||||
)
|
||||
|
||||
|
||||
class GemmRS:
|
||||
"""Own the max-sized symmetric workspace for Kimi-K3 GEMM-RS launches.
|
||||
class GemmRsAr:
|
||||
"""Own the symmetric workspace for Kimi-K3 GEMM-RS/AR launches.
|
||||
|
||||
All TP ranks must belong to one NVLink domain for multimem instructions.
|
||||
|
||||
Each instance is bound to either RS or AR. A vLLM worker has one static
|
||||
sequence-parallel topology, so the process-wide singleton only needs one
|
||||
mode. Two independent mode-specific singletons would lift that restriction
|
||||
but duplicate the large symmetric workspace. A future mixed-mode design
|
||||
should instead use lightweight RS/AR frontends over one shared multicast
|
||||
workspace; that is outside this integration's current scope.
|
||||
"""
|
||||
|
||||
def __init__(self, *, max_M: int, N: int) -> None:
|
||||
def __init__(self, *, max_M: int, N: int, all_reduce: bool = False) -> None:
|
||||
tp_group = get_tp_group()
|
||||
group = tp_group.device_group
|
||||
rank = tp_group.rank_in_group
|
||||
@@ -667,9 +723,12 @@ class GemmRS:
|
||||
self.max_M = max_M
|
||||
self.N = N
|
||||
self.device = device
|
||||
self.all_reduce = all_reduce
|
||||
|
||||
self.partial = symm_mem.empty((max_M, N), dtype=torch.bfloat16, device=device)
|
||||
self.partial_handle = symm_mem.rendezvous(self.partial, group)
|
||||
if self.partial_handle.multicast_ptr == 0:
|
||||
raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory")
|
||||
self.partial_mc_ptr = make_ptr(
|
||||
BFloat16,
|
||||
self.partial_handle.multicast_ptr,
|
||||
@@ -684,6 +743,8 @@ class GemmRS:
|
||||
max_flags = grid_m * (N // 128) + self.num_sms
|
||||
self.flags = symm_mem.empty(max_flags, dtype=torch.int32, device=device)
|
||||
self.flags_handle = symm_mem.rendezvous(self.flags, group)
|
||||
if self.flags_handle.multicast_ptr == 0:
|
||||
raise RuntimeError("GEMM-RS/AR requires NVLink multicast memory")
|
||||
self.flags.zero_()
|
||||
self.flags_mc_ptr = make_ptr(
|
||||
Int32,
|
||||
@@ -698,8 +759,6 @@ class GemmRS:
|
||||
assumed_align=8,
|
||||
)
|
||||
|
||||
assert self.partial_handle.multicast_ptr != 0
|
||||
assert self.flags_handle.multicast_ptr != 0
|
||||
torch.accelerator.synchronize(device)
|
||||
tp_group.barrier()
|
||||
|
||||
@@ -721,7 +780,8 @@ class GemmRS:
|
||||
)
|
||||
|
||||
def should_run(self, x: torch.Tensor) -> bool:
|
||||
# Small-M shapes are supported but faster on the existing LL path.
|
||||
# Use the same threshold for RS and AR for now. Small-M shapes are
|
||||
# supported but faster on the existing LL path.
|
||||
return x.shape[0] >= 128
|
||||
|
||||
def __call__(self, x: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
|
||||
@@ -752,12 +812,15 @@ class GemmRS:
|
||||
num_ctas = num_ctas // cta_group * cta_group
|
||||
assert self.flags.numel() >= num_tiles + num_ctas
|
||||
|
||||
output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device)
|
||||
compiled = Sm100GemmRsBF16.compile(
|
||||
output = None
|
||||
if not self.all_reduce:
|
||||
output = torch.empty((local_M, N), dtype=torch.bfloat16, device=self.device)
|
||||
compiled = Sm100GemmRsArBF16.compile(
|
||||
self.rank,
|
||||
self.world_size,
|
||||
BN,
|
||||
cta_group,
|
||||
self.all_reduce,
|
||||
)
|
||||
compiled(
|
||||
x,
|
||||
@@ -770,21 +833,52 @@ class GemmRS:
|
||||
self.peer_flag_ptr,
|
||||
num_ctas,
|
||||
)
|
||||
if self.all_reduce:
|
||||
# AttnRes may retain output past the next workspace reuse.
|
||||
# A future kernel could overlap this copy using an extra warp or
|
||||
# the communication warp.
|
||||
return self.partial[:M].clone()
|
||||
assert output is not None
|
||||
return output
|
||||
|
||||
|
||||
_gemm_rs: GemmRS | None = None
|
||||
_gemm_rs_ar: GemmRsAr | None = None
|
||||
|
||||
|
||||
def init_gemm_rs(max_M: int, N: int) -> None:
|
||||
"""Collectively initialize the process-wide GEMM-RS state."""
|
||||
global _gemm_rs
|
||||
if _gemm_rs is not None:
|
||||
assert _gemm_rs.max_M >= max_M and _gemm_rs.N == N
|
||||
def init_gemm_rs_ar(max_M: int, N: int, *, all_reduce: bool = False) -> None:
|
||||
"""Collectively initialize the process-wide, mode-bound GEMM-RS/AR state."""
|
||||
global _gemm_rs_ar
|
||||
if _gemm_rs_ar is not None:
|
||||
if _gemm_rs_ar.all_reduce != all_reduce:
|
||||
current = "AR" if _gemm_rs_ar.all_reduce else "RS"
|
||||
requested = "AR" if all_reduce else "RS"
|
||||
raise RuntimeError(
|
||||
f"GEMM-RS/AR is already initialized for {current}; "
|
||||
f"a worker cannot reinitialize it for {requested}"
|
||||
)
|
||||
assert _gemm_rs_ar.max_M >= max_M and _gemm_rs_ar.N == N
|
||||
return
|
||||
_gemm_rs = GemmRS(max_M=max_M, N=N)
|
||||
_gemm_rs_ar = GemmRsAr(max_M=max_M, N=N, all_reduce=all_reduce)
|
||||
|
||||
|
||||
def get_gemm_rs() -> GemmRS:
|
||||
assert _gemm_rs is not None, "GEMM-RS is not initialized"
|
||||
return _gemm_rs
|
||||
def warmup_gemm_rs_ar() -> int:
|
||||
"""Compile every reachable dispatch for the initialized GEMM-RS/AR mode."""
|
||||
# Initialization can be disabled or fail when multicast is unavailable.
|
||||
if _gemm_rs_ar is None:
|
||||
return 0
|
||||
# Keep these profiles in sync with the dispatch in GemmRsAr.__call__.
|
||||
profiles = ((128, 1), (128, 2), (256, 2))
|
||||
for BN, cta_group in profiles:
|
||||
Sm100GemmRsArBF16.compile(
|
||||
_gemm_rs_ar.rank,
|
||||
_gemm_rs_ar.world_size,
|
||||
BN,
|
||||
cta_group,
|
||||
_gemm_rs_ar.all_reduce,
|
||||
)
|
||||
return len(profiles)
|
||||
|
||||
|
||||
def get_gemm_rs_ar() -> GemmRsAr:
|
||||
assert _gemm_rs_ar is not None, "GEMM-RS/AR is not initialized"
|
||||
return _gemm_rs_ar
|
||||
Reference in New Issue
Block a user