[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
@@ -0,0 +1,367 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the Kimi-K3 latent MoE addmm against CuTe residual GEMM.
The benchmark covers ``BF16[M, 3584] @ BF16[7168, 3584].T + BF16[M, 7168]``
with FP32 accumulation and BF16 output. Both backends execute through CUDA
Graph replay. Weights and residuals rotate across buffers exceeding L2 so the
comparison models the full latent MoE projection-and-add path.
"""
from __future__ import annotations
import argparse
import dataclasses
import importlib.util
import json
import math
import statistics
from collections.abc import Callable, Sequence
from pathlib import Path
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings import driver as cuda
from cuda.bindings.driver import CUstream
from quack.compile_utils import make_fake_tensor
N = 7168
K = 3584
@dataclasses.dataclass(frozen=True, slots=True)
class Config:
block_size: int
outputs_per_block: int
k_unroll: int
vector_width: int = 8
def parse_config(value: str) -> Config:
try:
parts = [int(part) for part in value.split(",")]
except ValueError as error:
raise argparse.ArgumentTypeError(
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH]"
) from error
if len(parts) == 3:
return Config(*parts)
if len(parts) == 4:
return Config(*parts)
raise argparse.ArgumentTypeError(
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH]"
)
def production_residual_config(m: int) -> Config | None:
"""The measured Latent-MoE residual config for M, from the K3 table."""
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
spec = KIMI_K3_PROJECTIONS.get((N, K))
config = spec.residual_config(m) if spec is not None else None
if config is None:
return None
return Config(
config.block_size,
config.outputs_per_block,
config.k_unroll,
config.vector_width,
)
def candidate_configs(mode: str, selected: Config | None, m: int) -> list[Config]:
if mode == "selected":
if selected is not None:
return [selected]
# No explicit --config: fall back to the production table for this M.
config = production_residual_config(m)
return [config] if config is not None else []
if mode == "baseline":
return [Config(224, 4, 2)]
return [
Config(block_size, outputs_per_block, k_unroll, vector_width)
for vector_width in (4, 8)
for block_size in (32, 64, 128, 224, 448)
if block_size % 32 == 0 and K % (block_size * vector_width) == 0
for outputs_per_block in (1, 2, 4, 7, 8)
if N % outputs_per_block == 0
for k_unroll in (1, 2, 4)
]
def load_kernel_class(path: Path):
spec = importlib.util.spec_from_file_location("cute_skinny_device", path)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load CuTe kernel from {path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.CuteSkinnyGemm
def stream() -> CUstream:
return CUstream(torch.cuda.current_stream().cuda_stream)
def compile_kernel(kernel_class, m: int, config: Config, max_registers: int):
element_type = cutlass.BFloat16
n = cute.sym_int(divisibility=config.outputs_per_block)
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
a = make_fake_tensor(element_type, (m, k), divisibility=config.vector_width)
b = make_fake_tensor(element_type, (n, k), divisibility=config.vector_width)
residual = make_fake_tensor(element_type, (m, n), divisibility=1)
c = make_fake_tensor(element_type, (m, n), divisibility=1)
kernel = kernel_class(
element_type=element_type,
num_rows=m,
block_size=config.block_size,
outputs_per_block=config.outputs_per_block,
vector_width=config.vector_width,
k_unroll=config.k_unroll,
has_residual=True,
use_pdl=True,
)
return cute.compile(
kernel,
a,
b,
residual,
c,
stream(),
options=(
"--enable-tvm-ffi --keep-cubin "
f"--ptxas-options -maxrregcount={max_registers} "
"--ptxas-options -lineinfo"
),
)
def resource_usage(compiled) -> dict[str, Any]:
executor = getattr(compiled, "_default_executor", None)
context = getattr(executor, "exec_context", None)
functions = getattr(context, "kernel_functions", None)
if not functions:
return {"resource_metrics_available": False}
def attribute(name, function) -> int:
error, value = cuda.cuFuncGetAttribute(name, function)
if error != cuda.CUresult.CUDA_SUCCESS:
raise RuntimeError(f"cuFuncGetAttribute failed with {error}")
return int(value)
registers = [
attribute(cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_NUM_REGS, function)
for function in functions
]
local_bytes = [
attribute(
cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES,
function,
)
for function in functions
]
return {
"resource_metrics_available": True,
"registers_per_thread": max(registers, default=0),
"spill_bytes": max(local_bytes, default=0),
}
def rotating_buffer_count(m: int, multiplier: float, limit: int) -> int:
properties = torch.cuda.get_device_properties(0)
bytes_per_pair = (N * K + m * N) * 2
target = math.ceil(multiplier * properties.L2_cache_size)
return max(2, min(limit, math.ceil(target / bytes_per_pair)))
def graph_samples(
launch: Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], None],
activation: torch.Tensor,
weights: Sequence[torch.Tensor],
residuals: Sequence[torch.Tensor],
repeats: int,
replays: int,
) -> tuple[list[float], list[torch.Tensor]]:
outputs = [torch.empty_like(residual) for residual in residuals]
for weight, residual, output in zip(weights, residuals, outputs):
launch(activation, weight, residual, output)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for weight, residual, output in zip(weights, residuals, outputs):
launch(activation, weight, residual, output)
for _ in range(20):
graph.replay()
torch.accelerator.synchronize()
samples = []
for _ in range(repeats):
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(replays):
graph.replay()
end.record()
end.synchronize()
samples.append(start.elapsed_time(end) * 1000.0 / (replays * len(weights)))
return samples, outputs
def summarize(samples: Sequence[float]) -> dict[str, Any]:
ordered = sorted(samples)
def percentile(fraction: float) -> float:
position = fraction * (len(ordered) - 1)
lower = math.floor(position)
upper = math.ceil(position)
if lower == upper:
return ordered[lower]
weight = position - lower
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
mean = statistics.mean(samples)
return {
"median_us": statistics.median(samples),
"p10_us": percentile(0.1),
"p90_us": percentile(0.9),
"mean_us": mean,
"cv_pct": statistics.pstdev(samples) / mean * 100.0,
"samples_us": list(samples),
}
def correctness(
output: torch.Tensor,
activation: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
) -> dict[str, Any]:
actual = output.float()
reference = activation.float() @ weight.float().t() + residual.float()
error = (actual - reference).abs()
scaled_error = error / (reference.abs() + 1.0)
cosine = torch.nn.functional.cosine_similarity(
actual.flatten(), reference.flatten(), dim=0
).item()
return {
"valid": cosine > 0.999,
"cosine": cosine,
"max_abs_error": error.max().item(),
"max_scaled_error": scaled_error.max().item(),
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--kernel", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--mode", choices=("baseline", "sweep", "selected"), default="baseline"
)
parser.add_argument("--config", type=parse_config)
parser.add_argument("--m", type=int, action="append")
parser.add_argument("--config-shard", type=int, default=0)
parser.add_argument("--num-config-shards", type=int, default=1)
parser.add_argument("--repeats", type=int, default=21)
parser.add_argument("--replays", type=int, default=200)
parser.add_argument("--cache-multiplier", type=float, default=3.0)
parser.add_argument("--max-buffers", type=int, default=32)
parser.add_argument("--max-registers", type=int, default=64)
args = parser.parse_args()
token_counts = args.m or list(range(1, 17))
if any(not 1 <= m <= 16 for m in token_counts):
raise ValueError("expected 1 <= M <= 16")
if not 0 <= args.config_shard < args.num_config_shards:
raise ValueError("config shard must be in [0, num_config_shards)")
torch.accelerator.set_device_index(0)
if torch.cuda.get_device_capability() != (10, 3):
raise RuntimeError("this benchmark requires SM103")
kernel_class = load_kernel_class(args.kernel)
properties = torch.cuda.get_device_properties(0)
metadata = {
"device": properties.name,
"compute_capability": list(torch.cuda.get_device_capability()),
"torch_version": torch.__version__,
"cuda_version": torch.version.cuda,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("w", encoding="utf-8") as output_file:
for m in token_counts:
configs = candidate_configs(args.mode, args.config, m)
torch.manual_seed(20260722 + m)
count = rotating_buffer_count(m, args.cache_multiplier, args.max_buffers)
activation = torch.randn((m, K), device="cuda", dtype=torch.bfloat16)
weights = [
torch.randn((N, K), device="cuda", dtype=torch.bfloat16)
for _ in range(count)
]
residuals = [
torch.randn((m, N), device="cuda", dtype=torch.bfloat16)
for _ in range(count)
]
candidates: list[tuple[str, Config | None]] = [("cublas_addmm", None)]
candidates.extend(
("cute_residual", config)
for index, config in enumerate(configs)
if index % args.num_config_shards == args.config_shard
)
for backend, config in candidates:
row: dict[str, Any] = {
"m": m,
"n": N,
"k": K,
"backend": backend,
"mode": args.mode,
"config": dataclasses.asdict(config) if config else {},
"num_buffers": count,
"cache_multiplier": args.cache_multiplier,
**metadata,
}
try:
if backend == "cublas_addmm":
launch = lambda a, b, residual, c: torch.addmm(
residual, a, b.t(), out=c
)
else:
if config is None:
raise AssertionError("missing CuTe config")
compiled = compile_kernel(
kernel_class, m, config, args.max_registers
)
launch = lambda a, b, residual, c, fn=compiled: fn(
a, b, residual, c, stream()
)
row.update(resource_usage(compiled))
samples, outputs = graph_samples(
launch,
activation,
weights,
residuals,
args.repeats,
args.replays,
)
row.update(
correctness(outputs[0], activation, weights[0], residuals[0])
)
row.update(summarize(samples))
except Exception as error: # noqa: BLE001
row.update(
{
"valid": False,
"error": f"{type(error).__name__}: {error}",
}
)
output_file.write(json.dumps(row, sort_keys=True) + "\n")
output_file.flush()
print(json.dumps(row, sort_keys=True), flush=True)
del activation, weights, residuals
torch.accelerator.empty_cache()
if __name__ == "__main__":
main()
@@ -0,0 +1,806 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the Kimi K3 latent-MoE tail and its up-projection kernels.
The ``up-projection`` subcommand isolates the TP-local dynamic and static-M
skinny GEMMs. It rotates weights through a working set larger than L2 to model
successive model layers.
The ``whole-tail`` subcommand measures the distributed operator. Its reference
path includes two AllReduces, RMSNorm, the replicated up-projection, and the
final add. CUDA-event samples report the slowest rank so cross-rank skew is
included.
Examples:
.. code-block:: console
.venv/bin/python \
benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py up-projection
torchrun --nproc-per-node=8 \
benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py whole-tail
For multi-node runs, launch one ``torchrun`` agent per node and use a shared
rendezvous endpoint.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import statistics
from collections.abc import Callable, Sequence
from dataclasses import asdict
from pathlib import Path
from typing import Any
import cutlass
import cutlass.utils as utils
import torch
import torch.distributed as dist
import torch.nn.functional as F
from cuda.bindings import driver as cuda
from vllm.distributed import get_tp_group
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
set_custom_all_reduce,
)
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.models.kimi_k3.nvidia.ops import latent_moe_tail
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.latent_moe_tail import (
fused_add_multicast_gemm,
fused_add_multicast_skinny_gemm,
)
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
RMS_EPS = 0.1
MAX_NUM_TOKENS = 16
MMA_TILER_MN = (64, 32)
CLUSTER_SHAPE_MN = (1, 8)
B_PRIME_STAGES = 2
def parse_up_projection_config(
value: str,
) -> fused_add_multicast_skinny_gemm.SkinnyConfig:
try:
values = [int(part) for part in value.split(",")]
except ValueError as error:
raise argparse.ArgumentTypeError(
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH[,PREFETCH_B]]"
) from error
if len(values) in (3, 4):
return fused_add_multicast_skinny_gemm.SkinnyConfig(*values)
if len(values) == 5 and values[4] in (0, 1):
return fused_add_multicast_skinny_gemm.SkinnyConfig(
*values[:4],
prefetch_b_before_pdl=bool(values[4]),
)
raise argparse.ArgumentTypeError(
"config must be BLOCK,OUTPUTS,K_UNROLL"
"[,VECTOR_WIDTH[,PREFETCH_B]], where PREFETCH_B is 0 or 1"
)
def parse_tail_skinny_config(
value: str,
) -> tuple[int, fused_add_multicast_skinny_gemm.SkinnyConfig]:
try:
values = [int(part) for part in value.split(",")]
except ValueError as error:
raise argparse.ArgumentTypeError(
"config must be M,BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH[,PREFETCH_B]]"
) from error
if len(values) == 4:
num_tokens, *config = values
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(*config)
if len(values) == 5:
num_tokens, *config = values
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(*config)
if len(values) == 6 and values[5] in (0, 1):
num_tokens, block, outputs, unroll, vector_width, prefetch = values
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(
block,
outputs,
unroll,
vector_width,
bool(prefetch),
)
raise argparse.ArgumentTypeError(
"config must be M,BLOCK,OUTPUTS,K_UNROLL"
"[,VECTOR_WIDTH[,PREFETCH_B]], where PREFETCH_B is 0 or 1"
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="scope", required=True)
up_projection = subparsers.add_parser(
"up-projection",
help="Benchmark the isolated TP-local up-projection kernels.",
)
up_projection.add_argument(
"--backend",
choices=("dynamic", "skinny", "both"),
default="both",
)
up_projection.add_argument("--tp-size", type=int, default=16)
up_projection.add_argument(
"--num-tokens",
type=int,
nargs="+",
default=[*range(1, 9), 16],
)
up_projection.add_argument(
"--skinny-config",
type=parse_up_projection_config,
action="append",
help="Benchmark a static-M config for every selected token count.",
)
up_projection.add_argument("--cache-multiplier", type=float, default=2.0)
up_projection.add_argument("--max-weights", type=int, default=64)
up_projection.add_argument("--warmup-replays", type=int, default=10)
up_projection.add_argument("--samples", type=int, default=31)
up_projection.add_argument("--output", type=Path)
whole_tail = subparsers.add_parser(
"whole-tail",
help="Benchmark the distributed latent-MoE tail operator.",
)
whole_tail.add_argument(
"--backend",
choices=("reference", "fused", "both"),
default="both",
)
whole_tail.add_argument(
"--num-tokens",
type=int,
nargs="+",
default=[1, 5, 8, 16],
)
whole_tail.add_argument("--warmup-replays", type=int, default=20)
whole_tail.add_argument("--samples", type=int, default=51)
whole_tail.add_argument(
"--skinny-max-num-tokens",
type=int,
nargs="+",
help="Override the fused operator's static-M cutoff; use 0 for dynamic-only.",
)
whole_tail.add_argument(
"--skinny-config",
type=parse_tail_skinny_config,
action="append",
help="Override one static-M config for tuning.",
)
whole_tail.add_argument("--output", type=Path)
return parser.parse_args()
def percentile(samples: Sequence[float], fraction: float) -> float:
ordered = sorted(samples)
position = fraction * (len(ordered) - 1)
lower = math.floor(position)
upper = math.ceil(position)
if lower == upper:
return ordered[lower]
upper_weight = position - lower
return ordered[lower] * (1.0 - upper_weight) + ordered[upper] * upper_weight
def summarize(samples_us: Sequence[float]) -> dict[str, Any]:
mean_us = statistics.mean(samples_us)
return {
"median_us": statistics.median(samples_us),
"p10_us": percentile(samples_us, 0.1),
"p90_us": percentile(samples_us, 0.9),
"mean_us": mean_us,
"cv_pct": statistics.pstdev(samples_us) / mean_us * 100.0,
"samples_us": list(samples_us),
}
def rotating_weight_count(
shard_size: int,
cache_multiplier: float,
limit: int,
) -> int:
properties = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
)
weight_bytes = shard_size * LATENT_SIZE * 2
target_bytes = math.ceil(properties.L2_cache_size * cache_multiplier)
return max(2, min(limit, math.ceil(target_bytes / weight_bytes)))
def capture_up_projection_graph(
launches: Sequence[Callable[[], None]],
) -> torch.cuda.CUDAGraph:
for launch in launches:
launch()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for launch in launches:
launch()
torch.accelerator.synchronize()
return graph
def benchmark_up_projection_graph(
graph: torch.cuda.CUDAGraph,
*,
operations_per_replay: int,
warmup_replays: int,
samples: int,
) -> dict[str, Any]:
for _ in range(warmup_replays):
graph.replay()
torch.accelerator.synchronize()
samples_us = []
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
for _ in range(samples):
start.record()
graph.replay()
end.record()
end.synchronize()
samples_us.append(start.elapsed_time(end) * 1000.0 / operations_per_replay)
return summarize(samples_us)
class DynamicKernel:
def __init__(
self,
shard_size: int,
mailbox: torch.Tensor,
shared_shard: torch.Tensor,
) -> None:
self.shard_size = shard_size
self.mailbox = mailbox
self.mailbox_c = fused_add_multicast_gemm._as_cute(mailbox)
compile_latent = torch.empty(
(1, MAX_NUM_TOKENS, LATENT_SIZE),
dtype=torch.bfloat16,
device=mailbox.device,
)
compile_weight = torch.empty(
(1, shard_size, LATENT_SIZE),
dtype=torch.bfloat16,
device=mailbox.device,
)
cluster_size = math.prod(CLUSTER_SHAPE_MN)
max_active_clusters = utils.HardwareInfo().get_max_active_clusters(cluster_size)
self.compiled = fused_add_multicast_gemm.compile_kernel(
(MAX_NUM_TOKENS, shard_size, LATENT_SIZE, 1),
fused_add_multicast_gemm._as_cute(
compile_latent,
dynamic_m=True,
),
fused_add_multicast_gemm._as_cute(compile_weight),
self.mailbox_c,
fused_add_multicast_gemm._as_cute(shared_shard),
HIDDEN_SIZE,
shard_size,
MMA_TILER_MN,
CLUSTER_SHAPE_MN,
max_active_clusters,
B_PRIME_STAGES,
)
def launch(
self,
latent: torch.Tensor,
weight: torch.Tensor,
shared_shard: torch.Tensor,
) -> None:
stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)
self.compiled(
fused_add_multicast_gemm._as_cute(
latent.unsqueeze(0),
dynamic_m=True,
),
fused_add_multicast_gemm._as_cute(weight.unsqueeze(0)),
self.mailbox_c,
fused_add_multicast_gemm._as_cute(shared_shard),
cutlass.Int64(latent.shape[0]),
cutlass.Int64(self.mailbox.data_ptr()),
stream,
)
class SkinnyKernel:
def __init__(
self,
num_tokens: int,
shard_size: int,
config: fused_add_multicast_skinny_gemm.SkinnyConfig,
) -> None:
self.compiled = fused_add_multicast_skinny_gemm.compile_kernel(
num_rows=num_tokens,
latent_dim=LATENT_SIZE,
hidden_dim=HIDDEN_SIZE,
shard_dim=shard_size,
config=config,
)
def launch(
self,
latent: torch.Tensor,
weight: torch.Tensor,
shared_shard: torch.Tensor,
mailbox: torch.Tensor,
) -> None:
self.compiled(
fused_add_multicast_skinny_gemm._as_cute(latent),
fused_add_multicast_skinny_gemm._as_cute(weight),
fused_add_multicast_skinny_gemm._as_cute(shared_shard),
cutlass.Int64(mailbox.data_ptr()),
cuda.CUstream(torch.cuda.current_stream().cuda_stream),
)
def check_up_projection_output(
actual: torch.Tensor,
latent: torch.Tensor,
weight: torch.Tensor,
shared_shard: torch.Tensor,
) -> None:
gemm = F.linear(latent.float(), weight.float()).to(torch.bfloat16)
expected = (gemm.float() + shared_shard.float()).to(torch.bfloat16)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def make_up_projection_launches(
launch: Callable[[torch.Tensor, torch.Tensor, torch.Tensor], None],
latent: torch.Tensor,
weights: Sequence[torch.Tensor],
shared_shard: torch.Tensor,
) -> list[Callable[[], None]]:
return [
lambda weight=weight: launch(latent, weight, shared_shard) for weight in weights
]
def benchmark_up_projection(args: argparse.Namespace) -> None:
if args.tp_size <= 0 or HIDDEN_SIZE % args.tp_size:
raise ValueError("TP size must be positive and divide the hidden size")
if any(not 1 <= num_tokens <= MAX_NUM_TOKENS for num_tokens in args.num_tokens):
raise ValueError("--num-tokens values must be in [1, 16]")
if args.cache_multiplier <= 0 or args.max_weights <= 0:
raise ValueError("cache multiplier and max weights must be positive")
if args.warmup_replays < 0 or args.samples <= 0:
raise ValueError("warmup replays must be nonnegative and samples positive")
torch.accelerator.set_device_index(0)
device = torch.device("cuda", 0)
if torch.cuda.get_device_capability(device)[0] != 10:
raise RuntimeError("Kimi K3 latent-MoE tail requires SM100")
shard_size = HIDDEN_SIZE // args.tp_size
weight_count = rotating_weight_count(
shard_size,
args.cache_multiplier,
args.max_weights,
)
torch.manual_seed(20260726)
weights = [
torch.randn(
(shard_size, LATENT_SIZE),
dtype=torch.bfloat16,
device=device,
)
/ LATENT_SIZE**0.5
for _ in range(weight_count)
]
mailbox = torch.empty(
(1, MAX_NUM_TOKENS, HIDDEN_SIZE),
dtype=torch.bfloat16,
device=device,
)
shared = torch.randn(
(MAX_NUM_TOKENS, HIDDEN_SIZE),
dtype=torch.bfloat16,
device=device,
)
shared_shard = shared[:, :shard_size]
use_dynamic = args.backend in ("dynamic", "both")
use_skinny = args.backend in ("skinny", "both")
dynamic_kernel = (
DynamicKernel(shard_size, mailbox, shared_shard) if use_dynamic else None
)
results = []
for num_tokens in args.num_tokens:
latent = torch.randn(
(num_tokens, LATENT_SIZE),
dtype=torch.bfloat16,
device=device,
)
result: dict[str, Any] = {"num_tokens": num_tokens}
if dynamic_kernel is not None:
launches = make_up_projection_launches(
dynamic_kernel.launch,
latent,
weights,
shared_shard,
)
graph = capture_up_projection_graph(launches)
result["dynamic"] = benchmark_up_projection_graph(
graph,
operations_per_replay=len(launches),
warmup_replays=args.warmup_replays,
samples=args.samples,
)
check_up_projection_output(
mailbox[0, :num_tokens, :shard_size],
latent,
weights[-1],
shared_shard[:num_tokens],
)
if use_skinny:
configs = args.skinny_config or [
fused_add_multicast_skinny_gemm.config_for_m(
num_tokens,
shard_size,
)
]
skinny_results = []
for config in configs:
skinny_kernel = SkinnyKernel(num_tokens, shard_size, config)
def launch_skinny(
latent: torch.Tensor,
weight: torch.Tensor,
shared_shard: torch.Tensor,
*,
skinny_kernel: SkinnyKernel = skinny_kernel,
num_tokens: int = num_tokens,
) -> None:
skinny_kernel.launch(
latent,
weight,
shared_shard[:num_tokens],
mailbox,
)
launches = make_up_projection_launches(
launch_skinny,
latent,
weights,
shared_shard,
)
graph = capture_up_projection_graph(launches)
timing = benchmark_up_projection_graph(
graph,
operations_per_replay=len(launches),
warmup_replays=args.warmup_replays,
samples=args.samples,
)
check_up_projection_output(
mailbox[0, :num_tokens, :shard_size],
latent,
weights[-1],
shared_shard[:num_tokens],
)
skinny_results.append(
{
"config": asdict(config),
**timing,
}
)
result["skinny"] = skinny_results
results.append(result)
properties = torch.cuda.get_device_properties(device)
report = {
"scope": "up-projection",
"device": properties.name,
"compute_capability": list(torch.cuda.get_device_capability(device)),
"tp_size": args.tp_size,
"shard_size": shard_size,
"weight_count": weight_count,
"cache_multiplier": args.cache_multiplier,
"warmup_replays": args.warmup_replays,
"samples": args.samples,
"results": results,
}
rendered = json.dumps(report, indent=2)
print(rendered, flush=True)
if args.output is not None:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(rendered + "\n", encoding="utf-8")
def capture_tail_graph(
operation: Callable[[], torch.Tensor],
cpu_group: dist.ProcessGroup,
) -> tuple[torch.cuda.CUDAGraph, torch.Tensor]:
for _ in range(3):
dist.barrier(group=cpu_group)
output = operation()
torch.accelerator.synchronize()
dist.barrier(group=cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = operation()
torch.accelerator.synchronize()
return graph, output
def benchmark_tail_graph(
graph: torch.cuda.CUDAGraph,
*,
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
) -> dict[str, Any]:
for _ in range(warmup_replays):
graph.replay()
torch.accelerator.synchronize()
dist.barrier(group=cpu_group)
starts = [torch.cuda.Event(enable_timing=True) for _ in range(samples + 1)]
ends = [torch.cuda.Event(enable_timing=True) for _ in range(samples + 1)]
for start, end in zip(starts, ends):
start.record()
graph.replay()
end.record()
torch.accelerator.synchronize()
samples_us = torch.tensor(
[start.elapsed_time(end) * 1000.0 for start, end in zip(starts, ends)],
dtype=torch.float64,
device=torch.accelerator.current_device_index(),
)
dist.all_reduce(samples_us, op=dist.ReduceOp.MAX, group=device_group)
return summarize(samples_us[1:].tolist())
def make_inputs(
num_tokens: int,
rank: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
torch.manual_seed(20260726 + 100 * num_tokens + rank)
routed = torch.randn(
(num_tokens, LATENT_SIZE),
dtype=torch.bfloat16,
device=device,
).mul_(0.01)
shared = torch.randn(
(num_tokens, HIDDEN_SIZE),
dtype=torch.bfloat16,
device=device,
)
return routed, shared
def make_reference(
routed: torch.Tensor,
shared: torch.Tensor,
rms_weight: torch.Tensor,
up_weight: torch.Tensor,
device_group: dist.ProcessGroup,
) -> Callable[[], torch.Tensor]:
routed_workspace = torch.empty_like(routed)
shared_workspace = torch.empty_like(shared)
def reference() -> torch.Tensor:
routed_workspace.copy_(routed)
dist.all_reduce(routed_workspace, group=device_group)
normalized = F.rms_norm(
routed_workspace,
(LATENT_SIZE,),
rms_weight,
RMS_EPS,
)
projected = F.linear(normalized, up_weight)
shared_workspace.copy_(shared)
dist.all_reduce(shared_workspace, group=device_group)
return projected.add(shared_workspace)
return reference
def check_fused_output(
fused_output: torch.Tensor,
reference: Callable[[], torch.Tensor],
cpu_group: dist.ProcessGroup,
) -> None:
dist.barrier(group=cpu_group)
expected = reference()
torch.testing.assert_close(fused_output, expected, atol=8e-2, rtol=3e-2)
def benchmark_whole_tail(args: argparse.Namespace) -> None:
if any(not 1 <= num_tokens <= 16 for num_tokens in args.num_tokens):
raise ValueError("--num-tokens values must be in [1, 16]")
if args.warmup_replays < 0 or args.samples <= 0:
raise ValueError("warmup replays must be nonnegative and samples positive")
if args.skinny_max_num_tokens is not None and any(
not 0 <= cutoff <= 8 for cutoff in args.skinny_max_num_tokens
):
raise ValueError("--skinny-max-num-tokens must be in [0, 8]")
skinny_configs = dict(args.skinny_config or ())
if len(skinny_configs) != len(args.skinny_config or ()):
raise ValueError("--skinny-config must not repeat an M value")
if any(not 1 <= num_tokens <= 8 for num_tokens in skinny_configs):
raise ValueError("--skinny-config M values must be in [1, 8]")
if not {"RANK", "WORLD_SIZE", "LOCAL_RANK"} <= os.environ.keys():
raise RuntimeError("launch this benchmark with torchrun")
rank = int(os.environ["RANK"])
world_size = int(os.environ["WORLD_SIZE"])
local_rank = int(os.environ["LOCAL_RANK"])
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
init_distributed_environment()
if world_size > 8:
set_custom_all_reduce(False)
initialize_model_parallel(tensor_model_parallel_size=world_size)
device_group = get_tp_group().device_group
cpu_group = dist.new_group(backend="gloo")
if torch.cuda.get_device_capability(device)[0] != 10:
raise RuntimeError("Kimi K3 latent-MoE tail requires SM100")
torch.manual_seed(20260726)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
)
up_weight = (
torch.randn(
(HIDDEN_SIZE, LATENT_SIZE),
dtype=torch.bfloat16,
device=device,
)
/ LATENT_SIZE**0.5
)
use_reference = args.backend in ("reference", "both")
use_fused = args.backend in ("fused", "both")
fused_ops = []
if use_fused:
production_config_for_m = fused_add_multicast_skinny_gemm.config_for_m
def config_for_m(
num_rows: int,
shard_dim: int = 896,
) -> fused_add_multicast_skinny_gemm.SkinnyConfig:
config = skinny_configs.get(num_rows)
if config is not None:
return config
return production_config_for_m(num_rows, shard_dim)
fused_add_multicast_skinny_gemm.config_for_m = config_for_m
cutoffs = args.skinny_max_num_tokens or [latent_moe_tail._SKINNY_MAX_NUM_TOKENS]
for cutoff in cutoffs:
latent_moe_tail._SKINNY_MAX_NUM_TOKENS = cutoff
latent_moe_tail.KimiK3LatentMoETailOp._instances.clear()
fused_ops.append(
(
cutoff,
latent_moe_tail.KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=RMS_EPS,
),
)
)
cutedsl_warmup()
results = []
for num_tokens in args.num_tokens:
routed, shared = make_inputs(num_tokens, rank, device)
reference = make_reference(
routed,
shared,
rms_weight,
up_weight,
device_group,
)
result: dict[str, Any] = {"num_tokens": num_tokens}
if use_reference:
reference_graph, _ = capture_tail_graph(reference, cpu_group)
result["reference"] = benchmark_tail_graph(
reference_graph,
warmup_replays=args.warmup_replays,
samples=args.samples,
device_group=device_group,
cpu_group=cpu_group,
)
for cutoff, fused_op in fused_ops:
def fused(
routed: torch.Tensor = routed,
shared: torch.Tensor = shared,
fused_op: latent_moe_tail.KimiK3LatentMoETailOp = fused_op,
) -> torch.Tensor:
return fused_op(routed, shared, rms_weight, up_weight)
fused_graph, fused_output = capture_tail_graph(fused, cpu_group)
fused_key = "fused" if len(fused_ops) == 1 else f"fused_skinny_max_{cutoff}"
result[fused_key] = benchmark_tail_graph(
fused_graph,
warmup_replays=args.warmup_replays,
samples=args.samples,
device_group=device_group,
cpu_group=cpu_group,
)
check_fused_output(fused_output, reference, cpu_group)
if "reference" in result:
speedup = (
result["reference"]["median_us"] / result[fused_key]["median_us"]
)
if len(fused_ops) == 1:
result["speedup"] = speedup
else:
result[f"{fused_key}_speedup"] = speedup
results.append(result)
properties = torch.cuda.get_device_properties(device)
report = {
"scope": "whole-tail",
"device": properties.name,
"compute_capability": list(torch.cuda.get_device_capability(device)),
"world_size": world_size,
"torch_version": torch.__version__,
"cuda_version": torch.version.cuda,
"warmup_replays": args.warmup_replays,
"samples": args.samples,
"skinny_max_num_tokens": [cutoff for cutoff, _ in fused_ops],
"skinny_configs": {
str(num_tokens): asdict(config)
for num_tokens, config in skinny_configs.items()
},
"timing_scope": {
"reference": (
"two input copies, two AllReduces, RMSNorm, full replicated "
"up-projection GEMM, and final add"
),
"fused": (
"routed AllReduce/RMSNorm plus shared ReduceScatter, sharded "
"up-projection/multicast, and Lamport copy"
),
},
"results": results,
}
if rank == 0:
rendered = json.dumps(report, indent=2)
print(rendered, flush=True)
if args.output is not None:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(rendered + "\n", encoding="utf-8")
dist.barrier(group=cpu_group)
def main() -> None:
args = parse_args()
if args.scope == "up-projection":
benchmark_up_projection(args)
return
from vllm.config import VllmConfig, set_current_vllm_config
with set_current_vllm_config(VllmConfig()):
benchmark_whole_tail(args)
if __name__ == "__main__":
main()
@@ -0,0 +1,239 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
import os
import statistics
from collections.abc import Callable
import torch
import torch.distributed as dist
import vllm._custom_ops as ops
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--tokens", type=int, nargs="+", default=[8, 32, 128, 1024])
parser.add_argument("--hidden-size", type=int, default=7168)
parser.add_argument("--graph-repeats", type=int, default=20)
parser.add_argument("--warmup-replays", type=int, default=5)
parser.add_argument("--samples", type=int, default=15)
return parser.parse_args()
def capture_graph(op: Callable[[], None], repeats: int) -> torch.cuda.CUDAGraph:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
op()
stream.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
for _ in range(repeats):
op()
torch.cuda.current_stream().wait_stream(stream)
return graph
def max_rank_graph_time(
graph: torch.cuda.CUDAGraph,
repeats: int,
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
) -> float:
for _ in range(warmup_replays):
graph.replay()
torch.accelerator.synchronize()
timings = []
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
for _ in range(samples):
dist.barrier(group=cpu_group)
start.record()
graph.replay()
end.record()
end.synchronize()
elapsed = torch.tensor(
start.elapsed_time(end) / repeats,
dtype=torch.float64,
device=torch.accelerator.current_device_index(),
)
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
timings.append(elapsed.item())
return statistics.median(timings)
def check_outputs(
comm: CustomAllreduce,
local: torch.Tensor,
reduce_input: torch.Tensor,
device_group: dist.ProcessGroup,
) -> None:
expected_gather = torch.empty(
(local.shape[0] * dist.get_world_size(), local.shape[1]),
dtype=local.dtype,
device=local.device,
)
dist.all_gather_into_tensor(expected_gather, local, group=device_group)
gathered = comm.custom_all_gather(local)
assert gathered is not None
torch.testing.assert_close(gathered, expected_gather)
expected_scatter = torch.empty_like(local)
dist.reduce_scatter_tensor(
expected_scatter,
reduce_input.clone(),
group=device_group,
)
scattered = comm.custom_reduce_scatter(reduce_input)
assert scattered is not None
torch.testing.assert_close(scattered, expected_scatter)
def benchmark_shape(
comm: CustomAllreduce,
global_tokens: int,
hidden_size: int,
graph_repeats: int,
warmup_replays: int,
samples: int,
device_group: dist.ProcessGroup,
cpu_group: dist.ProcessGroup,
) -> dict[str, float | int]:
world_size = dist.get_world_size()
rank = dist.get_rank()
padded_tokens = (global_tokens + world_size - 1) // world_size * world_size
local_tokens = padded_tokens // world_size
local = torch.full(
(local_tokens, hidden_size),
rank + 1,
dtype=torch.bfloat16,
device=torch.accelerator.current_device_index(),
)
reduce_input = torch.full(
(padded_tokens, hidden_size),
rank + 1,
dtype=torch.bfloat16,
device=local.device,
)
check_outputs(comm, local, reduce_input, device_group)
custom_gather_out = torch.empty(
(padded_tokens, hidden_size),
dtype=local.dtype,
device=local.device,
)
custom_scatter_out = torch.empty_like(local)
nccl_gather_out = torch.empty_like(custom_gather_out)
nccl_scatter_out = torch.empty_like(local)
def custom_ag() -> None:
ops.mnnvl_lamport_all_gather(
comm._ptr,
local,
custom_gather_out,
comm.mnnvl_lamport_ag_local_ptr,
comm.mnnvl_lamport_ag_multicast_ptr,
comm.mnnvl_lamport_ag_epoch_ptr,
comm.mnnvl_buffer_size,
)
def custom_rs() -> None:
ops.mnnvl_lamport_reduce_scatter(
comm._ptr,
reduce_input,
custom_scatter_out,
comm.mnnvl_lamport_rs_local_ptr,
comm.mnnvl_lamport_rs_epoch_ptr,
comm.mnnvl_buffer_size,
)
def nccl_ag() -> None:
dist.all_gather_into_tensor(nccl_gather_out, local, group=device_group)
def nccl_rs() -> None:
dist.reduce_scatter_tensor(
nccl_scatter_out,
reduce_input,
group=device_group,
)
graphs = {
"custom_ag_us": capture_graph(custom_ag, graph_repeats),
"nccl_ag_us": capture_graph(nccl_ag, graph_repeats),
"custom_rs_us": capture_graph(custom_rs, graph_repeats),
"nccl_rs_us": capture_graph(nccl_rs, graph_repeats),
}
times = {
name: max_rank_graph_time(
graph,
graph_repeats,
warmup_replays,
samples,
device_group,
cpu_group,
)
* 1000
for name, graph in graphs.items()
}
torch.testing.assert_close(custom_gather_out, nccl_gather_out)
torch.testing.assert_close(custom_scatter_out, nccl_scatter_out)
return {
"global_tokens": global_tokens,
"padded_tokens": padded_tokens,
"local_bytes": local.nbytes,
"full_bytes": reduce_input.nbytes,
**times,
"ag_speedup": times["nccl_ag_us"] / times["custom_ag_us"],
"rs_speedup": times["nccl_rs_us"] / times["custom_rs_us"],
}
def main() -> None:
args = parse_args()
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
dist.init_process_group("nccl")
device_group = dist.group.WORLD
cpu_group = dist.new_group(backend="gloo")
comm = CustomAllreduce(
group=cpu_group,
device=torch.device("cuda", local_rank),
)
assert not comm.disabled
assert comm.world_size == 16
assert comm.mnnvl_only
assert comm.mnnvl_multicast_ptr
results = [
benchmark_shape(
comm,
tokens,
args.hidden_size,
args.graph_repeats,
args.warmup_replays,
args.samples,
device_group,
cpu_group,
)
for tokens in args.tokens
]
if dist.get_rank() == 0:
print(json.dumps(results, indent=2), flush=True)
comm.close()
dist.destroy_process_group(cpu_group)
dist.destroy_process_group()
if __name__ == "__main__":
main()