[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -0,0 +1,367 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the Kimi-K3 latent MoE addmm against CuTe residual GEMM.
|
||||
|
||||
The benchmark covers ``BF16[M, 3584] @ BF16[7168, 3584].T + BF16[M, 7168]``
|
||||
with FP32 accumulation and BF16 output. Both backends execute through CUDA
|
||||
Graph replay. Weights and residuals rotate across buffers exceeding L2 so the
|
||||
comparison models the full latent MoE projection-and-add path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from collections.abc import Callable, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings import driver as cuda
|
||||
from cuda.bindings.driver import CUstream
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
N = 7168
|
||||
K = 3584
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class Config:
|
||||
block_size: int
|
||||
outputs_per_block: int
|
||||
k_unroll: int
|
||||
vector_width: int = 8
|
||||
|
||||
|
||||
def parse_config(value: str) -> Config:
|
||||
try:
|
||||
parts = [int(part) for part in value.split(",")]
|
||||
except ValueError as error:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH]"
|
||||
) from error
|
||||
if len(parts) == 3:
|
||||
return Config(*parts)
|
||||
if len(parts) == 4:
|
||||
return Config(*parts)
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH]"
|
||||
)
|
||||
|
||||
|
||||
def production_residual_config(m: int) -> Config | None:
|
||||
"""The measured Latent-MoE residual config for M, from the K3 table."""
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
|
||||
|
||||
spec = KIMI_K3_PROJECTIONS.get((N, K))
|
||||
config = spec.residual_config(m) if spec is not None else None
|
||||
if config is None:
|
||||
return None
|
||||
return Config(
|
||||
config.block_size,
|
||||
config.outputs_per_block,
|
||||
config.k_unroll,
|
||||
config.vector_width,
|
||||
)
|
||||
|
||||
|
||||
def candidate_configs(mode: str, selected: Config | None, m: int) -> list[Config]:
|
||||
if mode == "selected":
|
||||
if selected is not None:
|
||||
return [selected]
|
||||
# No explicit --config: fall back to the production table for this M.
|
||||
config = production_residual_config(m)
|
||||
return [config] if config is not None else []
|
||||
if mode == "baseline":
|
||||
return [Config(224, 4, 2)]
|
||||
return [
|
||||
Config(block_size, outputs_per_block, k_unroll, vector_width)
|
||||
for vector_width in (4, 8)
|
||||
for block_size in (32, 64, 128, 224, 448)
|
||||
if block_size % 32 == 0 and K % (block_size * vector_width) == 0
|
||||
for outputs_per_block in (1, 2, 4, 7, 8)
|
||||
if N % outputs_per_block == 0
|
||||
for k_unroll in (1, 2, 4)
|
||||
]
|
||||
|
||||
|
||||
def load_kernel_class(path: Path):
|
||||
spec = importlib.util.spec_from_file_location("cute_skinny_device", path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"cannot load CuTe kernel from {path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.CuteSkinnyGemm
|
||||
|
||||
|
||||
def stream() -> CUstream:
|
||||
return CUstream(torch.cuda.current_stream().cuda_stream)
|
||||
|
||||
|
||||
def compile_kernel(kernel_class, m: int, config: Config, max_registers: int):
|
||||
element_type = cutlass.BFloat16
|
||||
n = cute.sym_int(divisibility=config.outputs_per_block)
|
||||
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
a = make_fake_tensor(element_type, (m, k), divisibility=config.vector_width)
|
||||
b = make_fake_tensor(element_type, (n, k), divisibility=config.vector_width)
|
||||
residual = make_fake_tensor(element_type, (m, n), divisibility=1)
|
||||
c = make_fake_tensor(element_type, (m, n), divisibility=1)
|
||||
kernel = kernel_class(
|
||||
element_type=element_type,
|
||||
num_rows=m,
|
||||
block_size=config.block_size,
|
||||
outputs_per_block=config.outputs_per_block,
|
||||
vector_width=config.vector_width,
|
||||
k_unroll=config.k_unroll,
|
||||
has_residual=True,
|
||||
use_pdl=True,
|
||||
)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
a,
|
||||
b,
|
||||
residual,
|
||||
c,
|
||||
stream(),
|
||||
options=(
|
||||
"--enable-tvm-ffi --keep-cubin "
|
||||
f"--ptxas-options -maxrregcount={max_registers} "
|
||||
"--ptxas-options -lineinfo"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resource_usage(compiled) -> dict[str, Any]:
|
||||
executor = getattr(compiled, "_default_executor", None)
|
||||
context = getattr(executor, "exec_context", None)
|
||||
functions = getattr(context, "kernel_functions", None)
|
||||
if not functions:
|
||||
return {"resource_metrics_available": False}
|
||||
|
||||
def attribute(name, function) -> int:
|
||||
error, value = cuda.cuFuncGetAttribute(name, function)
|
||||
if error != cuda.CUresult.CUDA_SUCCESS:
|
||||
raise RuntimeError(f"cuFuncGetAttribute failed with {error}")
|
||||
return int(value)
|
||||
|
||||
registers = [
|
||||
attribute(cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_NUM_REGS, function)
|
||||
for function in functions
|
||||
]
|
||||
local_bytes = [
|
||||
attribute(
|
||||
cuda.CUfunction_attribute.CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES,
|
||||
function,
|
||||
)
|
||||
for function in functions
|
||||
]
|
||||
return {
|
||||
"resource_metrics_available": True,
|
||||
"registers_per_thread": max(registers, default=0),
|
||||
"spill_bytes": max(local_bytes, default=0),
|
||||
}
|
||||
|
||||
|
||||
def rotating_buffer_count(m: int, multiplier: float, limit: int) -> int:
|
||||
properties = torch.cuda.get_device_properties(0)
|
||||
bytes_per_pair = (N * K + m * N) * 2
|
||||
target = math.ceil(multiplier * properties.L2_cache_size)
|
||||
return max(2, min(limit, math.ceil(target / bytes_per_pair)))
|
||||
|
||||
|
||||
def graph_samples(
|
||||
launch: Callable[[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], None],
|
||||
activation: torch.Tensor,
|
||||
weights: Sequence[torch.Tensor],
|
||||
residuals: Sequence[torch.Tensor],
|
||||
repeats: int,
|
||||
replays: int,
|
||||
) -> tuple[list[float], list[torch.Tensor]]:
|
||||
outputs = [torch.empty_like(residual) for residual in residuals]
|
||||
for weight, residual, output in zip(weights, residuals, outputs):
|
||||
launch(activation, weight, residual, output)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for weight, residual, output in zip(weights, residuals, outputs):
|
||||
launch(activation, weight, residual, output)
|
||||
for _ in range(20):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
samples = []
|
||||
for _ in range(repeats):
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
samples.append(start.elapsed_time(end) * 1000.0 / (replays * len(weights)))
|
||||
return samples, outputs
|
||||
|
||||
|
||||
def summarize(samples: Sequence[float]) -> dict[str, Any]:
|
||||
ordered = sorted(samples)
|
||||
|
||||
def percentile(fraction: float) -> float:
|
||||
position = fraction * (len(ordered) - 1)
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
weight = position - lower
|
||||
return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
|
||||
|
||||
mean = statistics.mean(samples)
|
||||
return {
|
||||
"median_us": statistics.median(samples),
|
||||
"p10_us": percentile(0.1),
|
||||
"p90_us": percentile(0.9),
|
||||
"mean_us": mean,
|
||||
"cv_pct": statistics.pstdev(samples) / mean * 100.0,
|
||||
"samples_us": list(samples),
|
||||
}
|
||||
|
||||
|
||||
def correctness(
|
||||
output: torch.Tensor,
|
||||
activation: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> dict[str, Any]:
|
||||
actual = output.float()
|
||||
reference = activation.float() @ weight.float().t() + residual.float()
|
||||
error = (actual - reference).abs()
|
||||
scaled_error = error / (reference.abs() + 1.0)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
actual.flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
return {
|
||||
"valid": cosine > 0.999,
|
||||
"cosine": cosine,
|
||||
"max_abs_error": error.max().item(),
|
||||
"max_scaled_error": scaled_error.max().item(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--kernel", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--mode", choices=("baseline", "sweep", "selected"), default="baseline"
|
||||
)
|
||||
parser.add_argument("--config", type=parse_config)
|
||||
parser.add_argument("--m", type=int, action="append")
|
||||
parser.add_argument("--config-shard", type=int, default=0)
|
||||
parser.add_argument("--num-config-shards", type=int, default=1)
|
||||
parser.add_argument("--repeats", type=int, default=21)
|
||||
parser.add_argument("--replays", type=int, default=200)
|
||||
parser.add_argument("--cache-multiplier", type=float, default=3.0)
|
||||
parser.add_argument("--max-buffers", type=int, default=32)
|
||||
parser.add_argument("--max-registers", type=int, default=64)
|
||||
args = parser.parse_args()
|
||||
|
||||
token_counts = args.m or list(range(1, 17))
|
||||
if any(not 1 <= m <= 16 for m in token_counts):
|
||||
raise ValueError("expected 1 <= M <= 16")
|
||||
if not 0 <= args.config_shard < args.num_config_shards:
|
||||
raise ValueError("config shard must be in [0, num_config_shards)")
|
||||
torch.accelerator.set_device_index(0)
|
||||
if torch.cuda.get_device_capability() != (10, 3):
|
||||
raise RuntimeError("this benchmark requires SM103")
|
||||
|
||||
kernel_class = load_kernel_class(args.kernel)
|
||||
properties = torch.cuda.get_device_properties(0)
|
||||
metadata = {
|
||||
"device": properties.name,
|
||||
"compute_capability": list(torch.cuda.get_device_capability()),
|
||||
"torch_version": torch.__version__,
|
||||
"cuda_version": torch.version.cuda,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("w", encoding="utf-8") as output_file:
|
||||
for m in token_counts:
|
||||
configs = candidate_configs(args.mode, args.config, m)
|
||||
torch.manual_seed(20260722 + m)
|
||||
count = rotating_buffer_count(m, args.cache_multiplier, args.max_buffers)
|
||||
activation = torch.randn((m, K), device="cuda", dtype=torch.bfloat16)
|
||||
weights = [
|
||||
torch.randn((N, K), device="cuda", dtype=torch.bfloat16)
|
||||
for _ in range(count)
|
||||
]
|
||||
residuals = [
|
||||
torch.randn((m, N), device="cuda", dtype=torch.bfloat16)
|
||||
for _ in range(count)
|
||||
]
|
||||
candidates: list[tuple[str, Config | None]] = [("cublas_addmm", None)]
|
||||
candidates.extend(
|
||||
("cute_residual", config)
|
||||
for index, config in enumerate(configs)
|
||||
if index % args.num_config_shards == args.config_shard
|
||||
)
|
||||
for backend, config in candidates:
|
||||
row: dict[str, Any] = {
|
||||
"m": m,
|
||||
"n": N,
|
||||
"k": K,
|
||||
"backend": backend,
|
||||
"mode": args.mode,
|
||||
"config": dataclasses.asdict(config) if config else {},
|
||||
"num_buffers": count,
|
||||
"cache_multiplier": args.cache_multiplier,
|
||||
**metadata,
|
||||
}
|
||||
try:
|
||||
if backend == "cublas_addmm":
|
||||
launch = lambda a, b, residual, c: torch.addmm(
|
||||
residual, a, b.t(), out=c
|
||||
)
|
||||
else:
|
||||
if config is None:
|
||||
raise AssertionError("missing CuTe config")
|
||||
compiled = compile_kernel(
|
||||
kernel_class, m, config, args.max_registers
|
||||
)
|
||||
launch = lambda a, b, residual, c, fn=compiled: fn(
|
||||
a, b, residual, c, stream()
|
||||
)
|
||||
row.update(resource_usage(compiled))
|
||||
samples, outputs = graph_samples(
|
||||
launch,
|
||||
activation,
|
||||
weights,
|
||||
residuals,
|
||||
args.repeats,
|
||||
args.replays,
|
||||
)
|
||||
row.update(
|
||||
correctness(outputs[0], activation, weights[0], residuals[0])
|
||||
)
|
||||
row.update(summarize(samples))
|
||||
except Exception as error: # noqa: BLE001
|
||||
row.update(
|
||||
{
|
||||
"valid": False,
|
||||
"error": f"{type(error).__name__}: {error}",
|
||||
}
|
||||
)
|
||||
output_file.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
output_file.flush()
|
||||
print(json.dumps(row, sort_keys=True), flush=True)
|
||||
|
||||
del activation, weights, residuals
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,806 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the Kimi K3 latent-MoE tail and its up-projection kernels.
|
||||
|
||||
The ``up-projection`` subcommand isolates the TP-local dynamic and static-M
|
||||
skinny GEMMs. It rotates weights through a working set larger than L2 to model
|
||||
successive model layers.
|
||||
|
||||
The ``whole-tail`` subcommand measures the distributed operator. Its reference
|
||||
path includes two AllReduces, RMSNorm, the replicated up-projection, and the
|
||||
final add. CUDA-event samples report the slowest rank so cross-rank skew is
|
||||
included.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
.venv/bin/python \
|
||||
benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py up-projection
|
||||
|
||||
torchrun --nproc-per-node=8 \
|
||||
benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py whole-tail
|
||||
|
||||
For multi-node runs, launch one ``torchrun`` agent per node and use a shared
|
||||
rendezvous endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.utils as utils
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
from cuda.bindings import driver as cuda
|
||||
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
set_custom_all_reduce,
|
||||
)
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.models.kimi_k3.nvidia.ops import latent_moe_tail
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.latent_moe_tail import (
|
||||
fused_add_multicast_gemm,
|
||||
fused_add_multicast_skinny_gemm,
|
||||
)
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
RMS_EPS = 0.1
|
||||
MAX_NUM_TOKENS = 16
|
||||
MMA_TILER_MN = (64, 32)
|
||||
CLUSTER_SHAPE_MN = (1, 8)
|
||||
B_PRIME_STAGES = 2
|
||||
|
||||
|
||||
def parse_up_projection_config(
|
||||
value: str,
|
||||
) -> fused_add_multicast_skinny_gemm.SkinnyConfig:
|
||||
try:
|
||||
values = [int(part) for part in value.split(",")]
|
||||
except ValueError as error:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH[,PREFETCH_B]]"
|
||||
) from error
|
||||
if len(values) in (3, 4):
|
||||
return fused_add_multicast_skinny_gemm.SkinnyConfig(*values)
|
||||
if len(values) == 5 and values[4] in (0, 1):
|
||||
return fused_add_multicast_skinny_gemm.SkinnyConfig(
|
||||
*values[:4],
|
||||
prefetch_b_before_pdl=bool(values[4]),
|
||||
)
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be BLOCK,OUTPUTS,K_UNROLL"
|
||||
"[,VECTOR_WIDTH[,PREFETCH_B]], where PREFETCH_B is 0 or 1"
|
||||
)
|
||||
|
||||
|
||||
def parse_tail_skinny_config(
|
||||
value: str,
|
||||
) -> tuple[int, fused_add_multicast_skinny_gemm.SkinnyConfig]:
|
||||
try:
|
||||
values = [int(part) for part in value.split(",")]
|
||||
except ValueError as error:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be M,BLOCK,OUTPUTS,K_UNROLL[,VECTOR_WIDTH[,PREFETCH_B]]"
|
||||
) from error
|
||||
if len(values) == 4:
|
||||
num_tokens, *config = values
|
||||
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(*config)
|
||||
if len(values) == 5:
|
||||
num_tokens, *config = values
|
||||
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(*config)
|
||||
if len(values) == 6 and values[5] in (0, 1):
|
||||
num_tokens, block, outputs, unroll, vector_width, prefetch = values
|
||||
return num_tokens, fused_add_multicast_skinny_gemm.SkinnyConfig(
|
||||
block,
|
||||
outputs,
|
||||
unroll,
|
||||
vector_width,
|
||||
bool(prefetch),
|
||||
)
|
||||
raise argparse.ArgumentTypeError(
|
||||
"config must be M,BLOCK,OUTPUTS,K_UNROLL"
|
||||
"[,VECTOR_WIDTH[,PREFETCH_B]], where PREFETCH_B is 0 or 1"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="scope", required=True)
|
||||
|
||||
up_projection = subparsers.add_parser(
|
||||
"up-projection",
|
||||
help="Benchmark the isolated TP-local up-projection kernels.",
|
||||
)
|
||||
up_projection.add_argument(
|
||||
"--backend",
|
||||
choices=("dynamic", "skinny", "both"),
|
||||
default="both",
|
||||
)
|
||||
up_projection.add_argument("--tp-size", type=int, default=16)
|
||||
up_projection.add_argument(
|
||||
"--num-tokens",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[*range(1, 9), 16],
|
||||
)
|
||||
up_projection.add_argument(
|
||||
"--skinny-config",
|
||||
type=parse_up_projection_config,
|
||||
action="append",
|
||||
help="Benchmark a static-M config for every selected token count.",
|
||||
)
|
||||
up_projection.add_argument("--cache-multiplier", type=float, default=2.0)
|
||||
up_projection.add_argument("--max-weights", type=int, default=64)
|
||||
up_projection.add_argument("--warmup-replays", type=int, default=10)
|
||||
up_projection.add_argument("--samples", type=int, default=31)
|
||||
up_projection.add_argument("--output", type=Path)
|
||||
|
||||
whole_tail = subparsers.add_parser(
|
||||
"whole-tail",
|
||||
help="Benchmark the distributed latent-MoE tail operator.",
|
||||
)
|
||||
whole_tail.add_argument(
|
||||
"--backend",
|
||||
choices=("reference", "fused", "both"),
|
||||
default="both",
|
||||
)
|
||||
whole_tail.add_argument(
|
||||
"--num-tokens",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1, 5, 8, 16],
|
||||
)
|
||||
whole_tail.add_argument("--warmup-replays", type=int, default=20)
|
||||
whole_tail.add_argument("--samples", type=int, default=51)
|
||||
whole_tail.add_argument(
|
||||
"--skinny-max-num-tokens",
|
||||
type=int,
|
||||
nargs="+",
|
||||
help="Override the fused operator's static-M cutoff; use 0 for dynamic-only.",
|
||||
)
|
||||
whole_tail.add_argument(
|
||||
"--skinny-config",
|
||||
type=parse_tail_skinny_config,
|
||||
action="append",
|
||||
help="Override one static-M config for tuning.",
|
||||
)
|
||||
whole_tail.add_argument("--output", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def percentile(samples: Sequence[float], fraction: float) -> float:
|
||||
ordered = sorted(samples)
|
||||
position = fraction * (len(ordered) - 1)
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
upper_weight = position - lower
|
||||
return ordered[lower] * (1.0 - upper_weight) + ordered[upper] * upper_weight
|
||||
|
||||
|
||||
def summarize(samples_us: Sequence[float]) -> dict[str, Any]:
|
||||
mean_us = statistics.mean(samples_us)
|
||||
return {
|
||||
"median_us": statistics.median(samples_us),
|
||||
"p10_us": percentile(samples_us, 0.1),
|
||||
"p90_us": percentile(samples_us, 0.9),
|
||||
"mean_us": mean_us,
|
||||
"cv_pct": statistics.pstdev(samples_us) / mean_us * 100.0,
|
||||
"samples_us": list(samples_us),
|
||||
}
|
||||
|
||||
|
||||
def rotating_weight_count(
|
||||
shard_size: int,
|
||||
cache_multiplier: float,
|
||||
limit: int,
|
||||
) -> int:
|
||||
properties = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
)
|
||||
weight_bytes = shard_size * LATENT_SIZE * 2
|
||||
target_bytes = math.ceil(properties.L2_cache_size * cache_multiplier)
|
||||
return max(2, min(limit, math.ceil(target_bytes / weight_bytes)))
|
||||
|
||||
|
||||
def capture_up_projection_graph(
|
||||
launches: Sequence[Callable[[], None]],
|
||||
) -> torch.cuda.CUDAGraph:
|
||||
for launch in launches:
|
||||
launch()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for launch in launches:
|
||||
launch()
|
||||
torch.accelerator.synchronize()
|
||||
return graph
|
||||
|
||||
|
||||
def benchmark_up_projection_graph(
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
*,
|
||||
operations_per_replay: int,
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
) -> dict[str, Any]:
|
||||
for _ in range(warmup_replays):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
samples_us = []
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
for _ in range(samples):
|
||||
start.record()
|
||||
graph.replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
samples_us.append(start.elapsed_time(end) * 1000.0 / operations_per_replay)
|
||||
return summarize(samples_us)
|
||||
|
||||
|
||||
class DynamicKernel:
|
||||
def __init__(
|
||||
self,
|
||||
shard_size: int,
|
||||
mailbox: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
) -> None:
|
||||
self.shard_size = shard_size
|
||||
self.mailbox = mailbox
|
||||
self.mailbox_c = fused_add_multicast_gemm._as_cute(mailbox)
|
||||
compile_latent = torch.empty(
|
||||
(1, MAX_NUM_TOKENS, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=mailbox.device,
|
||||
)
|
||||
compile_weight = torch.empty(
|
||||
(1, shard_size, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=mailbox.device,
|
||||
)
|
||||
cluster_size = math.prod(CLUSTER_SHAPE_MN)
|
||||
max_active_clusters = utils.HardwareInfo().get_max_active_clusters(cluster_size)
|
||||
self.compiled = fused_add_multicast_gemm.compile_kernel(
|
||||
(MAX_NUM_TOKENS, shard_size, LATENT_SIZE, 1),
|
||||
fused_add_multicast_gemm._as_cute(
|
||||
compile_latent,
|
||||
dynamic_m=True,
|
||||
),
|
||||
fused_add_multicast_gemm._as_cute(compile_weight),
|
||||
self.mailbox_c,
|
||||
fused_add_multicast_gemm._as_cute(shared_shard),
|
||||
HIDDEN_SIZE,
|
||||
shard_size,
|
||||
MMA_TILER_MN,
|
||||
CLUSTER_SHAPE_MN,
|
||||
max_active_clusters,
|
||||
B_PRIME_STAGES,
|
||||
)
|
||||
|
||||
def launch(
|
||||
self,
|
||||
latent: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
) -> None:
|
||||
stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)
|
||||
self.compiled(
|
||||
fused_add_multicast_gemm._as_cute(
|
||||
latent.unsqueeze(0),
|
||||
dynamic_m=True,
|
||||
),
|
||||
fused_add_multicast_gemm._as_cute(weight.unsqueeze(0)),
|
||||
self.mailbox_c,
|
||||
fused_add_multicast_gemm._as_cute(shared_shard),
|
||||
cutlass.Int64(latent.shape[0]),
|
||||
cutlass.Int64(self.mailbox.data_ptr()),
|
||||
stream,
|
||||
)
|
||||
|
||||
|
||||
class SkinnyKernel:
|
||||
def __init__(
|
||||
self,
|
||||
num_tokens: int,
|
||||
shard_size: int,
|
||||
config: fused_add_multicast_skinny_gemm.SkinnyConfig,
|
||||
) -> None:
|
||||
self.compiled = fused_add_multicast_skinny_gemm.compile_kernel(
|
||||
num_rows=num_tokens,
|
||||
latent_dim=LATENT_SIZE,
|
||||
hidden_dim=HIDDEN_SIZE,
|
||||
shard_dim=shard_size,
|
||||
config=config,
|
||||
)
|
||||
|
||||
def launch(
|
||||
self,
|
||||
latent: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
mailbox: torch.Tensor,
|
||||
) -> None:
|
||||
self.compiled(
|
||||
fused_add_multicast_skinny_gemm._as_cute(latent),
|
||||
fused_add_multicast_skinny_gemm._as_cute(weight),
|
||||
fused_add_multicast_skinny_gemm._as_cute(shared_shard),
|
||||
cutlass.Int64(mailbox.data_ptr()),
|
||||
cuda.CUstream(torch.cuda.current_stream().cuda_stream),
|
||||
)
|
||||
|
||||
|
||||
def check_up_projection_output(
|
||||
actual: torch.Tensor,
|
||||
latent: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
) -> None:
|
||||
gemm = F.linear(latent.float(), weight.float()).to(torch.bfloat16)
|
||||
expected = (gemm.float() + shared_shard.float()).to(torch.bfloat16)
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def make_up_projection_launches(
|
||||
launch: Callable[[torch.Tensor, torch.Tensor, torch.Tensor], None],
|
||||
latent: torch.Tensor,
|
||||
weights: Sequence[torch.Tensor],
|
||||
shared_shard: torch.Tensor,
|
||||
) -> list[Callable[[], None]]:
|
||||
return [
|
||||
lambda weight=weight: launch(latent, weight, shared_shard) for weight in weights
|
||||
]
|
||||
|
||||
|
||||
def benchmark_up_projection(args: argparse.Namespace) -> None:
|
||||
if args.tp_size <= 0 or HIDDEN_SIZE % args.tp_size:
|
||||
raise ValueError("TP size must be positive and divide the hidden size")
|
||||
if any(not 1 <= num_tokens <= MAX_NUM_TOKENS for num_tokens in args.num_tokens):
|
||||
raise ValueError("--num-tokens values must be in [1, 16]")
|
||||
if args.cache_multiplier <= 0 or args.max_weights <= 0:
|
||||
raise ValueError("cache multiplier and max weights must be positive")
|
||||
if args.warmup_replays < 0 or args.samples <= 0:
|
||||
raise ValueError("warmup replays must be nonnegative and samples positive")
|
||||
|
||||
torch.accelerator.set_device_index(0)
|
||||
device = torch.device("cuda", 0)
|
||||
if torch.cuda.get_device_capability(device)[0] != 10:
|
||||
raise RuntimeError("Kimi K3 latent-MoE tail requires SM100")
|
||||
|
||||
shard_size = HIDDEN_SIZE // args.tp_size
|
||||
weight_count = rotating_weight_count(
|
||||
shard_size,
|
||||
args.cache_multiplier,
|
||||
args.max_weights,
|
||||
)
|
||||
torch.manual_seed(20260726)
|
||||
weights = [
|
||||
torch.randn(
|
||||
(shard_size, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
for _ in range(weight_count)
|
||||
]
|
||||
mailbox = torch.empty(
|
||||
(1, MAX_NUM_TOKENS, HIDDEN_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
shared = torch.randn(
|
||||
(MAX_NUM_TOKENS, HIDDEN_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
shared_shard = shared[:, :shard_size]
|
||||
use_dynamic = args.backend in ("dynamic", "both")
|
||||
use_skinny = args.backend in ("skinny", "both")
|
||||
dynamic_kernel = (
|
||||
DynamicKernel(shard_size, mailbox, shared_shard) if use_dynamic else None
|
||||
)
|
||||
|
||||
results = []
|
||||
for num_tokens in args.num_tokens:
|
||||
latent = torch.randn(
|
||||
(num_tokens, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
result: dict[str, Any] = {"num_tokens": num_tokens}
|
||||
if dynamic_kernel is not None:
|
||||
launches = make_up_projection_launches(
|
||||
dynamic_kernel.launch,
|
||||
latent,
|
||||
weights,
|
||||
shared_shard,
|
||||
)
|
||||
graph = capture_up_projection_graph(launches)
|
||||
result["dynamic"] = benchmark_up_projection_graph(
|
||||
graph,
|
||||
operations_per_replay=len(launches),
|
||||
warmup_replays=args.warmup_replays,
|
||||
samples=args.samples,
|
||||
)
|
||||
check_up_projection_output(
|
||||
mailbox[0, :num_tokens, :shard_size],
|
||||
latent,
|
||||
weights[-1],
|
||||
shared_shard[:num_tokens],
|
||||
)
|
||||
if use_skinny:
|
||||
configs = args.skinny_config or [
|
||||
fused_add_multicast_skinny_gemm.config_for_m(
|
||||
num_tokens,
|
||||
shard_size,
|
||||
)
|
||||
]
|
||||
skinny_results = []
|
||||
for config in configs:
|
||||
skinny_kernel = SkinnyKernel(num_tokens, shard_size, config)
|
||||
|
||||
def launch_skinny(
|
||||
latent: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shared_shard: torch.Tensor,
|
||||
*,
|
||||
skinny_kernel: SkinnyKernel = skinny_kernel,
|
||||
num_tokens: int = num_tokens,
|
||||
) -> None:
|
||||
skinny_kernel.launch(
|
||||
latent,
|
||||
weight,
|
||||
shared_shard[:num_tokens],
|
||||
mailbox,
|
||||
)
|
||||
|
||||
launches = make_up_projection_launches(
|
||||
launch_skinny,
|
||||
latent,
|
||||
weights,
|
||||
shared_shard,
|
||||
)
|
||||
graph = capture_up_projection_graph(launches)
|
||||
timing = benchmark_up_projection_graph(
|
||||
graph,
|
||||
operations_per_replay=len(launches),
|
||||
warmup_replays=args.warmup_replays,
|
||||
samples=args.samples,
|
||||
)
|
||||
check_up_projection_output(
|
||||
mailbox[0, :num_tokens, :shard_size],
|
||||
latent,
|
||||
weights[-1],
|
||||
shared_shard[:num_tokens],
|
||||
)
|
||||
skinny_results.append(
|
||||
{
|
||||
"config": asdict(config),
|
||||
**timing,
|
||||
}
|
||||
)
|
||||
result["skinny"] = skinny_results
|
||||
results.append(result)
|
||||
|
||||
properties = torch.cuda.get_device_properties(device)
|
||||
report = {
|
||||
"scope": "up-projection",
|
||||
"device": properties.name,
|
||||
"compute_capability": list(torch.cuda.get_device_capability(device)),
|
||||
"tp_size": args.tp_size,
|
||||
"shard_size": shard_size,
|
||||
"weight_count": weight_count,
|
||||
"cache_multiplier": args.cache_multiplier,
|
||||
"warmup_replays": args.warmup_replays,
|
||||
"samples": args.samples,
|
||||
"results": results,
|
||||
}
|
||||
rendered = json.dumps(report, indent=2)
|
||||
print(rendered, flush=True)
|
||||
if args.output is not None:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(rendered + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def capture_tail_graph(
|
||||
operation: Callable[[], torch.Tensor],
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> tuple[torch.cuda.CUDAGraph, torch.Tensor]:
|
||||
for _ in range(3):
|
||||
dist.barrier(group=cpu_group)
|
||||
output = operation()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
dist.barrier(group=cpu_group)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = operation()
|
||||
torch.accelerator.synchronize()
|
||||
return graph, output
|
||||
|
||||
|
||||
def benchmark_tail_graph(
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
*,
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
device_group: dist.ProcessGroup,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> dict[str, Any]:
|
||||
for _ in range(warmup_replays):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
dist.barrier(group=cpu_group)
|
||||
starts = [torch.cuda.Event(enable_timing=True) for _ in range(samples + 1)]
|
||||
ends = [torch.cuda.Event(enable_timing=True) for _ in range(samples + 1)]
|
||||
for start, end in zip(starts, ends):
|
||||
start.record()
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
samples_us = torch.tensor(
|
||||
[start.elapsed_time(end) * 1000.0 for start, end in zip(starts, ends)],
|
||||
dtype=torch.float64,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
dist.all_reduce(samples_us, op=dist.ReduceOp.MAX, group=device_group)
|
||||
return summarize(samples_us[1:].tolist())
|
||||
|
||||
|
||||
def make_inputs(
|
||||
num_tokens: int,
|
||||
rank: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
torch.manual_seed(20260726 + 100 * num_tokens + rank)
|
||||
routed = torch.randn(
|
||||
(num_tokens, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
).mul_(0.01)
|
||||
shared = torch.randn(
|
||||
(num_tokens, HIDDEN_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
return routed, shared
|
||||
|
||||
|
||||
def make_reference(
|
||||
routed: torch.Tensor,
|
||||
shared: torch.Tensor,
|
||||
rms_weight: torch.Tensor,
|
||||
up_weight: torch.Tensor,
|
||||
device_group: dist.ProcessGroup,
|
||||
) -> Callable[[], torch.Tensor]:
|
||||
routed_workspace = torch.empty_like(routed)
|
||||
shared_workspace = torch.empty_like(shared)
|
||||
|
||||
def reference() -> torch.Tensor:
|
||||
routed_workspace.copy_(routed)
|
||||
dist.all_reduce(routed_workspace, group=device_group)
|
||||
normalized = F.rms_norm(
|
||||
routed_workspace,
|
||||
(LATENT_SIZE,),
|
||||
rms_weight,
|
||||
RMS_EPS,
|
||||
)
|
||||
projected = F.linear(normalized, up_weight)
|
||||
shared_workspace.copy_(shared)
|
||||
dist.all_reduce(shared_workspace, group=device_group)
|
||||
return projected.add(shared_workspace)
|
||||
|
||||
return reference
|
||||
|
||||
|
||||
def check_fused_output(
|
||||
fused_output: torch.Tensor,
|
||||
reference: Callable[[], torch.Tensor],
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> None:
|
||||
dist.barrier(group=cpu_group)
|
||||
expected = reference()
|
||||
torch.testing.assert_close(fused_output, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def benchmark_whole_tail(args: argparse.Namespace) -> None:
|
||||
if any(not 1 <= num_tokens <= 16 for num_tokens in args.num_tokens):
|
||||
raise ValueError("--num-tokens values must be in [1, 16]")
|
||||
if args.warmup_replays < 0 or args.samples <= 0:
|
||||
raise ValueError("warmup replays must be nonnegative and samples positive")
|
||||
if args.skinny_max_num_tokens is not None and any(
|
||||
not 0 <= cutoff <= 8 for cutoff in args.skinny_max_num_tokens
|
||||
):
|
||||
raise ValueError("--skinny-max-num-tokens must be in [0, 8]")
|
||||
skinny_configs = dict(args.skinny_config or ())
|
||||
if len(skinny_configs) != len(args.skinny_config or ()):
|
||||
raise ValueError("--skinny-config must not repeat an M value")
|
||||
if any(not 1 <= num_tokens <= 8 for num_tokens in skinny_configs):
|
||||
raise ValueError("--skinny-config M values must be in [1, 8]")
|
||||
if not {"RANK", "WORLD_SIZE", "LOCAL_RANK"} <= os.environ.keys():
|
||||
raise RuntimeError("launch this benchmark with torchrun")
|
||||
|
||||
rank = int(os.environ["RANK"])
|
||||
world_size = int(os.environ["WORLD_SIZE"])
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_distributed_environment()
|
||||
if world_size > 8:
|
||||
set_custom_all_reduce(False)
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
device_group = get_tp_group().device_group
|
||||
cpu_group = dist.new_group(backend="gloo")
|
||||
|
||||
if torch.cuda.get_device_capability(device)[0] != 10:
|
||||
raise RuntimeError("Kimi K3 latent-MoE tail requires SM100")
|
||||
|
||||
torch.manual_seed(20260726)
|
||||
rms_weight = 1 + 0.1 * torch.randn(
|
||||
LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
up_weight = (
|
||||
torch.randn(
|
||||
(HIDDEN_SIZE, LATENT_SIZE),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
|
||||
use_reference = args.backend in ("reference", "both")
|
||||
use_fused = args.backend in ("fused", "both")
|
||||
fused_ops = []
|
||||
if use_fused:
|
||||
production_config_for_m = fused_add_multicast_skinny_gemm.config_for_m
|
||||
|
||||
def config_for_m(
|
||||
num_rows: int,
|
||||
shard_dim: int = 896,
|
||||
) -> fused_add_multicast_skinny_gemm.SkinnyConfig:
|
||||
config = skinny_configs.get(num_rows)
|
||||
if config is not None:
|
||||
return config
|
||||
return production_config_for_m(num_rows, shard_dim)
|
||||
|
||||
fused_add_multicast_skinny_gemm.config_for_m = config_for_m
|
||||
cutoffs = args.skinny_max_num_tokens or [latent_moe_tail._SKINNY_MAX_NUM_TOKENS]
|
||||
for cutoff in cutoffs:
|
||||
latent_moe_tail._SKINNY_MAX_NUM_TOKENS = cutoff
|
||||
latent_moe_tail.KimiK3LatentMoETailOp._instances.clear()
|
||||
fused_ops.append(
|
||||
(
|
||||
cutoff,
|
||||
latent_moe_tail.KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=RMS_EPS,
|
||||
),
|
||||
)
|
||||
)
|
||||
cutedsl_warmup()
|
||||
|
||||
results = []
|
||||
for num_tokens in args.num_tokens:
|
||||
routed, shared = make_inputs(num_tokens, rank, device)
|
||||
reference = make_reference(
|
||||
routed,
|
||||
shared,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
device_group,
|
||||
)
|
||||
result: dict[str, Any] = {"num_tokens": num_tokens}
|
||||
if use_reference:
|
||||
reference_graph, _ = capture_tail_graph(reference, cpu_group)
|
||||
result["reference"] = benchmark_tail_graph(
|
||||
reference_graph,
|
||||
warmup_replays=args.warmup_replays,
|
||||
samples=args.samples,
|
||||
device_group=device_group,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
for cutoff, fused_op in fused_ops:
|
||||
|
||||
def fused(
|
||||
routed: torch.Tensor = routed,
|
||||
shared: torch.Tensor = shared,
|
||||
fused_op: latent_moe_tail.KimiK3LatentMoETailOp = fused_op,
|
||||
) -> torch.Tensor:
|
||||
return fused_op(routed, shared, rms_weight, up_weight)
|
||||
|
||||
fused_graph, fused_output = capture_tail_graph(fused, cpu_group)
|
||||
fused_key = "fused" if len(fused_ops) == 1 else f"fused_skinny_max_{cutoff}"
|
||||
result[fused_key] = benchmark_tail_graph(
|
||||
fused_graph,
|
||||
warmup_replays=args.warmup_replays,
|
||||
samples=args.samples,
|
||||
device_group=device_group,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
check_fused_output(fused_output, reference, cpu_group)
|
||||
if "reference" in result:
|
||||
speedup = (
|
||||
result["reference"]["median_us"] / result[fused_key]["median_us"]
|
||||
)
|
||||
if len(fused_ops) == 1:
|
||||
result["speedup"] = speedup
|
||||
else:
|
||||
result[f"{fused_key}_speedup"] = speedup
|
||||
results.append(result)
|
||||
|
||||
properties = torch.cuda.get_device_properties(device)
|
||||
report = {
|
||||
"scope": "whole-tail",
|
||||
"device": properties.name,
|
||||
"compute_capability": list(torch.cuda.get_device_capability(device)),
|
||||
"world_size": world_size,
|
||||
"torch_version": torch.__version__,
|
||||
"cuda_version": torch.version.cuda,
|
||||
"warmup_replays": args.warmup_replays,
|
||||
"samples": args.samples,
|
||||
"skinny_max_num_tokens": [cutoff for cutoff, _ in fused_ops],
|
||||
"skinny_configs": {
|
||||
str(num_tokens): asdict(config)
|
||||
for num_tokens, config in skinny_configs.items()
|
||||
},
|
||||
"timing_scope": {
|
||||
"reference": (
|
||||
"two input copies, two AllReduces, RMSNorm, full replicated "
|
||||
"up-projection GEMM, and final add"
|
||||
),
|
||||
"fused": (
|
||||
"routed AllReduce/RMSNorm plus shared ReduceScatter, sharded "
|
||||
"up-projection/multicast, and Lamport copy"
|
||||
),
|
||||
},
|
||||
"results": results,
|
||||
}
|
||||
if rank == 0:
|
||||
rendered = json.dumps(report, indent=2)
|
||||
print(rendered, flush=True)
|
||||
if args.output is not None:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(rendered + "\n", encoding="utf-8")
|
||||
|
||||
dist.barrier(group=cpu_group)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.scope == "up-projection":
|
||||
benchmark_up_projection(args)
|
||||
return
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
benchmark_whole_tail(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,239 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tokens", type=int, nargs="+", default=[8, 32, 128, 1024])
|
||||
parser.add_argument("--hidden-size", type=int, default=7168)
|
||||
parser.add_argument("--graph-repeats", type=int, default=20)
|
||||
parser.add_argument("--warmup-replays", type=int, default=5)
|
||||
parser.add_argument("--samples", type=int, default=15)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def capture_graph(op: Callable[[], None], repeats: int) -> torch.cuda.CUDAGraph:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
op()
|
||||
stream.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
for _ in range(repeats):
|
||||
op()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
return graph
|
||||
|
||||
|
||||
def max_rank_graph_time(
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
repeats: int,
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
device_group: dist.ProcessGroup,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> float:
|
||||
for _ in range(warmup_replays):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
timings = []
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
for _ in range(samples):
|
||||
dist.barrier(group=cpu_group)
|
||||
start.record()
|
||||
graph.replay()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
elapsed = torch.tensor(
|
||||
start.elapsed_time(end) / repeats,
|
||||
dtype=torch.float64,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
dist.all_reduce(elapsed, op=dist.ReduceOp.MAX, group=device_group)
|
||||
timings.append(elapsed.item())
|
||||
return statistics.median(timings)
|
||||
|
||||
|
||||
def check_outputs(
|
||||
comm: CustomAllreduce,
|
||||
local: torch.Tensor,
|
||||
reduce_input: torch.Tensor,
|
||||
device_group: dist.ProcessGroup,
|
||||
) -> None:
|
||||
expected_gather = torch.empty(
|
||||
(local.shape[0] * dist.get_world_size(), local.shape[1]),
|
||||
dtype=local.dtype,
|
||||
device=local.device,
|
||||
)
|
||||
dist.all_gather_into_tensor(expected_gather, local, group=device_group)
|
||||
gathered = comm.custom_all_gather(local)
|
||||
assert gathered is not None
|
||||
torch.testing.assert_close(gathered, expected_gather)
|
||||
|
||||
expected_scatter = torch.empty_like(local)
|
||||
dist.reduce_scatter_tensor(
|
||||
expected_scatter,
|
||||
reduce_input.clone(),
|
||||
group=device_group,
|
||||
)
|
||||
scattered = comm.custom_reduce_scatter(reduce_input)
|
||||
assert scattered is not None
|
||||
torch.testing.assert_close(scattered, expected_scatter)
|
||||
|
||||
|
||||
def benchmark_shape(
|
||||
comm: CustomAllreduce,
|
||||
global_tokens: int,
|
||||
hidden_size: int,
|
||||
graph_repeats: int,
|
||||
warmup_replays: int,
|
||||
samples: int,
|
||||
device_group: dist.ProcessGroup,
|
||||
cpu_group: dist.ProcessGroup,
|
||||
) -> dict[str, float | int]:
|
||||
world_size = dist.get_world_size()
|
||||
rank = dist.get_rank()
|
||||
padded_tokens = (global_tokens + world_size - 1) // world_size * world_size
|
||||
local_tokens = padded_tokens // world_size
|
||||
local = torch.full(
|
||||
(local_tokens, hidden_size),
|
||||
rank + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
reduce_input = torch.full(
|
||||
(padded_tokens, hidden_size),
|
||||
rank + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=local.device,
|
||||
)
|
||||
check_outputs(comm, local, reduce_input, device_group)
|
||||
|
||||
custom_gather_out = torch.empty(
|
||||
(padded_tokens, hidden_size),
|
||||
dtype=local.dtype,
|
||||
device=local.device,
|
||||
)
|
||||
custom_scatter_out = torch.empty_like(local)
|
||||
nccl_gather_out = torch.empty_like(custom_gather_out)
|
||||
nccl_scatter_out = torch.empty_like(local)
|
||||
|
||||
def custom_ag() -> None:
|
||||
ops.mnnvl_lamport_all_gather(
|
||||
comm._ptr,
|
||||
local,
|
||||
custom_gather_out,
|
||||
comm.mnnvl_lamport_ag_local_ptr,
|
||||
comm.mnnvl_lamport_ag_multicast_ptr,
|
||||
comm.mnnvl_lamport_ag_epoch_ptr,
|
||||
comm.mnnvl_buffer_size,
|
||||
)
|
||||
|
||||
def custom_rs() -> None:
|
||||
ops.mnnvl_lamport_reduce_scatter(
|
||||
comm._ptr,
|
||||
reduce_input,
|
||||
custom_scatter_out,
|
||||
comm.mnnvl_lamport_rs_local_ptr,
|
||||
comm.mnnvl_lamport_rs_epoch_ptr,
|
||||
comm.mnnvl_buffer_size,
|
||||
)
|
||||
|
||||
def nccl_ag() -> None:
|
||||
dist.all_gather_into_tensor(nccl_gather_out, local, group=device_group)
|
||||
|
||||
def nccl_rs() -> None:
|
||||
dist.reduce_scatter_tensor(
|
||||
nccl_scatter_out,
|
||||
reduce_input,
|
||||
group=device_group,
|
||||
)
|
||||
|
||||
graphs = {
|
||||
"custom_ag_us": capture_graph(custom_ag, graph_repeats),
|
||||
"nccl_ag_us": capture_graph(nccl_ag, graph_repeats),
|
||||
"custom_rs_us": capture_graph(custom_rs, graph_repeats),
|
||||
"nccl_rs_us": capture_graph(nccl_rs, graph_repeats),
|
||||
}
|
||||
times = {
|
||||
name: max_rank_graph_time(
|
||||
graph,
|
||||
graph_repeats,
|
||||
warmup_replays,
|
||||
samples,
|
||||
device_group,
|
||||
cpu_group,
|
||||
)
|
||||
* 1000
|
||||
for name, graph in graphs.items()
|
||||
}
|
||||
torch.testing.assert_close(custom_gather_out, nccl_gather_out)
|
||||
torch.testing.assert_close(custom_scatter_out, nccl_scatter_out)
|
||||
return {
|
||||
"global_tokens": global_tokens,
|
||||
"padded_tokens": padded_tokens,
|
||||
"local_bytes": local.nbytes,
|
||||
"full_bytes": reduce_input.nbytes,
|
||||
**times,
|
||||
"ag_speedup": times["nccl_ag_us"] / times["custom_ag_us"],
|
||||
"rs_speedup": times["nccl_rs_us"] / times["custom_rs_us"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
dist.init_process_group("nccl")
|
||||
device_group = dist.group.WORLD
|
||||
cpu_group = dist.new_group(backend="gloo")
|
||||
|
||||
comm = CustomAllreduce(
|
||||
group=cpu_group,
|
||||
device=torch.device("cuda", local_rank),
|
||||
)
|
||||
assert not comm.disabled
|
||||
assert comm.world_size == 16
|
||||
assert comm.mnnvl_only
|
||||
assert comm.mnnvl_multicast_ptr
|
||||
|
||||
results = [
|
||||
benchmark_shape(
|
||||
comm,
|
||||
tokens,
|
||||
args.hidden_size,
|
||||
args.graph_repeats,
|
||||
args.warmup_replays,
|
||||
args.samples,
|
||||
device_group,
|
||||
cpu_group,
|
||||
)
|
||||
for tokens in args.tokens
|
||||
]
|
||||
if dist.get_rank() == 0:
|
||||
print(json.dumps(results, indent=2), flush=True)
|
||||
|
||||
comm.close()
|
||||
dist.destroy_process_group(cpu_group)
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user