[9/N][warmup][DSv4] Migrate MHC TileLang kernels (#50178)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:25:23 -07:00
committed by GitHub
co-authored by Codex
parent aed894c190
commit e52be1a62d
9 changed files with 1463 additions and 498 deletions
+126 -273
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
@@ -18,73 +19,56 @@ def _torch_hc_prenorm_gemm(
sqrsum[0].copy_(x_float.square().sum(dim=-1))
def _tilelang_hc_prenorm_gemm(
def _hc_prenorm_gemm_outputs(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
*,
hidden_size: int,
hc_mult: int,
tile_n: int = 12,
n_thr: int = 512,
n_splits: int = 1,
) -> None:
use_tilelang_fallback: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
hc_prenorm_gemm_block_m_tilelang,
hc_prenorm_gemm_tilelang,
compute_num_split,
)
from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
tf32_hc_prenorm_gemm,
)
assert out.shape[0] == n_splits
assert sqrsum.shape[0] == n_splits
assert x.shape[1] == hc_mult * hidden_size
assert x.shape[1] % n_splits == 0
assert (x.shape[1] // n_splits) % n_thr == 0
use_default_config = tile_n == 12 and n_thr == 512
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
hc_prenorm_gemm_block_m_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
2,
)
return
if (
n_splits == 1
and use_default_config
and x.shape[0] < 128
and x.shape[1] % 1024 == 0
):
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
1024,
4,
n_splits,
)
return
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
n_splits,
use_deep_gemm = is_deep_gemm_supported() or not use_tilelang_fallback
num_tokens = x.shape[0]
n_splits = (
compute_num_split(64, x.shape[1], cdiv(num_tokens, 64)) if use_deep_gemm else 1
)
out = torch.empty(
n_splits,
num_tokens,
fn.shape[0],
dtype=torch.float32,
device=x.device,
)
sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=x.device,
)
if use_deep_gemm:
tf32_hc_prenorm_gemm(x, fn, out, sqrsum, n_splits)
else:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
)
_HC_PRENORM_GEMM_TILELANG_KERNEL(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
)
return out, sqrsum
def mhc_pre_delayed_tilelang(
@@ -127,6 +111,7 @@ def mhc_pre_delayed_tilelang(
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
mhc_pre_big_fuse_tilelang,
)
from vllm.model_executor.kernels.mhc.warmup import (
@@ -186,7 +171,14 @@ def mhc_pre_delayed_tilelang(
if use_deep_gemm:
tf32_hc_prenorm_gemm(x, fn, mixes, sqrsum, n_splits)
else:
_tilelang_hc_prenorm_gemm(x, fn, mixes, sqrsum, input_size, 1)
_HC_PRENORM_GEMM_TILELANG_KERNEL(
x,
fn,
mixes,
sqrsum,
input_size,
1,
)
if norm_weight is not None:
assert norm_weight.shape == (hidden_size,)
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
@@ -298,7 +290,8 @@ def mhc_pre_tilelang(
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
n_splits: retained for the shared MHC operator API; the active GEMM
backend selects its split factor internally.
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
torch.bfloat16. When provided, RMSNorm is fused into the
layer_input write path of the big_fuse kernel.
@@ -311,12 +304,8 @@ def mhc_pre_tilelang(
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
@@ -346,17 +335,6 @@ def mhc_pre_tilelang(
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
if use_deep_gemm:
# these numbers are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
else:
n_splits = 1
post_mix = torch.empty(
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
)
@@ -367,78 +345,30 @@ def mhc_pre_tilelang(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
gemm_out_mul = torch.empty(
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_flat.view(num_tokens, hc_mult * hidden_size),
fn,
hidden_size=hidden_size,
hc_mult=hc_mult,
)
gemm_out_sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
tf32_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
# Pre-mix buffers are unused in V4 mode.
post_mix,
post_mix,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
norm_weight,
post_mix,
post_mix,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
@@ -505,11 +435,10 @@ def mhc_pre_broadcast_tilelang(
fn_broadcast: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""First-layer mHC pre for a residual broadcast from ``(T, H)``."""
# n_splits is retained for the shared API; split selection is internal.
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_broadcast_with_norm_tilelang,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.math_utils import cdiv
assert norm_weight is not None, "broadcast mHC pre currently requires fused RMSNorm"
assert residual.dtype == torch.bfloat16
@@ -537,8 +466,6 @@ def mhc_pre_broadcast_tilelang(
residual_flat = residual
num_tokens = residual.shape[0]
n_splits = compute_num_split(64, hidden_size, cdiv(num_tokens, 64))
residual_out = torch.empty(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=residual.device
)
@@ -551,42 +478,31 @@ def mhc_pre_broadcast_tilelang(
layer_input = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
gemm_out_mul = torch.empty(
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
)
gemm_out_sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_flat,
fn_broadcast,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
hidden_size=hidden_size,
hc_mult=hc_mult,
use_tilelang_fallback=False,
)
mhc_pre_big_fuse_broadcast_with_norm_tilelang(
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
residual_out,
post_mix,
comb_mix,
layer_input,
norm_weight,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
residual_out=residual_out,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
return (
residual_out,
@@ -603,11 +519,11 @@ def mhc_post_tilelang(
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
mhc_post_tilelang as _mhc_post_kernel,
_MHC_POST_TILELANG_KERNEL,
)
out = torch.empty_like(residual)
_mhc_post_kernel(
_MHC_POST_TILELANG_KERNEL(
comb_res_mix,
residual,
post_layer_mix.squeeze(-1),
@@ -644,6 +560,9 @@ def mhc_fused_post_pre_tilelang(
RMSNorm'd activation (fused into the kernel); otherwise it is the
raw pre-norm activation as before.
``n_splits`` and ``tile_n`` are retained for the shared MHC operator API.
The TileLang path selects both values internally from the runtime shape.
Returns:
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
post_mix_cur: shape (..., hc_mult, 1)
@@ -652,13 +571,10 @@ def mhc_fused_post_pre_tilelang(
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_fused_tilelang,
mhc_post_tilelang,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert x.dtype == torch.bfloat16
@@ -692,48 +608,14 @@ def mhc_fused_post_pre_tilelang(
if not norm_weight.is_contiguous():
norm_weight = norm_weight.contiguous()
assert n_splits in (1, 2, 4, 8)
assert hidden_size % n_splits == 0
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
x_flat = x.view(num_tokens, hidden_size)
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
use_small_fma = num_tokens <= 16
if use_small_fma:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
else:
if use_deep_gemm:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
)
else:
n_splits = 1
gemm_out_mul = torch.empty(
n_splits,
num_tokens,
hc_mult3,
dtype=torch.float32,
device=residual.device,
)
gemm_out_sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=residual.device,
)
residual_cur = torch.empty_like(residual_flat)
post_mix_cur = torch.empty(
num_tokens,
hc_mult,
@@ -754,23 +636,36 @@ def mhc_fused_post_pre_tilelang(
)
if use_small_fma:
mhc_fused_tilelang(
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
fn.view(hc_mult3, hc_mult, hidden_size),
gemm_out_mul,
gemm_out_sqrsum,
residual_cur,
hc_mult,
hidden_size,
hc_mult3,
tile_n=tile_n,
n_splits=n_splits,
)
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
else:
mhc_post_tilelang(
residual_cur = torch.empty_like(residual_flat)
_MHC_POST_TILELANG_KERNEL(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
@@ -779,52 +674,13 @@ def mhc_fused_post_pre_tilelang(
residual.shape[-2],
residual.shape[-1],
)
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
# Pre-mix buffers are unused in V4 mode.
post_mix_cur,
post_mix_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_cur.view(num_tokens, hc_mult * hidden_size),
fn,
hidden_size=hidden_size,
hc_mult=hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
@@ -833,18 +689,13 @@ def mhc_fused_post_pre_tilelang(
post_mix_cur,
comb_mix_cur,
layer_input_cur,
norm_weight,
post_mix_cur,
post_mix_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
return (
@@ -926,9 +777,11 @@ def hc_head_fused_kernel_tilelang(
)
if num_tokens == 0:
return out
from vllm.model_executor.kernels.mhc.tilelang_kernels import hc_head_fuse_tilelang
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_HEAD_FUSED_TILELANG_KERNEL,
)
hc_head_fuse_tilelang(
_HC_HEAD_FUSED_TILELANG_KERNEL(
hs_flat,
fn,
hc_scale,
@@ -4,10 +4,19 @@
from __future__ import annotations
import math
from dataclasses import dataclass
from functools import cache
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, zip_inputs
from vllm.model_executor.warmup.jit_warmup_tilelang_helper import (
TileLangLaunchSpec,
VllmTileLangJitKernel,
kernel_launcher,
make_tilelang_warmup_tensor,
)
from vllm.platforms import current_platform
from vllm.tilelang_utils import T, tilelang, tilelang_jit
from vllm.utils.math_utils import cdiv
@@ -28,6 +37,10 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
return split_k
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
@tilelang_jit
def mhc_pre_big_fuse_tilelang(
gemm_out_mul,
@@ -987,3 +1000,745 @@ def hc_head_fuse_tilelang(
if ENABLE_PDL:
T.pdl_trigger()
class HcPrenormGemmTileLangKernel(
VllmTileLangJitKernel["HcPrenormGemmTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_out: int
n_thr: int
tile_n: int
n_splits: int
use_block_m: bool
block_m: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
if compile_key.use_block_m:
return hc_prenorm_gemm_block_m_tilelang
return hc_prenorm_gemm_tilelang
def dispatch( # type: ignore[override]
self,
*,
num_tokens: int,
hc_hidden_size: int,
n_thr: int = 512,
tile_n: int = 12,
n_splits: int = 1,
**compile_key_fields: int,
) -> CompileKey:
use_default_config = n_splits == 1 and tile_n == 12 and n_thr == 512
use_block_m = use_default_config and num_tokens >= 1024
use_small_tile = (
use_default_config
and not use_block_m
and num_tokens < 128
and hc_hidden_size % 1024 == 0
)
effective_n_thr = 1024 if use_small_tile else n_thr
effective_tile_n = 4 if use_small_tile else tile_n
return self.CompileKey(
**compile_key_fields,
n_thr=effective_n_thr,
tile_n=effective_tile_n,
n_splits=n_splits,
use_block_m=use_block_m,
block_m=2 if use_block_m else 1,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
n_out: int,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if max_tokens <= 0:
return []
return self._trace_dispatch(self.dispatch)(
num_tokens=WarmupIntRange(
1,
max_tokens + 1,
advance=lambda value: (
128 if value < 128 else 1024 if value < 1024 else max_tokens + 1
),
),
hc_hidden_size=hidden_size * hc_mult,
hidden_size=hidden_size,
hc_mult=hc_mult,
n_out=n_out,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 128
if compile_key.use_block_m:
num_tokens = 1024
elif compile_key.n_thr == 1024:
num_tokens = 1
hc_hidden_size = compile_key.hidden_size * compile_key.hc_mult
return dict(
x=make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hc_hidden_size),
fn=make_tilelang_warmup_tensor(
torch.float32, compile_key.n_out, hc_hidden_size
),
out=make_tilelang_warmup_tensor(
torch.float32,
compile_key.n_splits,
num_tokens,
compile_key.n_out,
),
sqrsum=make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens
),
hidden_size=compile_key.hidden_size,
hc_mult=compile_key.hc_mult,
tile_n=compile_key.tile_n,
n_thr=compile_key.n_thr,
n_splits=compile_key.n_splits,
)
@kernel_launcher
def __call__(
self,
x: Any,
fn: Any,
out: Any,
sqrsum: Any,
hidden_size: int,
hc_mult: int,
tile_n: int = 12,
n_thr: int = 512,
n_splits: int = 1,
) -> TileLangLaunchSpec:
assert out.shape[0] == n_splits
assert sqrsum.shape[0] == n_splits
assert x.shape[1] == hc_mult * hidden_size
assert x.shape[1] % n_splits == 0
assert (x.shape[1] // n_splits) % n_thr == 0
compile_key = self.dispatch(
num_tokens=x.shape[0],
hc_hidden_size=x.shape[1],
hidden_size=hidden_size,
hc_mult=hc_mult,
n_out=fn.shape[0],
n_thr=n_thr,
tile_n=tile_n,
n_splits=n_splits,
)
kernel_arg = (
compile_key.block_m if compile_key.use_block_m else compile_key.n_splits
)
return (compile_key,), (
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
compile_key.n_thr,
compile_key.tile_n,
kernel_arg,
)
class MhcPreBigFuseTileLangKernel(
VllmTileLangJitKernel["MhcPreBigFuseTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_splits: int
use_norm_weight: bool
is_broadcast: bool
rms_eps: float
hc_pre_eps: float
hc_sinkhorn_eps: float
hc_post_mult_value: float
sinkhorn_repeat: int
norm_eps: float
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
if compile_key.is_broadcast:
return mhc_pre_big_fuse_broadcast_with_norm_tilelang
if compile_key.use_norm_weight:
return mhc_pre_big_fuse_with_norm_tilelang
return mhc_pre_big_fuse_tilelang
@staticmethod
def _kernel_args(
compile_key: CompileKey,
*,
gemm_out_mul: Any,
gemm_out_sqrsum: Any,
hc_scale: Any,
hc_base: Any,
residual: Any,
residual_out: Any | None,
post_mix: Any,
comb_mix: Any,
layer_input: Any,
norm_weight: Any | None,
) -> tuple[Any, ...]:
common_args = (
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual,
)
output_args = (post_mix, comb_mix, layer_input)
eps_args = (
compile_key.rms_eps,
compile_key.hc_pre_eps,
compile_key.hc_sinkhorn_eps,
compile_key.hc_post_mult_value,
compile_key.sinkhorn_repeat,
)
middle_args: tuple[Any, ...]
if compile_key.is_broadcast:
assert residual_out is not None
assert norm_weight is not None
middle_args = (residual_out, *output_args, norm_weight)
elif compile_key.use_norm_weight:
assert norm_weight is not None
middle_args = (*output_args, norm_weight, post_mix, post_mix)
else:
middle_args = (*output_args, post_mix, post_mix)
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
return (
*common_args,
*middle_args,
compile_key.hidden_size,
*eps_args,
*norm_eps_args,
compile_key.n_splits,
compile_key.hc_mult,
)
def dispatch( # type: ignore[override]
self,
*,
hidden_size: int,
hc_mult: int,
n_splits: int,
is_broadcast: bool,
use_norm_weight: bool,
use_pre_gemm_splits: bool = False,
pre_gemm_k: int = 0,
sinkhorn_repeat: int,
norm_eps: float,
broadcast_norm_eps: float = 0.0,
num_tokens: int = 1,
use_fused_tilelang: bool = False,
**compile_key_fields: float,
) -> CompileKey:
pre_gemm_n_splits = (
compute_num_split(64, pre_gemm_k, (num_tokens + 63) // 64)
if use_pre_gemm_splits
else n_splits
)
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
actual_use_norm_weight = use_norm_weight or is_broadcast
return self.CompileKey(
**compile_key_fields,
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=actual_n_splits,
use_norm_weight=actual_use_norm_weight,
is_broadcast=is_broadcast,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=actual_norm_eps if actual_use_norm_weight else 0.0,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
use_norm_weight: bool,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
norm_eps: float | tuple[float, ...],
broadcast_norm_eps: float | tuple[float, ...] | None = None,
include_pre_gemm_splits: bool = False,
include_broadcast_splits: bool = False,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if max_tokens <= 0:
return []
warmup_cases = zip_inputs(
dict(
is_broadcast=False,
use_pre_gemm_splits=False,
pre_gemm_k=0,
use_fused_tilelang=False,
),
dict(
is_broadcast=False,
use_pre_gemm_splits=include_pre_gemm_splits,
pre_gemm_k=hc_mult * hidden_size,
use_fused_tilelang=False,
),
dict(
is_broadcast=include_broadcast_splits,
use_pre_gemm_splits=include_broadcast_splits,
pre_gemm_k=hidden_size,
use_fused_tilelang=False,
),
dict(
is_broadcast=False,
use_pre_gemm_splits=False,
pre_gemm_k=0,
use_fused_tilelang=True,
),
)
return self._trace_dispatch(self.dispatch)(
warmup_cases,
num_tokens=WarmupIntRange(
1,
max_tokens + 1,
advance=lambda value: 8 if value < 8 else cdiv(value, 64) * 64 + 1,
),
n_splits=1,
hidden_size=hidden_size,
hc_mult=hc_mult,
use_norm_weight=use_norm_weight,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
broadcast_norm_eps=broadcast_norm_eps
if broadcast_norm_eps is not None
else norm_eps,
_when=lambda *, use_fused_tilelang, num_tokens: (
not use_fused_tilelang or num_tokens <= 16
),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
hc_mult3 = hc_mult * (2 + hc_mult)
num_tokens = 1
gemm_out_mul = make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens, hc_mult3
)
gemm_out_sqrsum = make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens
)
hc_scale = make_tilelang_warmup_tensor(torch.float32, 3)
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult3)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult * hc_mult
)
layer_input = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hidden_size
)
residual_shape = (
(num_tokens, hidden_size)
if compile_key.is_broadcast
else (num_tokens, hc_mult, hidden_size)
)
residual = make_tilelang_warmup_tensor(torch.bfloat16, *residual_shape)
residual_out = (
make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
if compile_key.is_broadcast
else None
)
norm_weight = (
make_tilelang_warmup_tensor(torch.bfloat16, hidden_size)
if compile_key.is_broadcast or compile_key.use_norm_weight
else None
)
return dict(
gemm_out_mul=gemm_out_mul,
gemm_out_sqrsum=gemm_out_sqrsum,
hc_scale=hc_scale,
hc_base=hc_base,
residual=residual,
post_mix=post_mix,
comb_mix=comb_mix,
layer_input=layer_input,
rms_eps=compile_key.rms_eps,
hc_pre_eps=compile_key.hc_pre_eps,
hc_sinkhorn_eps=compile_key.hc_sinkhorn_eps,
hc_post_mult_value=compile_key.hc_post_mult_value,
sinkhorn_repeat=compile_key.sinkhorn_repeat,
residual_out=residual_out,
norm_weight=norm_weight,
norm_eps=compile_key.norm_eps,
)
@kernel_launcher
def __call__(
self,
gemm_out_mul: Any,
gemm_out_sqrsum: Any,
hc_scale: Any,
hc_base: Any,
residual: Any,
post_mix: Any,
comb_mix: Any,
layer_input: Any,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
*,
residual_out: Any | None = None,
norm_weight: Any | None = None,
norm_eps: float = 0.0,
) -> TileLangLaunchSpec:
is_broadcast = residual_out is not None
use_norm_weight = norm_weight is not None
n_splits = gemm_out_mul.shape[0]
if is_broadcast:
assert residual_out is not None
hidden_size = residual.shape[-1]
hc_mult = residual_out.shape[-2]
else:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
compile_key = self.dispatch(
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=n_splits,
is_broadcast=is_broadcast,
use_norm_weight=use_norm_weight,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
broadcast_norm_eps=norm_eps,
)
return (
(compile_key,),
self._kernel_args(
compile_key,
gemm_out_mul=gemm_out_mul,
gemm_out_sqrsum=gemm_out_sqrsum,
hc_scale=hc_scale,
hc_base=hc_base,
residual=residual,
residual_out=residual_out,
post_mix=post_mix,
comb_mix=comb_mix,
layer_input=layer_input,
norm_weight=norm_weight,
),
)
class MhcPostTileLangKernel(VllmTileLangJitKernel["MhcPostTileLangKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
@staticmethod
def kernel() -> Any:
return mhc_post_tilelang
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(
self,
*,
hidden_size: int,
hc_mult: int,
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
hidden_size=hidden_size,
hc_mult=hc_mult,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult, hc_mult
)
residual = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
layer_input = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hidden_size
)
out = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
return dict(
comb_mix=comb_mix,
residual=residual,
post_mix=post_mix,
layer_input=layer_input,
out=out,
hc_mult=hc_mult,
hidden_size=hidden_size,
)
@kernel_launcher
def __call__(
self,
comb_mix: Any,
residual: Any,
post_mix: Any,
layer_input: Any,
out: Any,
hc_mult: int,
hidden_size: int,
) -> TileLangLaunchSpec:
return (), (
comb_mix,
residual,
post_mix,
layer_input,
out,
hc_mult,
hidden_size,
)
class MhcFusedTileLangKernel(
VllmTileLangJitKernel["MhcFusedTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_splits: int
tile_n: int
@staticmethod
def kernel() -> Any:
return mhc_fused_tilelang
def dispatch( # type: ignore[override]
self,
*,
num_tokens: int,
hidden_size: int,
hc_mult: int,
) -> CompileKey:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
return self.CompileKey(
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=n_splits,
tile_n=tile_n,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
return self._trace_dispatch(self.dispatch)(
num_tokens=WarmupIntRange(1, max_tokens + 1),
hidden_size=hidden_size,
hc_mult=hc_mult,
_when=lambda *, num_tokens: num_tokens <= 16,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1 if compile_key.tile_n == 2 else 8
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
hc_mult3 = hc_mult * (2 + hc_mult)
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult, hc_mult
)
residual_in = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
x_in = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
weight_t = make_tilelang_warmup_tensor(
torch.float32, hc_mult3, hc_mult, hidden_size
)
return dict(
comb_mix=comb_mix,
residual_in=residual_in,
post_mix=post_mix,
x_in=x_in,
weight_t=weight_t,
hc_mult=hc_mult,
hidden_size=hidden_size,
hc_mult3=hc_mult3,
)
@kernel_launcher
def __call__(
self,
comb_mix: Any,
residual_in: Any,
post_mix: Any,
x_in: Any,
weight_t: Any,
hc_mult: int,
hidden_size: int,
hc_mult3: int,
) -> TileLangLaunchSpec:
num_tokens = residual_in.shape[0]
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
yp_out = residual_in.new_empty(
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
)
rp_out = residual_in.new_empty((n_splits, num_tokens), dtype=torch.float32)
residual_out = residual_in.new_empty(residual_in.shape)
return (
(),
(
comb_mix,
residual_in,
post_mix,
x_in,
weight_t,
yp_out,
rp_out,
residual_out,
hc_mult,
hidden_size,
hc_mult3,
),
dict(tile_n=tile_n, split_k=n_splits),
(yp_out, rp_out, residual_out),
)
class HcHeadFusedTileLangKernel(
VllmTileLangJitKernel["HcHeadFusedTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
rms_eps: float
hc_eps: float
@staticmethod
def kernel() -> Any:
return hc_head_fuse_tilelang
def dispatch( # type: ignore[override]
self,
*,
hidden_size: int,
hc_mult: int,
rms_eps: float,
hc_eps: float,
) -> CompileKey:
return self.CompileKey(
hidden_size=hidden_size,
hc_mult=hc_mult,
rms_eps=rms_eps,
hc_eps=hc_eps,
)
def get_warmup_keys(
self,
*,
hidden_size: int,
hc_mult: int,
rms_eps: float,
hc_eps: float,
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
hidden_size=hidden_size,
hc_mult=hc_mult,
rms_eps=rms_eps,
hc_eps=hc_eps,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
residual = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
fn = make_tilelang_warmup_tensor(torch.float32, hc_mult, hc_mult * hidden_size)
hc_scale = make_tilelang_warmup_tensor(torch.float32, 1)
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult)
out = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
return dict(
residual=residual,
fn=fn,
hc_scale=hc_scale,
hc_base=hc_base,
out=out,
hidden_size=hidden_size,
rms_eps=compile_key.rms_eps,
hc_eps=compile_key.hc_eps,
hc_mult=hc_mult,
)
@kernel_launcher
def __call__(
self,
residual: Any,
fn: Any,
hc_scale: Any,
hc_base: Any,
out: Any,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> TileLangLaunchSpec:
return (), (
residual,
fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_eps,
hc_eps,
hc_mult,
)
_HC_PRENORM_GEMM_TILELANG_KERNEL = HcPrenormGemmTileLangKernel()
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL = MhcPreBigFuseTileLangKernel()
_MHC_POST_TILELANG_KERNEL = MhcPostTileLangKernel()
_MHC_FUSED_TILELANG_KERNEL = MhcFusedTileLangKernel()
_HC_HEAD_FUSED_TILELANG_KERNEL = HcHeadFusedTileLangKernel()