[9/N][warmup][DSv4] Migrate MHC TileLang kernels (#50178)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
aed894c190
commit
e52be1a62d
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@@ -18,73 +19,56 @@ def _torch_hc_prenorm_gemm(
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
def _tilelang_hc_prenorm_gemm(
|
||||
def _hc_prenorm_gemm_outputs(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
tile_n: int = 12,
|
||||
n_thr: int = 512,
|
||||
n_splits: int = 1,
|
||||
) -> None:
|
||||
use_tilelang_fallback: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
hc_prenorm_gemm_block_m_tilelang,
|
||||
hc_prenorm_gemm_tilelang,
|
||||
compute_num_split,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
tf32_hc_prenorm_gemm,
|
||||
)
|
||||
|
||||
assert out.shape[0] == n_splits
|
||||
assert sqrsum.shape[0] == n_splits
|
||||
assert x.shape[1] == hc_mult * hidden_size
|
||||
assert x.shape[1] % n_splits == 0
|
||||
assert (x.shape[1] // n_splits) % n_thr == 0
|
||||
use_default_config = tile_n == 12 and n_thr == 512
|
||||
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
|
||||
hc_prenorm_gemm_block_m_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if (
|
||||
n_splits == 1
|
||||
and use_default_config
|
||||
and x.shape[0] < 128
|
||||
and x.shape[1] % 1024 == 0
|
||||
):
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
1024,
|
||||
4,
|
||||
n_splits,
|
||||
)
|
||||
return
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
n_splits,
|
||||
use_deep_gemm = is_deep_gemm_supported() or not use_tilelang_fallback
|
||||
num_tokens = x.shape[0]
|
||||
n_splits = (
|
||||
compute_num_split(64, x.shape[1], cdiv(num_tokens, 64)) if use_deep_gemm else 1
|
||||
)
|
||||
out = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
fn.shape[0],
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(x, fn, out, sqrsum, n_splits)
|
||||
else:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
return out, sqrsum
|
||||
|
||||
|
||||
def mhc_pre_delayed_tilelang(
|
||||
@@ -127,6 +111,7 @@ def mhc_pre_delayed_tilelang(
|
||||
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.warmup import (
|
||||
@@ -186,7 +171,14 @@ def mhc_pre_delayed_tilelang(
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(x, fn, mixes, sqrsum, n_splits)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(x, fn, mixes, sqrsum, input_size, 1)
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
x,
|
||||
fn,
|
||||
mixes,
|
||||
sqrsum,
|
||||
input_size,
|
||||
1,
|
||||
)
|
||||
if norm_weight is not None:
|
||||
assert norm_weight.shape == (hidden_size,)
|
||||
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
|
||||
@@ -298,7 +290,8 @@ def mhc_pre_tilelang(
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
n_splits: retained for the shared MHC operator API; the active GEMM
|
||||
backend selects its split factor internally.
|
||||
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
|
||||
torch.bfloat16. When provided, RMSNorm is fused into the
|
||||
layer_input write path of the big_fuse kernel.
|
||||
@@ -311,12 +304,8 @@ def mhc_pre_tilelang(
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
@@ -346,17 +335,6 @@ def mhc_pre_tilelang(
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
if use_deep_gemm:
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
@@ -367,78 +345,30 @@ def mhc_pre_tilelang(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
# Pre-mix buffers are unused in V4 mode.
|
||||
post_mix,
|
||||
post_mix,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
post_mix,
|
||||
post_mix,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
@@ -505,11 +435,10 @@ def mhc_pre_broadcast_tilelang(
|
||||
fn_broadcast: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""First-layer mHC pre for a residual broadcast from ``(T, H)``."""
|
||||
# n_splits is retained for the shared API; split selection is internal.
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_broadcast_with_norm_tilelang,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert norm_weight is not None, "broadcast mHC pre currently requires fused RMSNorm"
|
||||
assert residual.dtype == torch.bfloat16
|
||||
@@ -537,8 +466,6 @@ def mhc_pre_broadcast_tilelang(
|
||||
residual_flat = residual
|
||||
num_tokens = residual.shape[0]
|
||||
|
||||
n_splits = compute_num_split(64, hidden_size, cdiv(num_tokens, 64))
|
||||
|
||||
residual_out = torch.empty(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
@@ -551,42 +478,31 @@ def mhc_pre_broadcast_tilelang(
|
||||
layer_input = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_flat,
|
||||
fn_broadcast,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
use_tilelang_fallback=False,
|
||||
)
|
||||
mhc_pre_big_fuse_broadcast_with_norm_tilelang(
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
residual_out,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
residual_out=residual_out,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
return (
|
||||
residual_out,
|
||||
@@ -603,11 +519,11 @@ def mhc_post_tilelang(
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
mhc_post_tilelang as _mhc_post_kernel,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
out = torch.empty_like(residual)
|
||||
_mhc_post_kernel(
|
||||
_MHC_POST_TILELANG_KERNEL(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.squeeze(-1),
|
||||
@@ -644,6 +560,9 @@ def mhc_fused_post_pre_tilelang(
|
||||
RMSNorm'd activation (fused into the kernel); otherwise it is the
|
||||
raw pre-norm activation as before.
|
||||
|
||||
``n_splits`` and ``tile_n`` are retained for the shared MHC operator API.
|
||||
The TileLang path selects both values internally from the runtime shape.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
@@ -652,13 +571,10 @@ def mhc_fused_post_pre_tilelang(
|
||||
"""
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_fused_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert x.dtype == torch.bfloat16
|
||||
@@ -692,48 +608,14 @@ def mhc_fused_post_pre_tilelang(
|
||||
if not norm_weight.is_contiguous():
|
||||
norm_weight = norm_weight.contiguous()
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
x_flat = x.view(num_tokens, hidden_size)
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
use_small_fma = num_tokens <= 16
|
||||
if use_small_fma:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
if use_deep_gemm:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(
|
||||
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
|
||||
)
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
@@ -754,23 +636,36 @@ def mhc_fused_post_pre_tilelang(
|
||||
)
|
||||
|
||||
if use_small_fma:
|
||||
mhc_fused_tilelang(
|
||||
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
fn.view(hc_mult3, hc_mult, hidden_size),
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
else:
|
||||
mhc_post_tilelang(
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
_MHC_POST_TILELANG_KERNEL(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
@@ -779,52 +674,13 @@ def mhc_fused_post_pre_tilelang(
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
# Pre-mix buffers are unused in V4 mode.
|
||||
post_mix_cur,
|
||||
post_mix_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
@@ -833,18 +689,13 @@ def mhc_fused_post_pre_tilelang(
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
norm_weight,
|
||||
post_mix_cur,
|
||||
post_mix_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
return (
|
||||
@@ -926,9 +777,11 @@ def hc_head_fused_kernel_tilelang(
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import hc_head_fuse_tilelang
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
hc_head_fuse_tilelang(
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
|
||||
@@ -4,10 +4,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, zip_inputs
|
||||
from vllm.model_executor.warmup.jit_warmup_tilelang_helper import (
|
||||
TileLangLaunchSpec,
|
||||
VllmTileLangJitKernel,
|
||||
kernel_launcher,
|
||||
make_tilelang_warmup_tensor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tilelang_utils import T, tilelang, tilelang_jit
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -28,6 +37,10 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
|
||||
return split_k
|
||||
|
||||
|
||||
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
|
||||
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
|
||||
|
||||
|
||||
@tilelang_jit
|
||||
def mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
@@ -987,3 +1000,745 @@ def hc_head_fuse_tilelang(
|
||||
|
||||
if ENABLE_PDL:
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
class HcPrenormGemmTileLangKernel(
|
||||
VllmTileLangJitKernel["HcPrenormGemmTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_out: int
|
||||
n_thr: int
|
||||
tile_n: int
|
||||
n_splits: int
|
||||
use_block_m: bool
|
||||
block_m: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
if compile_key.use_block_m:
|
||||
return hc_prenorm_gemm_block_m_tilelang
|
||||
return hc_prenorm_gemm_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
num_tokens: int,
|
||||
hc_hidden_size: int,
|
||||
n_thr: int = 512,
|
||||
tile_n: int = 12,
|
||||
n_splits: int = 1,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
use_default_config = n_splits == 1 and tile_n == 12 and n_thr == 512
|
||||
use_block_m = use_default_config and num_tokens >= 1024
|
||||
use_small_tile = (
|
||||
use_default_config
|
||||
and not use_block_m
|
||||
and num_tokens < 128
|
||||
and hc_hidden_size % 1024 == 0
|
||||
)
|
||||
effective_n_thr = 1024 if use_small_tile else n_thr
|
||||
effective_tile_n = 4 if use_small_tile else tile_n
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
n_thr=effective_n_thr,
|
||||
tile_n=effective_tile_n,
|
||||
n_splits=n_splits,
|
||||
use_block_m=use_block_m,
|
||||
block_m=2 if use_block_m else 1,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
n_out: int,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
num_tokens=WarmupIntRange(
|
||||
1,
|
||||
max_tokens + 1,
|
||||
advance=lambda value: (
|
||||
128 if value < 128 else 1024 if value < 1024 else max_tokens + 1
|
||||
),
|
||||
),
|
||||
hc_hidden_size=hidden_size * hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_out=n_out,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 128
|
||||
if compile_key.use_block_m:
|
||||
num_tokens = 1024
|
||||
elif compile_key.n_thr == 1024:
|
||||
num_tokens = 1
|
||||
hc_hidden_size = compile_key.hidden_size * compile_key.hc_mult
|
||||
return dict(
|
||||
x=make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hc_hidden_size),
|
||||
fn=make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_out, hc_hidden_size
|
||||
),
|
||||
out=make_tilelang_warmup_tensor(
|
||||
torch.float32,
|
||||
compile_key.n_splits,
|
||||
num_tokens,
|
||||
compile_key.n_out,
|
||||
),
|
||||
sqrsum=make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens
|
||||
),
|
||||
hidden_size=compile_key.hidden_size,
|
||||
hc_mult=compile_key.hc_mult,
|
||||
tile_n=compile_key.tile_n,
|
||||
n_thr=compile_key.n_thr,
|
||||
n_splits=compile_key.n_splits,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
x: Any,
|
||||
fn: Any,
|
||||
out: Any,
|
||||
sqrsum: Any,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
tile_n: int = 12,
|
||||
n_thr: int = 512,
|
||||
n_splits: int = 1,
|
||||
) -> TileLangLaunchSpec:
|
||||
assert out.shape[0] == n_splits
|
||||
assert sqrsum.shape[0] == n_splits
|
||||
assert x.shape[1] == hc_mult * hidden_size
|
||||
assert x.shape[1] % n_splits == 0
|
||||
assert (x.shape[1] // n_splits) % n_thr == 0
|
||||
compile_key = self.dispatch(
|
||||
num_tokens=x.shape[0],
|
||||
hc_hidden_size=x.shape[1],
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_out=fn.shape[0],
|
||||
n_thr=n_thr,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
kernel_arg = (
|
||||
compile_key.block_m if compile_key.use_block_m else compile_key.n_splits
|
||||
)
|
||||
return (compile_key,), (
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
compile_key.n_thr,
|
||||
compile_key.tile_n,
|
||||
kernel_arg,
|
||||
)
|
||||
|
||||
|
||||
class MhcPreBigFuseTileLangKernel(
|
||||
VllmTileLangJitKernel["MhcPreBigFuseTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_splits: int
|
||||
use_norm_weight: bool
|
||||
is_broadcast: bool
|
||||
rms_eps: float
|
||||
hc_pre_eps: float
|
||||
hc_sinkhorn_eps: float
|
||||
hc_post_mult_value: float
|
||||
sinkhorn_repeat: int
|
||||
norm_eps: float
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
if compile_key.is_broadcast:
|
||||
return mhc_pre_big_fuse_broadcast_with_norm_tilelang
|
||||
if compile_key.use_norm_weight:
|
||||
return mhc_pre_big_fuse_with_norm_tilelang
|
||||
return mhc_pre_big_fuse_tilelang
|
||||
|
||||
@staticmethod
|
||||
def _kernel_args(
|
||||
compile_key: CompileKey,
|
||||
*,
|
||||
gemm_out_mul: Any,
|
||||
gemm_out_sqrsum: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
residual: Any,
|
||||
residual_out: Any | None,
|
||||
post_mix: Any,
|
||||
comb_mix: Any,
|
||||
layer_input: Any,
|
||||
norm_weight: Any | None,
|
||||
) -> tuple[Any, ...]:
|
||||
common_args = (
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
)
|
||||
output_args = (post_mix, comb_mix, layer_input)
|
||||
eps_args = (
|
||||
compile_key.rms_eps,
|
||||
compile_key.hc_pre_eps,
|
||||
compile_key.hc_sinkhorn_eps,
|
||||
compile_key.hc_post_mult_value,
|
||||
compile_key.sinkhorn_repeat,
|
||||
)
|
||||
middle_args: tuple[Any, ...]
|
||||
if compile_key.is_broadcast:
|
||||
assert residual_out is not None
|
||||
assert norm_weight is not None
|
||||
middle_args = (residual_out, *output_args, norm_weight)
|
||||
elif compile_key.use_norm_weight:
|
||||
assert norm_weight is not None
|
||||
middle_args = (*output_args, norm_weight, post_mix, post_mix)
|
||||
else:
|
||||
middle_args = (*output_args, post_mix, post_mix)
|
||||
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
|
||||
return (
|
||||
*common_args,
|
||||
*middle_args,
|
||||
compile_key.hidden_size,
|
||||
*eps_args,
|
||||
*norm_eps_args,
|
||||
compile_key.n_splits,
|
||||
compile_key.hc_mult,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
n_splits: int,
|
||||
is_broadcast: bool,
|
||||
use_norm_weight: bool,
|
||||
use_pre_gemm_splits: bool = False,
|
||||
pre_gemm_k: int = 0,
|
||||
sinkhorn_repeat: int,
|
||||
norm_eps: float,
|
||||
broadcast_norm_eps: float = 0.0,
|
||||
num_tokens: int = 1,
|
||||
use_fused_tilelang: bool = False,
|
||||
**compile_key_fields: float,
|
||||
) -> CompileKey:
|
||||
pre_gemm_n_splits = (
|
||||
compute_num_split(64, pre_gemm_k, (num_tokens + 63) // 64)
|
||||
if use_pre_gemm_splits
|
||||
else n_splits
|
||||
)
|
||||
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
|
||||
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
|
||||
actual_use_norm_weight = use_norm_weight or is_broadcast
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=actual_n_splits,
|
||||
use_norm_weight=actual_use_norm_weight,
|
||||
is_broadcast=is_broadcast,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=actual_norm_eps if actual_use_norm_weight else 0.0,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
use_norm_weight: bool,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
norm_eps: float | tuple[float, ...],
|
||||
broadcast_norm_eps: float | tuple[float, ...] | None = None,
|
||||
include_pre_gemm_splits: bool = False,
|
||||
include_broadcast_splits: bool = False,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
|
||||
warmup_cases = zip_inputs(
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=False,
|
||||
pre_gemm_k=0,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=include_pre_gemm_splits,
|
||||
pre_gemm_k=hc_mult * hidden_size,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=include_broadcast_splits,
|
||||
use_pre_gemm_splits=include_broadcast_splits,
|
||||
pre_gemm_k=hidden_size,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=False,
|
||||
pre_gemm_k=0,
|
||||
use_fused_tilelang=True,
|
||||
),
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
warmup_cases,
|
||||
num_tokens=WarmupIntRange(
|
||||
1,
|
||||
max_tokens + 1,
|
||||
advance=lambda value: 8 if value < 8 else cdiv(value, 64) * 64 + 1,
|
||||
),
|
||||
n_splits=1,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
use_norm_weight=use_norm_weight,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
broadcast_norm_eps=broadcast_norm_eps
|
||||
if broadcast_norm_eps is not None
|
||||
else norm_eps,
|
||||
_when=lambda *, use_fused_tilelang, num_tokens: (
|
||||
not use_fused_tilelang or num_tokens <= 16
|
||||
),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
num_tokens = 1
|
||||
|
||||
gemm_out_mul = make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens, hc_mult3
|
||||
)
|
||||
gemm_out_sqrsum = make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens
|
||||
)
|
||||
hc_scale = make_tilelang_warmup_tensor(torch.float32, 3)
|
||||
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult3)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult * hc_mult
|
||||
)
|
||||
layer_input = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hidden_size
|
||||
)
|
||||
residual_shape = (
|
||||
(num_tokens, hidden_size)
|
||||
if compile_key.is_broadcast
|
||||
else (num_tokens, hc_mult, hidden_size)
|
||||
)
|
||||
residual = make_tilelang_warmup_tensor(torch.bfloat16, *residual_shape)
|
||||
residual_out = (
|
||||
make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
if compile_key.is_broadcast
|
||||
else None
|
||||
)
|
||||
norm_weight = (
|
||||
make_tilelang_warmup_tensor(torch.bfloat16, hidden_size)
|
||||
if compile_key.is_broadcast or compile_key.use_norm_weight
|
||||
else None
|
||||
)
|
||||
return dict(
|
||||
gemm_out_mul=gemm_out_mul,
|
||||
gemm_out_sqrsum=gemm_out_sqrsum,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
residual=residual,
|
||||
post_mix=post_mix,
|
||||
comb_mix=comb_mix,
|
||||
layer_input=layer_input,
|
||||
rms_eps=compile_key.rms_eps,
|
||||
hc_pre_eps=compile_key.hc_pre_eps,
|
||||
hc_sinkhorn_eps=compile_key.hc_sinkhorn_eps,
|
||||
hc_post_mult_value=compile_key.hc_post_mult_value,
|
||||
sinkhorn_repeat=compile_key.sinkhorn_repeat,
|
||||
residual_out=residual_out,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=compile_key.norm_eps,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gemm_out_mul: Any,
|
||||
gemm_out_sqrsum: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
residual: Any,
|
||||
post_mix: Any,
|
||||
comb_mix: Any,
|
||||
layer_input: Any,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
*,
|
||||
residual_out: Any | None = None,
|
||||
norm_weight: Any | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> TileLangLaunchSpec:
|
||||
is_broadcast = residual_out is not None
|
||||
use_norm_weight = norm_weight is not None
|
||||
n_splits = gemm_out_mul.shape[0]
|
||||
if is_broadcast:
|
||||
assert residual_out is not None
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual_out.shape[-2]
|
||||
else:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
compile_key = self.dispatch(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=n_splits,
|
||||
is_broadcast=is_broadcast,
|
||||
use_norm_weight=use_norm_weight,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
broadcast_norm_eps=norm_eps,
|
||||
)
|
||||
return (
|
||||
(compile_key,),
|
||||
self._kernel_args(
|
||||
compile_key,
|
||||
gemm_out_mul=gemm_out_mul,
|
||||
gemm_out_sqrsum=gemm_out_sqrsum,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
residual=residual,
|
||||
residual_out=residual_out,
|
||||
post_mix=post_mix,
|
||||
comb_mix=comb_mix,
|
||||
layer_input=layer_input,
|
||||
norm_weight=norm_weight,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MhcPostTileLangKernel(VllmTileLangJitKernel["MhcPostTileLangKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return mhc_post_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult, hc_mult
|
||||
)
|
||||
residual = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
layer_input = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hidden_size
|
||||
)
|
||||
out = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
return dict(
|
||||
comb_mix=comb_mix,
|
||||
residual=residual,
|
||||
post_mix=post_mix,
|
||||
layer_input=layer_input,
|
||||
out=out,
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
comb_mix: Any,
|
||||
residual: Any,
|
||||
post_mix: Any,
|
||||
layer_input: Any,
|
||||
out: Any,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
return (), (
|
||||
comb_mix,
|
||||
residual,
|
||||
post_mix,
|
||||
layer_input,
|
||||
out,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
)
|
||||
|
||||
|
||||
class MhcFusedTileLangKernel(
|
||||
VllmTileLangJitKernel["MhcFusedTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_splits: int
|
||||
tile_n: int
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return mhc_fused_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> CompileKey:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
return self.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=n_splits,
|
||||
tile_n=tile_n,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
num_tokens=WarmupIntRange(1, max_tokens + 1),
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
_when=lambda *, num_tokens: num_tokens <= 16,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1 if compile_key.tile_n == 2 else 8
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult, hc_mult
|
||||
)
|
||||
residual_in = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
x_in = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
|
||||
weight_t = make_tilelang_warmup_tensor(
|
||||
torch.float32, hc_mult3, hc_mult, hidden_size
|
||||
)
|
||||
return dict(
|
||||
comb_mix=comb_mix,
|
||||
residual_in=residual_in,
|
||||
post_mix=post_mix,
|
||||
x_in=x_in,
|
||||
weight_t=weight_t,
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult3=hc_mult3,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
comb_mix: Any,
|
||||
residual_in: Any,
|
||||
post_mix: Any,
|
||||
x_in: Any,
|
||||
weight_t: Any,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_mult3: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
num_tokens = residual_in.shape[0]
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
yp_out = residual_in.new_empty(
|
||||
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
|
||||
)
|
||||
rp_out = residual_in.new_empty((n_splits, num_tokens), dtype=torch.float32)
|
||||
residual_out = residual_in.new_empty(residual_in.shape)
|
||||
return (
|
||||
(),
|
||||
(
|
||||
comb_mix,
|
||||
residual_in,
|
||||
post_mix,
|
||||
x_in,
|
||||
weight_t,
|
||||
yp_out,
|
||||
rp_out,
|
||||
residual_out,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
),
|
||||
dict(tile_n=tile_n, split_k=n_splits),
|
||||
(yp_out, rp_out, residual_out),
|
||||
)
|
||||
|
||||
|
||||
class HcHeadFusedTileLangKernel(
|
||||
VllmTileLangJitKernel["HcHeadFusedTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
rms_eps: float
|
||||
hc_eps: float
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return hc_head_fuse_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
rms_eps=rms_eps,
|
||||
hc_eps=hc_eps,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
rms_eps=rms_eps,
|
||||
hc_eps=hc_eps,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
residual = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
fn = make_tilelang_warmup_tensor(torch.float32, hc_mult, hc_mult * hidden_size)
|
||||
hc_scale = make_tilelang_warmup_tensor(torch.float32, 1)
|
||||
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult)
|
||||
out = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
|
||||
return dict(
|
||||
residual=residual,
|
||||
fn=fn,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
out=out,
|
||||
hidden_size=hidden_size,
|
||||
rms_eps=compile_key.rms_eps,
|
||||
hc_eps=compile_key.hc_eps,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
residual: Any,
|
||||
fn: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
out: Any,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
return (), (
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL = HcPrenormGemmTileLangKernel()
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL = MhcPreBigFuseTileLangKernel()
|
||||
_MHC_POST_TILELANG_KERNEL = MhcPostTileLangKernel()
|
||||
_MHC_FUSED_TILELANG_KERNEL = MhcFusedTileLangKernel()
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL = HcHeadFusedTileLangKernel()
|
||||
|
||||
Reference in New Issue
Block a user