[9/N][warmup][DSv4] Migrate MHC TileLang kernels (#50178)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:25:23 -07:00
committed by GitHub
co-authored by Codex
parent aed894c190
commit e52be1a62d
9 changed files with 1463 additions and 498 deletions
+296
View File
@@ -0,0 +1,296 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Validate JIT dispatch against pre-contract behavior."""
from typing import Any
import pytest
from vllm.platforms import current_platform
if not current_platform.is_cuda_alike():
pytest.skip("NVIDIA dispatch tests require CUDA", allow_module_level=True)
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
HcHeadFusedTileLangKernel,
HcPrenormGemmTileLangKernel,
MhcFusedTileLangKernel,
MhcPostTileLangKernel,
MhcPreBigFuseTileLangKernel,
)
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
@pytest.mark.parametrize(
("kwargs", "expected"),
[
(
dict(
num_tokens=64,
hc_hidden_size=4096,
hidden_size=2048,
hc_mult=2,
n_out=128,
),
(2048, 2, 128, 1024, 4, 1, False, 1),
),
(
dict(
num_tokens=1024,
hc_hidden_size=4096,
hidden_size=2048,
hc_mult=2,
n_out=128,
),
(2048, 2, 128, 512, 12, 1, True, 2),
),
(
dict(
num_tokens=64,
hc_hidden_size=4096,
hidden_size=2048,
hc_mult=2,
n_out=128,
n_thr=256,
tile_n=8,
n_splits=4,
),
(2048, 2, 128, 256, 8, 4, False, 1),
),
],
)
def test_hc_prenorm_gemm_dispatch_matches_legacy_runtime_config(
kwargs: dict[str, Any],
expected: tuple[int, int, int, int, int, int, bool, int],
) -> None:
kernel = HcPrenormGemmTileLangKernel()
assert kernel.dispatch(**kwargs) == kernel.CompileKey(*expected)
@pytest.mark.parametrize(
("is_broadcast", "use_norm_weight", "expected_use_norm", "expected_eps"),
[
(False, False, False, 0.0),
(False, True, True, 1.0e-5),
(True, False, True, 2.0e-5),
],
)
def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
is_broadcast: bool,
use_norm_weight: bool,
expected_use_norm: bool,
expected_eps: float,
) -> None:
kernel = MhcPreBigFuseTileLangKernel()
assert kernel.dispatch(
hidden_size=4096,
hc_mult=4,
n_splits=2,
is_broadcast=is_broadcast,
use_norm_weight=use_norm_weight,
rms_eps=1.0e-6,
hc_pre_eps=2.0e-6,
hc_sinkhorn_eps=3.0e-6,
hc_post_mult_value=0.5,
sinkhorn_repeat=3,
norm_eps=1.0e-5,
broadcast_norm_eps=2.0e-5,
) == kernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=2,
use_norm_weight=expected_use_norm,
is_broadcast=is_broadcast,
rms_eps=1.0e-6,
hc_pre_eps=2.0e-6,
hc_sinkhorn_eps=3.0e-6,
hc_post_mult_value=0.5,
sinkhorn_repeat=3,
norm_eps=expected_eps,
)
@pytest.mark.parametrize(
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
)
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
num_tokens: int,
hidden_size: int,
expected_n_splits: int,
expected_tile_n: int,
) -> None:
kernel = MhcFusedTileLangKernel()
assert kernel.dispatch(
num_tokens=num_tokens,
hidden_size=hidden_size,
hc_mult=4,
) == kernel.CompileKey(
hidden_size=hidden_size,
hc_mult=4,
n_splits=expected_n_splits,
tile_n=expected_tile_n,
)
@pytest.mark.parametrize(
("kernel", "compile_key"),
[
(
HcPrenormGemmTileLangKernel(),
HcPrenormGemmTileLangKernel.CompileKey(
hidden_size=2048,
hc_mult=2,
n_out=128,
n_thr=1024,
tile_n=4,
n_splits=1,
use_block_m=False,
block_m=1,
),
),
(
HcPrenormGemmTileLangKernel(),
HcPrenormGemmTileLangKernel.CompileKey(
hidden_size=2048,
hc_mult=2,
n_out=128,
n_thr=512,
tile_n=12,
n_splits=1,
use_block_m=False,
block_m=1,
),
),
(
HcPrenormGemmTileLangKernel(),
HcPrenormGemmTileLangKernel.CompileKey(
hidden_size=2048,
hc_mult=2,
n_out=128,
n_thr=512,
tile_n=12,
n_splits=1,
use_block_m=True,
block_m=2,
),
),
(
MhcPreBigFuseTileLangKernel(),
MhcPreBigFuseTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=2,
use_norm_weight=True,
is_broadcast=False,
rms_eps=1.0e-6,
hc_pre_eps=2.0e-6,
hc_sinkhorn_eps=3.0e-6,
hc_post_mult_value=0.5,
sinkhorn_repeat=3,
norm_eps=1.0e-5,
),
),
(
MhcPreBigFuseTileLangKernel(),
MhcPreBigFuseTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=1,
use_norm_weight=False,
is_broadcast=False,
rms_eps=1.0e-6,
hc_pre_eps=2.0e-6,
hc_sinkhorn_eps=3.0e-6,
hc_post_mult_value=0.5,
sinkhorn_repeat=3,
norm_eps=0.0,
),
),
(
MhcPreBigFuseTileLangKernel(),
MhcPreBigFuseTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=2,
use_norm_weight=True,
is_broadcast=True,
rms_eps=1.0e-6,
hc_pre_eps=2.0e-6,
hc_sinkhorn_eps=3.0e-6,
hc_post_mult_value=0.5,
sinkhorn_repeat=3,
norm_eps=1.0e-5,
),
),
(
MhcPostTileLangKernel(),
MhcPostTileLangKernel.CompileKey(hidden_size=4096, hc_mult=4),
),
(
MhcFusedTileLangKernel(),
MhcFusedTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=4,
tile_n=3,
),
),
(
MhcFusedTileLangKernel(),
MhcFusedTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=8,
tile_n=2,
),
),
(
MhcFusedTileLangKernel(),
MhcFusedTileLangKernel.CompileKey(
hidden_size=8192,
hc_mult=4,
n_splits=4,
tile_n=2,
),
),
(
HcHeadFusedTileLangKernel(),
HcHeadFusedTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
rms_eps=1.0e-6,
hc_eps=2.0e-6,
),
),
],
)
def test_tilelang_warmup_inputs_reproduce_compile_key(
monkeypatch: pytest.MonkeyPatch,
kernel: Any,
compile_key: Any,
) -> None:
compiled: list[Any] = []
single_kernel = isinstance(
kernel,
(MhcPostTileLangKernel, MhcFusedTileLangKernel, HcHeadFusedTileLangKernel),
)
expected_kernel = kernel.kernel() if single_kernel else kernel.kernel(compile_key)
if single_kernel:
monkeypatch.setattr(
kernel,
"dispatch",
lambda **kwargs: pytest.fail("single-kernel launch must not dispatch"),
)
monkeypatch.setattr(
jit_warmup_tilelang_helper,
"compile_tilelang",
lambda jit_impl, *args, **kwargs: compiled.append(jit_impl),
)
kernel.compile(compile_key)
assert compiled == [expected_kernel]
+11 -2
View File
@@ -10,10 +10,12 @@ import torch.nn.functional as F
import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_tilelang_hc_prenorm_gemm,
_torch_hc_prenorm_gemm,
mhc_pre_delayed_tilelang,
)
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
)
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
from vllm.model_executor.layers.mhc import (
@@ -485,7 +487,14 @@ def test_hc_prenorm_gemm_tilelang(num_tokens, hidden_size):
sqrsum = torch.empty_like(sqrsum_ref)
_torch_hc_prenorm_gemm(x, fn, out_ref, sqrsum_ref)
_tilelang_hc_prenorm_gemm(x, fn, out, sqrsum, hidden_size, hc_mult)
_HC_PRENORM_GEMM_TILELANG_KERNEL(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
)
torch.testing.assert_close(out, out_ref, atol=1e-5, rtol=1e-4)
torch.testing.assert_close(sqrsum, sqrsum_ref, atol=1e-2, rtol=1e-6)
+126 -273
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
@@ -18,73 +19,56 @@ def _torch_hc_prenorm_gemm(
sqrsum[0].copy_(x_float.square().sum(dim=-1))
def _tilelang_hc_prenorm_gemm(
def _hc_prenorm_gemm_outputs(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
*,
hidden_size: int,
hc_mult: int,
tile_n: int = 12,
n_thr: int = 512,
n_splits: int = 1,
) -> None:
use_tilelang_fallback: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
hc_prenorm_gemm_block_m_tilelang,
hc_prenorm_gemm_tilelang,
compute_num_split,
)
from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
tf32_hc_prenorm_gemm,
)
assert out.shape[0] == n_splits
assert sqrsum.shape[0] == n_splits
assert x.shape[1] == hc_mult * hidden_size
assert x.shape[1] % n_splits == 0
assert (x.shape[1] // n_splits) % n_thr == 0
use_default_config = tile_n == 12 and n_thr == 512
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
hc_prenorm_gemm_block_m_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
2,
)
return
if (
n_splits == 1
and use_default_config
and x.shape[0] < 128
and x.shape[1] % 1024 == 0
):
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
1024,
4,
n_splits,
)
return
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
n_splits,
use_deep_gemm = is_deep_gemm_supported() or not use_tilelang_fallback
num_tokens = x.shape[0]
n_splits = (
compute_num_split(64, x.shape[1], cdiv(num_tokens, 64)) if use_deep_gemm else 1
)
out = torch.empty(
n_splits,
num_tokens,
fn.shape[0],
dtype=torch.float32,
device=x.device,
)
sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=x.device,
)
if use_deep_gemm:
tf32_hc_prenorm_gemm(x, fn, out, sqrsum, n_splits)
else:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
)
_HC_PRENORM_GEMM_TILELANG_KERNEL(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
)
return out, sqrsum
def mhc_pre_delayed_tilelang(
@@ -127,6 +111,7 @@ def mhc_pre_delayed_tilelang(
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
mhc_pre_big_fuse_tilelang,
)
from vllm.model_executor.kernels.mhc.warmup import (
@@ -186,7 +171,14 @@ def mhc_pre_delayed_tilelang(
if use_deep_gemm:
tf32_hc_prenorm_gemm(x, fn, mixes, sqrsum, n_splits)
else:
_tilelang_hc_prenorm_gemm(x, fn, mixes, sqrsum, input_size, 1)
_HC_PRENORM_GEMM_TILELANG_KERNEL(
x,
fn,
mixes,
sqrsum,
input_size,
1,
)
if norm_weight is not None:
assert norm_weight.shape == (hidden_size,)
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
@@ -298,7 +290,8 @@ def mhc_pre_tilelang(
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
n_splits: retained for the shared MHC operator API; the active GEMM
backend selects its split factor internally.
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
torch.bfloat16. When provided, RMSNorm is fused into the
layer_input write path of the big_fuse kernel.
@@ -311,12 +304,8 @@ def mhc_pre_tilelang(
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
@@ -346,17 +335,6 @@ def mhc_pre_tilelang(
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
if use_deep_gemm:
# these numbers are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
else:
n_splits = 1
post_mix = torch.empty(
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
)
@@ -367,78 +345,30 @@ def mhc_pre_tilelang(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
gemm_out_mul = torch.empty(
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_flat.view(num_tokens, hc_mult * hidden_size),
fn,
hidden_size=hidden_size,
hc_mult=hc_mult,
)
gemm_out_sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
tf32_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
# Pre-mix buffers are unused in V4 mode.
post_mix,
post_mix,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
norm_weight,
post_mix,
post_mix,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
@@ -505,11 +435,10 @@ def mhc_pre_broadcast_tilelang(
fn_broadcast: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""First-layer mHC pre for a residual broadcast from ``(T, H)``."""
# n_splits is retained for the shared API; split selection is internal.
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_broadcast_with_norm_tilelang,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.math_utils import cdiv
assert norm_weight is not None, "broadcast mHC pre currently requires fused RMSNorm"
assert residual.dtype == torch.bfloat16
@@ -537,8 +466,6 @@ def mhc_pre_broadcast_tilelang(
residual_flat = residual
num_tokens = residual.shape[0]
n_splits = compute_num_split(64, hidden_size, cdiv(num_tokens, 64))
residual_out = torch.empty(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=residual.device
)
@@ -551,42 +478,31 @@ def mhc_pre_broadcast_tilelang(
layer_input = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
gemm_out_mul = torch.empty(
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
)
gemm_out_sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_flat,
fn_broadcast,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
hidden_size=hidden_size,
hc_mult=hc_mult,
use_tilelang_fallback=False,
)
mhc_pre_big_fuse_broadcast_with_norm_tilelang(
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
residual_out,
post_mix,
comb_mix,
layer_input,
norm_weight,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
residual_out=residual_out,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
return (
residual_out,
@@ -603,11 +519,11 @@ def mhc_post_tilelang(
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
mhc_post_tilelang as _mhc_post_kernel,
_MHC_POST_TILELANG_KERNEL,
)
out = torch.empty_like(residual)
_mhc_post_kernel(
_MHC_POST_TILELANG_KERNEL(
comb_res_mix,
residual,
post_layer_mix.squeeze(-1),
@@ -644,6 +560,9 @@ def mhc_fused_post_pre_tilelang(
RMSNorm'd activation (fused into the kernel); otherwise it is the
raw pre-norm activation as before.
``n_splits`` and ``tile_n`` are retained for the shared MHC operator API.
The TileLang path selects both values internally from the runtime shape.
Returns:
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
post_mix_cur: shape (..., hc_mult, 1)
@@ -652,13 +571,10 @@ def mhc_fused_post_pre_tilelang(
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_fused_tilelang,
mhc_post_tilelang,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert x.dtype == torch.bfloat16
@@ -692,48 +608,14 @@ def mhc_fused_post_pre_tilelang(
if not norm_weight.is_contiguous():
norm_weight = norm_weight.contiguous()
assert n_splits in (1, 2, 4, 8)
assert hidden_size % n_splits == 0
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
x_flat = x.view(num_tokens, hidden_size)
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
use_small_fma = num_tokens <= 16
if use_small_fma:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
else:
if use_deep_gemm:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
)
else:
n_splits = 1
gemm_out_mul = torch.empty(
n_splits,
num_tokens,
hc_mult3,
dtype=torch.float32,
device=residual.device,
)
gemm_out_sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=residual.device,
)
residual_cur = torch.empty_like(residual_flat)
post_mix_cur = torch.empty(
num_tokens,
hc_mult,
@@ -754,23 +636,36 @@ def mhc_fused_post_pre_tilelang(
)
if use_small_fma:
mhc_fused_tilelang(
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
fn.view(hc_mult3, hc_mult, hidden_size),
gemm_out_mul,
gemm_out_sqrsum,
residual_cur,
hc_mult,
hidden_size,
hc_mult3,
tile_n=tile_n,
n_splits=n_splits,
)
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
else:
mhc_post_tilelang(
residual_cur = torch.empty_like(residual_flat)
_MHC_POST_TILELANG_KERNEL(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
@@ -779,52 +674,13 @@ def mhc_fused_post_pre_tilelang(
residual.shape[-2],
residual.shape[-1],
)
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
# Pre-mix buffers are unused in V4 mode.
post_mix_cur,
post_mix_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
residual_cur.view(num_tokens, hc_mult * hidden_size),
fn,
hidden_size=hidden_size,
hc_mult=hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
@@ -833,18 +689,13 @@ def mhc_fused_post_pre_tilelang(
post_mix_cur,
comb_mix_cur,
layer_input_cur,
norm_weight,
post_mix_cur,
post_mix_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
return (
@@ -926,9 +777,11 @@ def hc_head_fused_kernel_tilelang(
)
if num_tokens == 0:
return out
from vllm.model_executor.kernels.mhc.tilelang_kernels import hc_head_fuse_tilelang
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_HEAD_FUSED_TILELANG_KERNEL,
)
hc_head_fuse_tilelang(
_HC_HEAD_FUSED_TILELANG_KERNEL(
hs_flat,
fn,
hc_scale,
@@ -4,10 +4,19 @@
from __future__ import annotations
import math
from dataclasses import dataclass
from functools import cache
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, zip_inputs
from vllm.model_executor.warmup.jit_warmup_tilelang_helper import (
TileLangLaunchSpec,
VllmTileLangJitKernel,
kernel_launcher,
make_tilelang_warmup_tensor,
)
from vllm.platforms import current_platform
from vllm.tilelang_utils import T, tilelang, tilelang_jit
from vllm.utils.math_utils import cdiv
@@ -28,6 +37,10 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
return split_k
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
@tilelang_jit
def mhc_pre_big_fuse_tilelang(
gemm_out_mul,
@@ -987,3 +1000,745 @@ def hc_head_fuse_tilelang(
if ENABLE_PDL:
T.pdl_trigger()
class HcPrenormGemmTileLangKernel(
VllmTileLangJitKernel["HcPrenormGemmTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_out: int
n_thr: int
tile_n: int
n_splits: int
use_block_m: bool
block_m: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
if compile_key.use_block_m:
return hc_prenorm_gemm_block_m_tilelang
return hc_prenorm_gemm_tilelang
def dispatch( # type: ignore[override]
self,
*,
num_tokens: int,
hc_hidden_size: int,
n_thr: int = 512,
tile_n: int = 12,
n_splits: int = 1,
**compile_key_fields: int,
) -> CompileKey:
use_default_config = n_splits == 1 and tile_n == 12 and n_thr == 512
use_block_m = use_default_config and num_tokens >= 1024
use_small_tile = (
use_default_config
and not use_block_m
and num_tokens < 128
and hc_hidden_size % 1024 == 0
)
effective_n_thr = 1024 if use_small_tile else n_thr
effective_tile_n = 4 if use_small_tile else tile_n
return self.CompileKey(
**compile_key_fields,
n_thr=effective_n_thr,
tile_n=effective_tile_n,
n_splits=n_splits,
use_block_m=use_block_m,
block_m=2 if use_block_m else 1,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
n_out: int,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if max_tokens <= 0:
return []
return self._trace_dispatch(self.dispatch)(
num_tokens=WarmupIntRange(
1,
max_tokens + 1,
advance=lambda value: (
128 if value < 128 else 1024 if value < 1024 else max_tokens + 1
),
),
hc_hidden_size=hidden_size * hc_mult,
hidden_size=hidden_size,
hc_mult=hc_mult,
n_out=n_out,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 128
if compile_key.use_block_m:
num_tokens = 1024
elif compile_key.n_thr == 1024:
num_tokens = 1
hc_hidden_size = compile_key.hidden_size * compile_key.hc_mult
return dict(
x=make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hc_hidden_size),
fn=make_tilelang_warmup_tensor(
torch.float32, compile_key.n_out, hc_hidden_size
),
out=make_tilelang_warmup_tensor(
torch.float32,
compile_key.n_splits,
num_tokens,
compile_key.n_out,
),
sqrsum=make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens
),
hidden_size=compile_key.hidden_size,
hc_mult=compile_key.hc_mult,
tile_n=compile_key.tile_n,
n_thr=compile_key.n_thr,
n_splits=compile_key.n_splits,
)
@kernel_launcher
def __call__(
self,
x: Any,
fn: Any,
out: Any,
sqrsum: Any,
hidden_size: int,
hc_mult: int,
tile_n: int = 12,
n_thr: int = 512,
n_splits: int = 1,
) -> TileLangLaunchSpec:
assert out.shape[0] == n_splits
assert sqrsum.shape[0] == n_splits
assert x.shape[1] == hc_mult * hidden_size
assert x.shape[1] % n_splits == 0
assert (x.shape[1] // n_splits) % n_thr == 0
compile_key = self.dispatch(
num_tokens=x.shape[0],
hc_hidden_size=x.shape[1],
hidden_size=hidden_size,
hc_mult=hc_mult,
n_out=fn.shape[0],
n_thr=n_thr,
tile_n=tile_n,
n_splits=n_splits,
)
kernel_arg = (
compile_key.block_m if compile_key.use_block_m else compile_key.n_splits
)
return (compile_key,), (
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
compile_key.n_thr,
compile_key.tile_n,
kernel_arg,
)
class MhcPreBigFuseTileLangKernel(
VllmTileLangJitKernel["MhcPreBigFuseTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_splits: int
use_norm_weight: bool
is_broadcast: bool
rms_eps: float
hc_pre_eps: float
hc_sinkhorn_eps: float
hc_post_mult_value: float
sinkhorn_repeat: int
norm_eps: float
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
if compile_key.is_broadcast:
return mhc_pre_big_fuse_broadcast_with_norm_tilelang
if compile_key.use_norm_weight:
return mhc_pre_big_fuse_with_norm_tilelang
return mhc_pre_big_fuse_tilelang
@staticmethod
def _kernel_args(
compile_key: CompileKey,
*,
gemm_out_mul: Any,
gemm_out_sqrsum: Any,
hc_scale: Any,
hc_base: Any,
residual: Any,
residual_out: Any | None,
post_mix: Any,
comb_mix: Any,
layer_input: Any,
norm_weight: Any | None,
) -> tuple[Any, ...]:
common_args = (
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual,
)
output_args = (post_mix, comb_mix, layer_input)
eps_args = (
compile_key.rms_eps,
compile_key.hc_pre_eps,
compile_key.hc_sinkhorn_eps,
compile_key.hc_post_mult_value,
compile_key.sinkhorn_repeat,
)
middle_args: tuple[Any, ...]
if compile_key.is_broadcast:
assert residual_out is not None
assert norm_weight is not None
middle_args = (residual_out, *output_args, norm_weight)
elif compile_key.use_norm_weight:
assert norm_weight is not None
middle_args = (*output_args, norm_weight, post_mix, post_mix)
else:
middle_args = (*output_args, post_mix, post_mix)
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
return (
*common_args,
*middle_args,
compile_key.hidden_size,
*eps_args,
*norm_eps_args,
compile_key.n_splits,
compile_key.hc_mult,
)
def dispatch( # type: ignore[override]
self,
*,
hidden_size: int,
hc_mult: int,
n_splits: int,
is_broadcast: bool,
use_norm_weight: bool,
use_pre_gemm_splits: bool = False,
pre_gemm_k: int = 0,
sinkhorn_repeat: int,
norm_eps: float,
broadcast_norm_eps: float = 0.0,
num_tokens: int = 1,
use_fused_tilelang: bool = False,
**compile_key_fields: float,
) -> CompileKey:
pre_gemm_n_splits = (
compute_num_split(64, pre_gemm_k, (num_tokens + 63) // 64)
if use_pre_gemm_splits
else n_splits
)
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
actual_use_norm_weight = use_norm_weight or is_broadcast
return self.CompileKey(
**compile_key_fields,
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=actual_n_splits,
use_norm_weight=actual_use_norm_weight,
is_broadcast=is_broadcast,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=actual_norm_eps if actual_use_norm_weight else 0.0,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
use_norm_weight: bool,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
norm_eps: float | tuple[float, ...],
broadcast_norm_eps: float | tuple[float, ...] | None = None,
include_pre_gemm_splits: bool = False,
include_broadcast_splits: bool = False,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if max_tokens <= 0:
return []
warmup_cases = zip_inputs(
dict(
is_broadcast=False,
use_pre_gemm_splits=False,
pre_gemm_k=0,
use_fused_tilelang=False,
),
dict(
is_broadcast=False,
use_pre_gemm_splits=include_pre_gemm_splits,
pre_gemm_k=hc_mult * hidden_size,
use_fused_tilelang=False,
),
dict(
is_broadcast=include_broadcast_splits,
use_pre_gemm_splits=include_broadcast_splits,
pre_gemm_k=hidden_size,
use_fused_tilelang=False,
),
dict(
is_broadcast=False,
use_pre_gemm_splits=False,
pre_gemm_k=0,
use_fused_tilelang=True,
),
)
return self._trace_dispatch(self.dispatch)(
warmup_cases,
num_tokens=WarmupIntRange(
1,
max_tokens + 1,
advance=lambda value: 8 if value < 8 else cdiv(value, 64) * 64 + 1,
),
n_splits=1,
hidden_size=hidden_size,
hc_mult=hc_mult,
use_norm_weight=use_norm_weight,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
broadcast_norm_eps=broadcast_norm_eps
if broadcast_norm_eps is not None
else norm_eps,
_when=lambda *, use_fused_tilelang, num_tokens: (
not use_fused_tilelang or num_tokens <= 16
),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
hc_mult3 = hc_mult * (2 + hc_mult)
num_tokens = 1
gemm_out_mul = make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens, hc_mult3
)
gemm_out_sqrsum = make_tilelang_warmup_tensor(
torch.float32, compile_key.n_splits, num_tokens
)
hc_scale = make_tilelang_warmup_tensor(torch.float32, 3)
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult3)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult * hc_mult
)
layer_input = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hidden_size
)
residual_shape = (
(num_tokens, hidden_size)
if compile_key.is_broadcast
else (num_tokens, hc_mult, hidden_size)
)
residual = make_tilelang_warmup_tensor(torch.bfloat16, *residual_shape)
residual_out = (
make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
if compile_key.is_broadcast
else None
)
norm_weight = (
make_tilelang_warmup_tensor(torch.bfloat16, hidden_size)
if compile_key.is_broadcast or compile_key.use_norm_weight
else None
)
return dict(
gemm_out_mul=gemm_out_mul,
gemm_out_sqrsum=gemm_out_sqrsum,
hc_scale=hc_scale,
hc_base=hc_base,
residual=residual,
post_mix=post_mix,
comb_mix=comb_mix,
layer_input=layer_input,
rms_eps=compile_key.rms_eps,
hc_pre_eps=compile_key.hc_pre_eps,
hc_sinkhorn_eps=compile_key.hc_sinkhorn_eps,
hc_post_mult_value=compile_key.hc_post_mult_value,
sinkhorn_repeat=compile_key.sinkhorn_repeat,
residual_out=residual_out,
norm_weight=norm_weight,
norm_eps=compile_key.norm_eps,
)
@kernel_launcher
def __call__(
self,
gemm_out_mul: Any,
gemm_out_sqrsum: Any,
hc_scale: Any,
hc_base: Any,
residual: Any,
post_mix: Any,
comb_mix: Any,
layer_input: Any,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
*,
residual_out: Any | None = None,
norm_weight: Any | None = None,
norm_eps: float = 0.0,
) -> TileLangLaunchSpec:
is_broadcast = residual_out is not None
use_norm_weight = norm_weight is not None
n_splits = gemm_out_mul.shape[0]
if is_broadcast:
assert residual_out is not None
hidden_size = residual.shape[-1]
hc_mult = residual_out.shape[-2]
else:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
compile_key = self.dispatch(
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=n_splits,
is_broadcast=is_broadcast,
use_norm_weight=use_norm_weight,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
broadcast_norm_eps=norm_eps,
)
return (
(compile_key,),
self._kernel_args(
compile_key,
gemm_out_mul=gemm_out_mul,
gemm_out_sqrsum=gemm_out_sqrsum,
hc_scale=hc_scale,
hc_base=hc_base,
residual=residual,
residual_out=residual_out,
post_mix=post_mix,
comb_mix=comb_mix,
layer_input=layer_input,
norm_weight=norm_weight,
),
)
class MhcPostTileLangKernel(VllmTileLangJitKernel["MhcPostTileLangKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
@staticmethod
def kernel() -> Any:
return mhc_post_tilelang
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(
self,
*,
hidden_size: int,
hc_mult: int,
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
hidden_size=hidden_size,
hc_mult=hc_mult,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult, hc_mult
)
residual = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
layer_input = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hidden_size
)
out = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
return dict(
comb_mix=comb_mix,
residual=residual,
post_mix=post_mix,
layer_input=layer_input,
out=out,
hc_mult=hc_mult,
hidden_size=hidden_size,
)
@kernel_launcher
def __call__(
self,
comb_mix: Any,
residual: Any,
post_mix: Any,
layer_input: Any,
out: Any,
hc_mult: int,
hidden_size: int,
) -> TileLangLaunchSpec:
return (), (
comb_mix,
residual,
post_mix,
layer_input,
out,
hc_mult,
hidden_size,
)
class MhcFusedTileLangKernel(
VllmTileLangJitKernel["MhcFusedTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
n_splits: int
tile_n: int
@staticmethod
def kernel() -> Any:
return mhc_fused_tilelang
def dispatch( # type: ignore[override]
self,
*,
num_tokens: int,
hidden_size: int,
hc_mult: int,
) -> CompileKey:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
return self.CompileKey(
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=n_splits,
tile_n=tile_n,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
hidden_size: int,
hc_mult: int,
) -> list[CompileKey]:
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
return self._trace_dispatch(self.dispatch)(
num_tokens=WarmupIntRange(1, max_tokens + 1),
hidden_size=hidden_size,
hc_mult=hc_mult,
_when=lambda *, num_tokens: num_tokens <= 16,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1 if compile_key.tile_n == 2 else 8
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
hc_mult3 = hc_mult * (2 + hc_mult)
comb_mix = make_tilelang_warmup_tensor(
torch.float32, num_tokens, hc_mult, hc_mult
)
residual_in = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
x_in = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
weight_t = make_tilelang_warmup_tensor(
torch.float32, hc_mult3, hc_mult, hidden_size
)
return dict(
comb_mix=comb_mix,
residual_in=residual_in,
post_mix=post_mix,
x_in=x_in,
weight_t=weight_t,
hc_mult=hc_mult,
hidden_size=hidden_size,
hc_mult3=hc_mult3,
)
@kernel_launcher
def __call__(
self,
comb_mix: Any,
residual_in: Any,
post_mix: Any,
x_in: Any,
weight_t: Any,
hc_mult: int,
hidden_size: int,
hc_mult3: int,
) -> TileLangLaunchSpec:
num_tokens = residual_in.shape[0]
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
yp_out = residual_in.new_empty(
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
)
rp_out = residual_in.new_empty((n_splits, num_tokens), dtype=torch.float32)
residual_out = residual_in.new_empty(residual_in.shape)
return (
(),
(
comb_mix,
residual_in,
post_mix,
x_in,
weight_t,
yp_out,
rp_out,
residual_out,
hc_mult,
hidden_size,
hc_mult3,
),
dict(tile_n=tile_n, split_k=n_splits),
(yp_out, rp_out, residual_out),
)
class HcHeadFusedTileLangKernel(
VllmTileLangJitKernel["HcHeadFusedTileLangKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
hc_mult: int
rms_eps: float
hc_eps: float
@staticmethod
def kernel() -> Any:
return hc_head_fuse_tilelang
def dispatch( # type: ignore[override]
self,
*,
hidden_size: int,
hc_mult: int,
rms_eps: float,
hc_eps: float,
) -> CompileKey:
return self.CompileKey(
hidden_size=hidden_size,
hc_mult=hc_mult,
rms_eps=rms_eps,
hc_eps=hc_eps,
)
def get_warmup_keys(
self,
*,
hidden_size: int,
hc_mult: int,
rms_eps: float,
hc_eps: float,
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
hidden_size=hidden_size,
hc_mult=hc_mult,
rms_eps=rms_eps,
hc_eps=hc_eps,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
residual = make_tilelang_warmup_tensor(
torch.bfloat16, num_tokens, hc_mult, hidden_size
)
fn = make_tilelang_warmup_tensor(torch.float32, hc_mult, hc_mult * hidden_size)
hc_scale = make_tilelang_warmup_tensor(torch.float32, 1)
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult)
out = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
return dict(
residual=residual,
fn=fn,
hc_scale=hc_scale,
hc_base=hc_base,
out=out,
hidden_size=hidden_size,
rms_eps=compile_key.rms_eps,
hc_eps=compile_key.hc_eps,
hc_mult=hc_mult,
)
@kernel_launcher
def __call__(
self,
residual: Any,
fn: Any,
hc_scale: Any,
hc_base: Any,
out: Any,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> TileLangLaunchSpec:
return (), (
residual,
fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_eps,
hc_eps,
hc_mult,
)
_HC_PRENORM_GEMM_TILELANG_KERNEL = HcPrenormGemmTileLangKernel()
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL = MhcPreBigFuseTileLangKernel()
_MHC_POST_TILELANG_KERNEL = MhcPostTileLangKernel()
_MHC_FUSED_TILELANG_KERNEL = MhcFusedTileLangKernel()
_HC_HEAD_FUSED_TILELANG_KERNEL = HcHeadFusedTileLangKernel()
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up DeepSeek V4 mHC TileLang kernels before serving requests.
Ported from lucifer1004/vllm-jasl with the two env-var knobs removed
(`VLLM_ENABLE_DEEPSEEK_V4_MHC_WARMUP`, `VLLM_DEEPSEEK_V4_MHC_WARMUP_TOKEN_SIZES`).
Gating is intrinsic: non-DSv4 models and layers without hc_* attributes
return early, so the warmup is a no-op except where it's needed.
"""
import time
from collections.abc import Iterable
import torch
from vllm.logger import init_logger
from vllm.tracing import instrument
logger = init_logger(__name__)
_AUTO_WARMUP_MAX_TOKENS = 16_384
_DEFAULT_TOKEN_SIZE_CANDIDATES = (
1,
2,
4,
8,
16,
32,
64,
128,
256,
512,
1024,
2048,
4096,
8192,
16_384,
)
def _normalize_token_sizes(
token_sizes: Iterable[int],
*,
max_tokens: int,
) -> list[int]:
return sorted({size for size in token_sizes if 1 <= size <= max_tokens})
def _select_mhc_warmup_token_sizes(
*,
max_tokens: int,
cudagraph_capture_sizes: list[int],
) -> list[int]:
if max_tokens <= 0:
return []
max_auto_tokens = min(max_tokens, _AUTO_WARMUP_MAX_TOKENS)
candidates = list(_DEFAULT_TOKEN_SIZE_CANDIDATES)
candidates.extend(cudagraph_capture_sizes)
candidates.append(max_auto_tokens)
return _normalize_token_sizes(candidates, max_tokens=max_auto_tokens)
def _find_first_mhc_layer(model: torch.nn.Module) -> torch.nn.Module | None:
for module in model.modules():
if module.__class__.__name__ != "DeepseekV4DecoderLayer":
continue
if all(
hasattr(module, attr)
for attr in (
"hc_pre",
"hc_post",
"hc_attn_fn",
"hc_attn_scale",
"hc_attn_base",
"hc_ffn_fn",
"hc_ffn_scale",
"hc_ffn_base",
)
):
return module
return None
def _find_deepseek_v4_model(model: torch.nn.Module) -> torch.nn.Module | None:
for module in model.modules():
if module.__class__.__name__ != "DeepseekV4Model":
continue
if all(
hasattr(module, attr)
for attr in ("hc_head_fn", "hc_head_scale", "hc_head_base")
):
return module
return None
def _warmup_layer_mhc(
layer: torch.nn.Module,
token_sizes: list[int],
) -> None:
max_tokens = max(token_sizes)
hidden_size = int(layer.hidden_size)
hc_mult = int(layer.hc_mult)
device = layer.hc_attn_fn.device
residual = torch.zeros(
max_tokens,
hc_mult,
hidden_size,
dtype=torch.bfloat16,
device=device,
)
for size in token_sizes:
residual_slice = residual[:size]
for fn, scale, base in (
(layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base),
(layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base),
):
layer_input, post_mix, comb_mix = layer.hc_pre(
residual_slice,
fn,
scale,
base,
)
layer.hc_post(layer_input, residual_slice, post_mix, comb_mix)
def _warmup_hc_head(
model: torch.nn.Module,
token_sizes: list[int],
) -> None:
# Upstream a8887c208 ("[DSV4] aiter mhc support (ROCm)") refactored
# ``hc_head`` from a free function into the ``HCHeadOp`` CustomOp
# instance attached to the model as ``hc_head_op``. We call through
# that instance so the warmup exercises the same dispatched
# implementation as the inference path.
hc_head_op = getattr(model, "hc_head_op", None)
if hc_head_op is None:
return
max_tokens = max(token_sizes)
hidden_size = int(model.config.hidden_size)
hc_mult = int(model.hc_mult)
device = model.hc_head_fn.device
hidden_states = torch.zeros(
max_tokens,
hc_mult,
hidden_size,
dtype=torch.bfloat16,
device=device,
)
for size in token_sizes:
hc_head_op(
hidden_states[:size],
model.hc_head_fn,
model.hc_head_scale,
model.hc_head_base,
model.rms_norm_eps,
model.hc_eps,
)
@instrument(span_name="DeepSeek V4 mHC warmup")
def deepseek_v4_mhc_warmup(
model: torch.nn.Module,
*,
max_tokens: int,
cudagraph_capture_sizes: list[int] | None = None,
) -> None:
# Cheap model-type gate before walking ``model.modules()``. The class
# walk below is O(num_layers) and shows up in startup time on very
# large checkpoints; bail out for any model that is not DeepSeek V4.
config = getattr(model, "config", None)
model_type = getattr(config, "model_type", None) if config is not None else None
if model_type is not None and model_type not in (
"deepseek_v4",
"deepseek_v41",
):
return
layer = _find_first_mhc_layer(model)
if layer is None:
return
device = layer.hc_attn_fn.device
if device.type != "cuda":
return
deepseek_model = _find_deepseek_v4_model(model)
token_sizes = _select_mhc_warmup_token_sizes(
max_tokens=max_tokens,
cudagraph_capture_sizes=cudagraph_capture_sizes or [],
)
if not token_sizes:
return
started = time.perf_counter()
logger.info(
"Warming up DeepSeek V4 mHC TileLang kernels for token sizes: %s",
token_sizes,
)
with torch.inference_mode():
_warmup_layer_mhc(layer, token_sizes)
if deepseek_model is not None:
_warmup_hc_head(deepseek_model, token_sizes)
torch.accelerator.synchronize()
logger.info(
"DeepSeek V4 mHC TileLang warmup finished in %.2f seconds.",
time.perf_counter() - started,
)
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compile-only helpers for TileLang JIT warmup."""
from __future__ import annotations
import logging
from abc import abstractmethod
from collections.abc import Callable, Iterator, Mapping, MutableMapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from functools import wraps
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
import torch
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
CompileKeyT = TypeVar("CompileKeyT")
# (kernel_args, launch_args), optionally followed by keywords and a result.
TileLangLaunchSpec: TypeAlias = (
tuple[tuple[Any, ...], tuple[Any, ...]]
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None]
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None, Any]
)
@contextmanager
def _quiet_tilelang_warmup_logs() -> Iterator[None]:
loggers = [
logging.getLogger("tilelang.cache.kernel_cache"),
logging.getLogger("tilelang.jit.kernel"),
]
previous_levels = [logger.level for logger in loggers]
try:
for logger in loggers:
logger.setLevel(logging.ERROR)
yield
finally:
for logger, level in zip(loggers, previous_levels):
logger.setLevel(level)
@dataclass(frozen=True)
class TileLangWarmupTensor:
"""Minimal tensor-like object accepted by TileLang compile().
TileLang builds its cache key and TIR from tensor dtype, shape and stride.
This object deliberately has no storage, so compile-only warmup does not
allocate GPU memory and does not launch the kernel.
"""
dtype: torch.dtype
shape: tuple[int, ...] = (1,)
strides: tuple[int, ...] | None = field(default=None)
def stride(self) -> tuple[int, ...]:
if self.strides is not None:
return self.strides
strides: list[int] = []
stride = 1
for size in reversed(self.shape):
strides.append(stride)
stride *= size
return tuple(reversed(strides))
def new_empty(
self,
shape: tuple[int, ...],
*,
dtype: torch.dtype | None = None,
device: Any = None,
) -> TileLangWarmupTensor:
return TileLangWarmupTensor(dtype or self.dtype, shape)
def make_tilelang_warmup_tensor(
dtype: torch.dtype,
*shape: int,
strides: tuple[int, ...] | None = None,
) -> TileLangWarmupTensor:
return TileLangWarmupTensor(dtype=dtype, shape=tuple(shape), strides=strides)
def _tilelang_call_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
call_kwargs = dict(kwargs)
tune_params = call_kwargs.pop("__tune_params", {})
if isinstance(tune_params, dict):
call_kwargs.update(tune_params)
return call_kwargs
def compile_tilelang(jit_impl: Any, *args: Any, **kwargs: Any) -> None:
"""Compile one TileLang specialization and populate its call cache.
TileLang's ``compile()`` materializes the kernel without launching it.
We also store the compiled kernel in ``_kernel_cache`` using the same
parsed key as ``__call__`` so runtime does not report a cache miss for an
already materialized specialization.
"""
with _quiet_tilelang_warmup_logs():
compiled = jit_impl.compile(*args, **kwargs)
func = getattr(jit_impl, "func", None)
parse_args = getattr(func, "parse_args", None)
cache = getattr(jit_impl, "_kernel_cache", None)
if not callable(parse_args) or not isinstance(cache, MutableMapping):
return
key, _ = parse_args(*args, **_tilelang_call_kwargs(kwargs))
cache[key] = compiled
class VllmTileLangJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""TileLang owner whose runtime launch specification is reused for warmup."""
kernel: ClassVar[Any]
_warming_key: CompileKeyT | None = None
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
"""Return runtime-shaped inputs that reproduce one compile key."""
raise NotImplementedError
def compile(self, compile_key: CompileKeyT) -> None:
self._warming_key = compile_key
try:
cast(Callable[..., Any], self)(**self.warmup_inputs(compile_key))
finally:
self._warming_key = None
def launch(
self,
jit_impl: Any,
args: tuple[Any, ...],
kwargs: Mapping[str, Any] | None = None,
) -> Any:
call_kwargs = dict(kwargs or {})
if self._warming_key is not None:
return compile_tilelang(jit_impl, *args, **call_kwargs)
return jit_impl(*args, **call_kwargs)
def kernel_launcher(
call_fn: Callable[..., TileLangLaunchSpec],
) -> Callable[..., Any]:
"""Launch TileLang from declarative kernel and runtime argument tuples."""
@wraps(call_fn)
def wrapper(
self: VllmTileLangJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> Any:
launch_spec = call_fn(self, *args, **kwargs)
kernel_args, launch_args = launch_spec[:2]
if self._warming_key is not None and kernel_args not in (
(),
(self._warming_key,),
):
raise RuntimeError(
"TileLang warmup inputs produced a different compile key: "
f"expected {self._warming_key!r}, got kernel arguments "
f"{kernel_args!r}"
)
launch_kwargs = launch_spec[2] if len(launch_spec) > 2 else None
output = launch_spec[3] if len(launch_spec) > 3 else None
result = self.launch(self.kernel(*kernel_args), launch_args, launch_kwargs)
return output if len(launch_spec) > 3 else result
return wrapper
@@ -17,9 +17,6 @@ from vllm.logger import init_logger
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
@@ -197,15 +194,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
compilation_config = worker.vllm_config.compilation_config
cudagraph_capture_sizes = list(compilation_config.cudagraph_capture_sizes or [])
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
# layer per token; warm them across token sizes first so the first real
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
deepseek_v4_mhc_warmup(
worker.get_model(),
max_tokens=worker.scheduler_config.max_num_batched_tokens,
cudagraph_capture_sizes=cudagraph_capture_sizes,
)
# Run next so input-prep kernels JIT against pristine runner state.
if enable_jit_warmup:
kimi_k3_triton_warmup(worker)
+59
View File
@@ -1210,6 +1210,53 @@ class DeepseekV4DecoderLayer(nn.Module):
requires_grad=False,
)
if vllm_config.kernel_config.enable_jit_warmup:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.deep_gemm import is_deep_gemm_supported
include_pre_gemm_splits = is_deep_gemm_supported()
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.hc_mult,
use_norm_weight=True,
include_pre_gemm_splits=include_pre_gemm_splits,
include_broadcast_splits=(
get_pp_group().is_first_rank and extract_layer_index(prefix) == 0
),
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
norm_eps=(
self.attn_norm.variance_epsilon,
self.ffn_norm.variance_epsilon,
),
broadcast_norm_eps=self.attn_norm.variance_epsilon,
)
if not include_pre_gemm_splits:
_HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.hc_mult,
n_out=self.hc_mult * (2 + self.hc_mult),
)
_MHC_POST_TILELANG_KERNEL.register_warmup(
hidden_size=self.hidden_size,
hc_mult=self.hc_mult,
)
_MHC_FUSED_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.hc_mult,
)
def forward(
self,
x: torch.Tensor,
@@ -1402,6 +1449,18 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
else:
self._mtp_hidden_buffer = None
if vllm_config.kernel_config.enable_jit_warmup and get_pp_group().is_last_rank:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_HEAD_FUSED_TILELANG_KERNEL,
)
_HC_HEAD_FUSED_TILELANG_KERNEL.register_warmup(
hidden_size=config.hidden_size,
hc_mult=self.hc_mult,
rms_eps=self.rms_norm_eps,
hc_eps=self.hc_eps,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
+44
View File
@@ -403,6 +403,50 @@ class Glm5NextDecoderLayer(nn.Module):
self.mhc_post_op = MHCPostOp()
self.mhc_fused_post_pre_op = MHCFusedPostPreOp()
if vllm_config.kernel_config.enable_jit_warmup:
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
)
from vllm.utils.deep_gemm import is_deep_gemm_supported
include_pre_gemm_splits = is_deep_gemm_supported()
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.n,
use_norm_weight=True,
include_pre_gemm_splits=include_pre_gemm_splits,
include_broadcast_splits=False,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.mhc_post_mult_value,
sinkhorn_repeat=self.mhc_sinkhorn_iterations,
norm_eps=(
self.input_layernorm.variance_epsilon,
self.post_attention_layernorm.variance_epsilon,
),
)
if not include_pre_gemm_splits:
_HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.n,
n_out=self.n * (2 + self.n),
)
_MHC_POST_TILELANG_KERNEL.register_warmup(
hidden_size=self.hidden_size,
hc_mult=self.n,
)
_MHC_FUSED_TILELANG_KERNEL.register_warmup(
vllm_config,
hidden_size=self.hidden_size,
hc_mult=self.n,
)
def forward(
self,
positions: torch.Tensor,