From e52be1a62d3879b1202f4f355d3c3472b560c6f2 Mon Sep 17 00:00:00 2001 From: "Roberto L. Castro" <38211239+LopezCastroRoberto@users.noreply.github.com> Date: Sun, 13 Sep 2026 02:25:23 +0200 Subject: [PATCH] [9/N][warmup][DSv4] Migrate MHC TileLang kernels (#50178) Signed-off-by: LopezCastroRoberto Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Codex --- tests/kernels/test_mhc_jit_warmup.py | 296 +++++++ tests/kernels/test_mhc_kernels.py | 13 +- vllm/model_executor/kernels/mhc/tilelang.py | 399 +++------ .../kernels/mhc/tilelang_kernels.py | 755 ++++++++++++++++++ .../warmup/deepseek_v4_mhc_warmup.py | 211 ----- .../warmup/jit_warmup_tilelang_helper.py | 172 ++++ vllm/model_executor/warmup/kernel_warmup.py | 12 - vllm/models/deepseek_v4/nvidia/model.py | 59 ++ vllm/models/glm5next/nvidia/model.py | 44 + 9 files changed, 1463 insertions(+), 498 deletions(-) create mode 100644 tests/kernels/test_mhc_jit_warmup.py delete mode 100644 vllm/model_executor/warmup/deepseek_v4_mhc_warmup.py create mode 100644 vllm/model_executor/warmup/jit_warmup_tilelang_helper.py diff --git a/tests/kernels/test_mhc_jit_warmup.py b/tests/kernels/test_mhc_jit_warmup.py new file mode 100644 index 0000000000..28e746c092 --- /dev/null +++ b/tests/kernels/test_mhc_jit_warmup.py @@ -0,0 +1,296 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Validate JIT dispatch against pre-contract behavior.""" + +from typing import Any + +import pytest + +from vllm.platforms import current_platform + +if not current_platform.is_cuda_alike(): + pytest.skip("NVIDIA dispatch tests require CUDA", allow_module_level=True) + +from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + HcHeadFusedTileLangKernel, + HcPrenormGemmTileLangKernel, + MhcFusedTileLangKernel, + MhcPostTileLangKernel, + MhcPreBigFuseTileLangKernel, +) +from vllm.model_executor.warmup import jit_warmup_tilelang_helper + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ( + dict( + num_tokens=64, + hc_hidden_size=4096, + hidden_size=2048, + hc_mult=2, + n_out=128, + ), + (2048, 2, 128, 1024, 4, 1, False, 1), + ), + ( + dict( + num_tokens=1024, + hc_hidden_size=4096, + hidden_size=2048, + hc_mult=2, + n_out=128, + ), + (2048, 2, 128, 512, 12, 1, True, 2), + ), + ( + dict( + num_tokens=64, + hc_hidden_size=4096, + hidden_size=2048, + hc_mult=2, + n_out=128, + n_thr=256, + tile_n=8, + n_splits=4, + ), + (2048, 2, 128, 256, 8, 4, False, 1), + ), + ], +) +def test_hc_prenorm_gemm_dispatch_matches_legacy_runtime_config( + kwargs: dict[str, Any], + expected: tuple[int, int, int, int, int, int, bool, int], +) -> None: + kernel = HcPrenormGemmTileLangKernel() + + assert kernel.dispatch(**kwargs) == kernel.CompileKey(*expected) + + +@pytest.mark.parametrize( + ("is_broadcast", "use_norm_weight", "expected_use_norm", "expected_eps"), + [ + (False, False, False, 0.0), + (False, True, True, 1.0e-5), + (True, False, True, 2.0e-5), + ], +) +def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config( + is_broadcast: bool, + use_norm_weight: bool, + expected_use_norm: bool, + expected_eps: float, +) -> None: + kernel = MhcPreBigFuseTileLangKernel() + + assert kernel.dispatch( + hidden_size=4096, + hc_mult=4, + n_splits=2, + is_broadcast=is_broadcast, + use_norm_weight=use_norm_weight, + rms_eps=1.0e-6, + hc_pre_eps=2.0e-6, + hc_sinkhorn_eps=3.0e-6, + hc_post_mult_value=0.5, + sinkhorn_repeat=3, + norm_eps=1.0e-5, + broadcast_norm_eps=2.0e-5, + ) == kernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=2, + use_norm_weight=expected_use_norm, + is_broadcast=is_broadcast, + rms_eps=1.0e-6, + hc_pre_eps=2.0e-6, + hc_sinkhorn_eps=3.0e-6, + hc_post_mult_value=0.5, + sinkhorn_repeat=3, + norm_eps=expected_eps, + ) + + +@pytest.mark.parametrize( + ("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"), + [(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)], +) +def test_mhc_fused_dispatch_matches_legacy_runtime_config( + num_tokens: int, + hidden_size: int, + expected_n_splits: int, + expected_tile_n: int, +) -> None: + kernel = MhcFusedTileLangKernel() + + assert kernel.dispatch( + num_tokens=num_tokens, + hidden_size=hidden_size, + hc_mult=4, + ) == kernel.CompileKey( + hidden_size=hidden_size, + hc_mult=4, + n_splits=expected_n_splits, + tile_n=expected_tile_n, + ) + + +@pytest.mark.parametrize( + ("kernel", "compile_key"), + [ + ( + HcPrenormGemmTileLangKernel(), + HcPrenormGemmTileLangKernel.CompileKey( + hidden_size=2048, + hc_mult=2, + n_out=128, + n_thr=1024, + tile_n=4, + n_splits=1, + use_block_m=False, + block_m=1, + ), + ), + ( + HcPrenormGemmTileLangKernel(), + HcPrenormGemmTileLangKernel.CompileKey( + hidden_size=2048, + hc_mult=2, + n_out=128, + n_thr=512, + tile_n=12, + n_splits=1, + use_block_m=False, + block_m=1, + ), + ), + ( + HcPrenormGemmTileLangKernel(), + HcPrenormGemmTileLangKernel.CompileKey( + hidden_size=2048, + hc_mult=2, + n_out=128, + n_thr=512, + tile_n=12, + n_splits=1, + use_block_m=True, + block_m=2, + ), + ), + ( + MhcPreBigFuseTileLangKernel(), + MhcPreBigFuseTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=2, + use_norm_weight=True, + is_broadcast=False, + rms_eps=1.0e-6, + hc_pre_eps=2.0e-6, + hc_sinkhorn_eps=3.0e-6, + hc_post_mult_value=0.5, + sinkhorn_repeat=3, + norm_eps=1.0e-5, + ), + ), + ( + MhcPreBigFuseTileLangKernel(), + MhcPreBigFuseTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=1, + use_norm_weight=False, + is_broadcast=False, + rms_eps=1.0e-6, + hc_pre_eps=2.0e-6, + hc_sinkhorn_eps=3.0e-6, + hc_post_mult_value=0.5, + sinkhorn_repeat=3, + norm_eps=0.0, + ), + ), + ( + MhcPreBigFuseTileLangKernel(), + MhcPreBigFuseTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=2, + use_norm_weight=True, + is_broadcast=True, + rms_eps=1.0e-6, + hc_pre_eps=2.0e-6, + hc_sinkhorn_eps=3.0e-6, + hc_post_mult_value=0.5, + sinkhorn_repeat=3, + norm_eps=1.0e-5, + ), + ), + ( + MhcPostTileLangKernel(), + MhcPostTileLangKernel.CompileKey(hidden_size=4096, hc_mult=4), + ), + ( + MhcFusedTileLangKernel(), + MhcFusedTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=4, + tile_n=3, + ), + ), + ( + MhcFusedTileLangKernel(), + MhcFusedTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + n_splits=8, + tile_n=2, + ), + ), + ( + MhcFusedTileLangKernel(), + MhcFusedTileLangKernel.CompileKey( + hidden_size=8192, + hc_mult=4, + n_splits=4, + tile_n=2, + ), + ), + ( + HcHeadFusedTileLangKernel(), + HcHeadFusedTileLangKernel.CompileKey( + hidden_size=4096, + hc_mult=4, + rms_eps=1.0e-6, + hc_eps=2.0e-6, + ), + ), + ], +) +def test_tilelang_warmup_inputs_reproduce_compile_key( + monkeypatch: pytest.MonkeyPatch, + kernel: Any, + compile_key: Any, +) -> None: + compiled: list[Any] = [] + single_kernel = isinstance( + kernel, + (MhcPostTileLangKernel, MhcFusedTileLangKernel, HcHeadFusedTileLangKernel), + ) + expected_kernel = kernel.kernel() if single_kernel else kernel.kernel(compile_key) + if single_kernel: + monkeypatch.setattr( + kernel, + "dispatch", + lambda **kwargs: pytest.fail("single-kernel launch must not dispatch"), + ) + monkeypatch.setattr( + jit_warmup_tilelang_helper, + "compile_tilelang", + lambda jit_impl, *args, **kwargs: compiled.append(jit_impl), + ) + + kernel.compile(compile_key) + + assert compiled == [expected_kernel] diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index ab5efb6578..7f0515d11a 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -10,10 +10,12 @@ import torch.nn.functional as F import vllm.model_executor.kernels.mhc # noqa: F401 import vllm.model_executor.layers.mhc as mhc_layers from vllm.model_executor.kernels.mhc.tilelang import ( - _tilelang_hc_prenorm_gemm, _torch_hc_prenorm_gemm, mhc_pre_delayed_tilelang, ) +from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, +) from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton from vllm.model_executor.layers.mhc import ( @@ -485,7 +487,14 @@ def test_hc_prenorm_gemm_tilelang(num_tokens, hidden_size): sqrsum = torch.empty_like(sqrsum_ref) _torch_hc_prenorm_gemm(x, fn, out_ref, sqrsum_ref) - _tilelang_hc_prenorm_gemm(x, fn, out, sqrsum, hidden_size, hc_mult) + _HC_PRENORM_GEMM_TILELANG_KERNEL( + x, + fn, + out, + sqrsum, + hidden_size, + hc_mult, + ) torch.testing.assert_close(out, out_ref, atol=1e-5, rtol=1e-4) torch.testing.assert_close(sqrsum, sqrsum_ref, atol=1e-2, rtol=1e-6) diff --git a/vllm/model_executor/kernels/mhc/tilelang.py b/vllm/model_executor/kernels/mhc/tilelang.py index c4fe6dd163..97ca23b541 100644 --- a/vllm/model_executor/kernels/mhc/tilelang.py +++ b/vllm/model_executor/kernels/mhc/tilelang.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch +from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import direct_register_custom_op @@ -18,73 +19,56 @@ def _torch_hc_prenorm_gemm( sqrsum[0].copy_(x_float.square().sum(dim=-1)) -def _tilelang_hc_prenorm_gemm( +def _hc_prenorm_gemm_outputs( x: torch.Tensor, fn: torch.Tensor, - out: torch.Tensor, - sqrsum: torch.Tensor, + *, hidden_size: int, hc_mult: int, - tile_n: int = 12, - n_thr: int = 512, - n_splits: int = 1, -) -> None: + use_tilelang_fallback: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: from vllm.model_executor.kernels.mhc.tilelang_kernels import ( - hc_prenorm_gemm_block_m_tilelang, - hc_prenorm_gemm_tilelang, + compute_num_split, + ) + from vllm.utils.deep_gemm import ( + is_deep_gemm_supported, + tf32_hc_prenorm_gemm, ) - assert out.shape[0] == n_splits - assert sqrsum.shape[0] == n_splits - assert x.shape[1] == hc_mult * hidden_size - assert x.shape[1] % n_splits == 0 - assert (x.shape[1] // n_splits) % n_thr == 0 - use_default_config = tile_n == 12 and n_thr == 512 - if n_splits == 1 and use_default_config and x.shape[0] >= 1024: - hc_prenorm_gemm_block_m_tilelang( - x, - fn, - out, - sqrsum, - hidden_size, - hc_mult, - fn.shape[0], - n_thr, - tile_n, - 2, - ) - return - if ( - n_splits == 1 - and use_default_config - and x.shape[0] < 128 - and x.shape[1] % 1024 == 0 - ): - hc_prenorm_gemm_tilelang( - x, - fn, - out, - sqrsum, - hidden_size, - hc_mult, - fn.shape[0], - 1024, - 4, - n_splits, - ) - return - hc_prenorm_gemm_tilelang( - x, - fn, - out, - sqrsum, - hidden_size, - hc_mult, - fn.shape[0], - n_thr, - tile_n, - n_splits, + use_deep_gemm = is_deep_gemm_supported() or not use_tilelang_fallback + num_tokens = x.shape[0] + n_splits = ( + compute_num_split(64, x.shape[1], cdiv(num_tokens, 64)) if use_deep_gemm else 1 ) + out = torch.empty( + n_splits, + num_tokens, + fn.shape[0], + dtype=torch.float32, + device=x.device, + ) + sqrsum = torch.empty( + n_splits, + num_tokens, + dtype=torch.float32, + device=x.device, + ) + if use_deep_gemm: + tf32_hc_prenorm_gemm(x, fn, out, sqrsum, n_splits) + else: + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, + ) + + _HC_PRENORM_GEMM_TILELANG_KERNEL( + x, + fn, + out, + sqrsum, + hidden_size, + hc_mult, + ) + return out, sqrsum def mhc_pre_delayed_tilelang( @@ -127,6 +111,7 @@ def mhc_pre_delayed_tilelang( (tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult). """ from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, mhc_pre_big_fuse_tilelang, ) from vllm.model_executor.kernels.mhc.warmup import ( @@ -186,7 +171,14 @@ def mhc_pre_delayed_tilelang( if use_deep_gemm: tf32_hc_prenorm_gemm(x, fn, mixes, sqrsum, n_splits) else: - _tilelang_hc_prenorm_gemm(x, fn, mixes, sqrsum, input_size, 1) + _HC_PRENORM_GEMM_TILELANG_KERNEL( + x, + fn, + mixes, + sqrsum, + input_size, + 1, + ) if norm_weight is not None: assert norm_weight.shape == (hidden_size,) assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous() @@ -298,7 +290,8 @@ def mhc_pre_tilelang( hc_sinkhorn_eps: sinkhorn epsilon hc_post_mult_value: post-mix multiplier value sinkhorn_repeat: number of sinkhorn iterations - n_splits: split-k factor; + n_splits: retained for the shared MHC operator API; the active GEMM + backend selects its split factor internally. norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype torch.bfloat16. When provided, RMSNorm is fused into the layer_input write path of the big_fuse kernel. @@ -311,12 +304,8 @@ def mhc_pre_tilelang( layer_input: shape (..., hidden_size), dtype torch.bfloat16 """ from vllm.model_executor.kernels.mhc.tilelang_kernels import ( - compute_num_split, - mhc_pre_big_fuse_tilelang, - mhc_pre_big_fuse_with_norm_tilelang, + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, ) - from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm - from vllm.utils.math_utils import cdiv assert residual.dtype == torch.bfloat16 assert fn.dtype == torch.float32 @@ -346,17 +335,6 @@ def mhc_pre_tilelang( residual_flat = residual.view(-1, hc_mult, hidden_size) num_tokens = residual_flat.shape[0] - from vllm.utils.deep_gemm import is_deep_gemm_supported - - use_deep_gemm = is_deep_gemm_supported() - if use_deep_gemm: - # these numbers are from deepgemm kernel impl - block_k = 64 - block_m = 64 - n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m)) - else: - n_splits = 1 - post_mix = torch.empty( num_tokens, hc_mult, dtype=torch.float32, device=residual.device ) @@ -367,78 +345,30 @@ def mhc_pre_tilelang( num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device ) - gemm_out_mul = torch.empty( - n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device + gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs( + residual_flat.view(num_tokens, hc_mult * hidden_size), + fn, + hidden_size=hidden_size, + hc_mult=hc_mult, ) - gemm_out_sqrsum = torch.empty( - n_splits, num_tokens, dtype=torch.float32, device=residual.device + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_flat, + post_mix, + comb_mix, + layer_input, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + norm_weight=norm_weight, + norm_eps=norm_eps, ) - residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size) - if use_deep_gemm: - tf32_hc_prenorm_gemm( - residual_2d, - fn, - gemm_out_mul, - gemm_out_sqrsum, - n_splits, - ) - else: - _tilelang_hc_prenorm_gemm( - residual_2d, - fn, - gemm_out_mul, - gemm_out_sqrsum, - hidden_size, - hc_mult, - ) - - if norm_weight is None: - mhc_pre_big_fuse_tilelang( - gemm_out_mul, - gemm_out_sqrsum, - hc_scale, - hc_base, - residual_flat, - post_mix, - comb_mix, - layer_input, - # Pre-mix buffers are unused in V4 mode. - post_mix, - post_mix, - hidden_size, - rms_eps, - hc_pre_eps, - hc_sinkhorn_eps, - hc_post_mult_value, - sinkhorn_repeat, - n_splits, - hc_mult, - ) - else: - mhc_pre_big_fuse_with_norm_tilelang( - gemm_out_mul, - gemm_out_sqrsum, - hc_scale, - hc_base, - residual_flat, - post_mix, - comb_mix, - layer_input, - norm_weight, - post_mix, - post_mix, - hidden_size, - rms_eps, - hc_pre_eps, - hc_sinkhorn_eps, - hc_post_mult_value, - sinkhorn_repeat, - norm_eps, - n_splits, - hc_mult, - ) - return ( post_mix.view(*outer_shape, hc_mult, 1), comb_mix.view(*outer_shape, hc_mult, hc_mult), @@ -505,11 +435,10 @@ def mhc_pre_broadcast_tilelang( fn_broadcast: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """First-layer mHC pre for a residual broadcast from ``(T, H)``.""" + # n_splits is retained for the shared API; split selection is internal. from vllm.model_executor.kernels.mhc.tilelang_kernels import ( - compute_num_split, - mhc_pre_big_fuse_broadcast_with_norm_tilelang, + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, ) - from vllm.utils.math_utils import cdiv assert norm_weight is not None, "broadcast mHC pre currently requires fused RMSNorm" assert residual.dtype == torch.bfloat16 @@ -537,8 +466,6 @@ def mhc_pre_broadcast_tilelang( residual_flat = residual num_tokens = residual.shape[0] - n_splits = compute_num_split(64, hidden_size, cdiv(num_tokens, 64)) - residual_out = torch.empty( num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=residual.device ) @@ -551,42 +478,31 @@ def mhc_pre_broadcast_tilelang( layer_input = torch.empty( num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device ) - gemm_out_mul = torch.empty( - n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device - ) - gemm_out_sqrsum = torch.empty( - n_splits, num_tokens, dtype=torch.float32, device=residual.device - ) - from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm - - tf32_hc_prenorm_gemm( + gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs( residual_flat, fn_broadcast, - gemm_out_mul, - gemm_out_sqrsum, - n_splits, + hidden_size=hidden_size, + hc_mult=hc_mult, + use_tilelang_fallback=False, ) - mhc_pre_big_fuse_broadcast_with_norm_tilelang( + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL( gemm_out_mul, gemm_out_sqrsum, hc_scale, hc_base, residual_flat, - residual_out, post_mix, comb_mix, layer_input, - norm_weight, - hidden_size, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, - norm_eps, - n_splits, - hc_mult, + residual_out=residual_out, + norm_weight=norm_weight, + norm_eps=norm_eps, ) return ( residual_out, @@ -603,11 +519,11 @@ def mhc_post_tilelang( comb_res_mix: torch.Tensor, ) -> torch.Tensor: from vllm.model_executor.kernels.mhc.tilelang_kernels import ( - mhc_post_tilelang as _mhc_post_kernel, + _MHC_POST_TILELANG_KERNEL, ) out = torch.empty_like(residual) - _mhc_post_kernel( + _MHC_POST_TILELANG_KERNEL( comb_res_mix, residual, post_layer_mix.squeeze(-1), @@ -644,6 +560,9 @@ def mhc_fused_post_pre_tilelang( RMSNorm'd activation (fused into the kernel); otherwise it is the raw pre-norm activation as before. + ``n_splits`` and ``tile_n`` are retained for the shared MHC operator API. + The TileLang path selects both values internally from the runtime shape. + Returns: residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size) post_mix_cur: shape (..., hc_mult, 1) @@ -652,13 +571,10 @@ def mhc_fused_post_pre_tilelang( """ from vllm.model_executor.kernels.mhc.tilelang_kernels import ( - compute_num_split, - mhc_fused_tilelang, - mhc_post_tilelang, - mhc_pre_big_fuse_tilelang, - mhc_pre_big_fuse_with_norm_tilelang, + _MHC_FUSED_TILELANG_KERNEL, + _MHC_POST_TILELANG_KERNEL, + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, ) - from vllm.utils.math_utils import cdiv assert residual.dtype == torch.bfloat16 assert x.dtype == torch.bfloat16 @@ -692,48 +608,14 @@ def mhc_fused_post_pre_tilelang( if not norm_weight.is_contiguous(): norm_weight = norm_weight.contiguous() - assert n_splits in (1, 2, 4, 8) - assert hidden_size % n_splits == 0 - residual_flat = residual.view(-1, hc_mult, hidden_size) num_tokens = residual_flat.shape[0] x_flat = x.view(num_tokens, hidden_size) post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult) comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult) - from vllm.utils.deep_gemm import is_deep_gemm_supported - - use_deep_gemm = is_deep_gemm_supported() use_small_fma = num_tokens <= 16 - if use_small_fma: - # TODO(gnovack): investigate autotuning these heuristics - tile_n = 2 if num_tokens < 8 else 3 - n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4 - else: - if use_deep_gemm: - # these number are from deepgemm kernel impl - block_k = 64 - block_m = 64 - n_splits = compute_num_split( - block_k, hc_hidden_size, cdiv(num_tokens, block_m) - ) - else: - n_splits = 1 - gemm_out_mul = torch.empty( - n_splits, - num_tokens, - hc_mult3, - dtype=torch.float32, - device=residual.device, - ) - gemm_out_sqrsum = torch.empty( - n_splits, - num_tokens, - dtype=torch.float32, - device=residual.device, - ) - residual_cur = torch.empty_like(residual_flat) post_mix_cur = torch.empty( num_tokens, hc_mult, @@ -754,23 +636,36 @@ def mhc_fused_post_pre_tilelang( ) if use_small_fma: - mhc_fused_tilelang( + gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL( comb_res_mix_flat, residual_flat, post_layer_mix_flat, x_flat, fn.view(hc_mult3, hc_mult, hidden_size), - gemm_out_mul, - gemm_out_sqrsum, - residual_cur, hc_mult, hidden_size, hc_mult3, - tile_n=tile_n, - n_splits=n_splits, + ) + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_cur, + post_mix_cur, + comb_mix_cur, + layer_input_cur, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + norm_weight=norm_weight, + norm_eps=norm_eps, ) else: - mhc_post_tilelang( + residual_cur = torch.empty_like(residual_flat) + _MHC_POST_TILELANG_KERNEL( comb_res_mix_flat, residual_flat, post_layer_mix_flat, @@ -779,52 +674,13 @@ def mhc_fused_post_pre_tilelang( residual.shape[-2], residual.shape[-1], ) - - residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size) - if use_deep_gemm: - from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm - - tf32_hc_prenorm_gemm( - residual_cur_2d, - fn, - gemm_out_mul, - gemm_out_sqrsum, - n_splits, - ) - else: - _tilelang_hc_prenorm_gemm( - residual_cur_2d, - fn, - gemm_out_mul, - gemm_out_sqrsum, - hidden_size, - hc_mult, - ) - - if norm_weight is None: - mhc_pre_big_fuse_tilelang( - gemm_out_mul, - gemm_out_sqrsum, - hc_scale, - hc_base, - residual_cur, - post_mix_cur, - comb_mix_cur, - layer_input_cur, - # Pre-mix buffers are unused in V4 mode. - post_mix_cur, - post_mix_cur, - hidden_size, - rms_eps, - hc_pre_eps, - hc_sinkhorn_eps, - hc_post_mult_value, - sinkhorn_repeat, - n_splits, - hc_mult, + gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs( + residual_cur.view(num_tokens, hc_mult * hidden_size), + fn, + hidden_size=hidden_size, + hc_mult=hc_mult, ) - else: - mhc_pre_big_fuse_with_norm_tilelang( + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL( gemm_out_mul, gemm_out_sqrsum, hc_scale, @@ -833,18 +689,13 @@ def mhc_fused_post_pre_tilelang( post_mix_cur, comb_mix_cur, layer_input_cur, - norm_weight, - post_mix_cur, - post_mix_cur, - hidden_size, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, - norm_eps, - n_splits, - hc_mult, + norm_weight=norm_weight, + norm_eps=norm_eps, ) return ( @@ -926,9 +777,11 @@ def hc_head_fused_kernel_tilelang( ) if num_tokens == 0: return out - from vllm.model_executor.kernels.mhc.tilelang_kernels import hc_head_fuse_tilelang + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_HEAD_FUSED_TILELANG_KERNEL, + ) - hc_head_fuse_tilelang( + _HC_HEAD_FUSED_TILELANG_KERNEL( hs_flat, fn, hc_scale, diff --git a/vllm/model_executor/kernels/mhc/tilelang_kernels.py b/vllm/model_executor/kernels/mhc/tilelang_kernels.py index 1c23d152b9..b4d3190c0e 100644 --- a/vllm/model_executor/kernels/mhc/tilelang_kernels.py +++ b/vllm/model_executor/kernels/mhc/tilelang_kernels.py @@ -4,10 +4,19 @@ from __future__ import annotations import math +from dataclasses import dataclass from functools import cache +from typing import Any import torch +from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, zip_inputs +from vllm.model_executor.warmup.jit_warmup_tilelang_helper import ( + TileLangLaunchSpec, + VllmTileLangJitKernel, + kernel_launcher, + make_tilelang_warmup_tensor, +) from vllm.platforms import current_platform from vllm.tilelang_utils import T, tilelang, tilelang_jit from vllm.utils.math_utils import cdiv @@ -28,6 +37,10 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int: return split_k +def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int: + return 8 if num_tokens < 8 and hidden_size <= 4096 else 4 + + @tilelang_jit def mhc_pre_big_fuse_tilelang( gemm_out_mul, @@ -987,3 +1000,745 @@ def hc_head_fuse_tilelang( if ENABLE_PDL: T.pdl_trigger() + + +class HcPrenormGemmTileLangKernel( + VllmTileLangJitKernel["HcPrenormGemmTileLangKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + hidden_size: int + hc_mult: int + n_out: int + n_thr: int + tile_n: int + n_splits: int + use_block_m: bool + block_m: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + if compile_key.use_block_m: + return hc_prenorm_gemm_block_m_tilelang + return hc_prenorm_gemm_tilelang + + def dispatch( # type: ignore[override] + self, + *, + num_tokens: int, + hc_hidden_size: int, + n_thr: int = 512, + tile_n: int = 12, + n_splits: int = 1, + **compile_key_fields: int, + ) -> CompileKey: + use_default_config = n_splits == 1 and tile_n == 12 and n_thr == 512 + use_block_m = use_default_config and num_tokens >= 1024 + use_small_tile = ( + use_default_config + and not use_block_m + and num_tokens < 128 + and hc_hidden_size % 1024 == 0 + ) + effective_n_thr = 1024 if use_small_tile else n_thr + effective_tile_n = 4 if use_small_tile else tile_n + return self.CompileKey( + **compile_key_fields, + n_thr=effective_n_thr, + tile_n=effective_tile_n, + n_splits=n_splits, + use_block_m=use_block_m, + block_m=2 if use_block_m else 1, + ) + + def get_warmup_keys( + self, + vllm_config: Any, + *, + hidden_size: int, + hc_mult: int, + n_out: int, + ) -> list[CompileKey]: + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + if max_tokens <= 0: + return [] + return self._trace_dispatch(self.dispatch)( + num_tokens=WarmupIntRange( + 1, + max_tokens + 1, + advance=lambda value: ( + 128 if value < 128 else 1024 if value < 1024 else max_tokens + 1 + ), + ), + hc_hidden_size=hidden_size * hc_mult, + hidden_size=hidden_size, + hc_mult=hc_mult, + n_out=n_out, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + num_tokens = 128 + if compile_key.use_block_m: + num_tokens = 1024 + elif compile_key.n_thr == 1024: + num_tokens = 1 + hc_hidden_size = compile_key.hidden_size * compile_key.hc_mult + return dict( + x=make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hc_hidden_size), + fn=make_tilelang_warmup_tensor( + torch.float32, compile_key.n_out, hc_hidden_size + ), + out=make_tilelang_warmup_tensor( + torch.float32, + compile_key.n_splits, + num_tokens, + compile_key.n_out, + ), + sqrsum=make_tilelang_warmup_tensor( + torch.float32, compile_key.n_splits, num_tokens + ), + hidden_size=compile_key.hidden_size, + hc_mult=compile_key.hc_mult, + tile_n=compile_key.tile_n, + n_thr=compile_key.n_thr, + n_splits=compile_key.n_splits, + ) + + @kernel_launcher + def __call__( + self, + x: Any, + fn: Any, + out: Any, + sqrsum: Any, + hidden_size: int, + hc_mult: int, + tile_n: int = 12, + n_thr: int = 512, + n_splits: int = 1, + ) -> TileLangLaunchSpec: + assert out.shape[0] == n_splits + assert sqrsum.shape[0] == n_splits + assert x.shape[1] == hc_mult * hidden_size + assert x.shape[1] % n_splits == 0 + assert (x.shape[1] // n_splits) % n_thr == 0 + compile_key = self.dispatch( + num_tokens=x.shape[0], + hc_hidden_size=x.shape[1], + hidden_size=hidden_size, + hc_mult=hc_mult, + n_out=fn.shape[0], + n_thr=n_thr, + tile_n=tile_n, + n_splits=n_splits, + ) + kernel_arg = ( + compile_key.block_m if compile_key.use_block_m else compile_key.n_splits + ) + return (compile_key,), ( + x, + fn, + out, + sqrsum, + hidden_size, + hc_mult, + fn.shape[0], + compile_key.n_thr, + compile_key.tile_n, + kernel_arg, + ) + + +class MhcPreBigFuseTileLangKernel( + VllmTileLangJitKernel["MhcPreBigFuseTileLangKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + hidden_size: int + hc_mult: int + n_splits: int + use_norm_weight: bool + is_broadcast: bool + rms_eps: float + hc_pre_eps: float + hc_sinkhorn_eps: float + hc_post_mult_value: float + sinkhorn_repeat: int + norm_eps: float + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + if compile_key.is_broadcast: + return mhc_pre_big_fuse_broadcast_with_norm_tilelang + if compile_key.use_norm_weight: + return mhc_pre_big_fuse_with_norm_tilelang + return mhc_pre_big_fuse_tilelang + + @staticmethod + def _kernel_args( + compile_key: CompileKey, + *, + gemm_out_mul: Any, + gemm_out_sqrsum: Any, + hc_scale: Any, + hc_base: Any, + residual: Any, + residual_out: Any | None, + post_mix: Any, + comb_mix: Any, + layer_input: Any, + norm_weight: Any | None, + ) -> tuple[Any, ...]: + common_args = ( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual, + ) + output_args = (post_mix, comb_mix, layer_input) + eps_args = ( + compile_key.rms_eps, + compile_key.hc_pre_eps, + compile_key.hc_sinkhorn_eps, + compile_key.hc_post_mult_value, + compile_key.sinkhorn_repeat, + ) + middle_args: tuple[Any, ...] + if compile_key.is_broadcast: + assert residual_out is not None + assert norm_weight is not None + middle_args = (residual_out, *output_args, norm_weight) + elif compile_key.use_norm_weight: + assert norm_weight is not None + middle_args = (*output_args, norm_weight, post_mix, post_mix) + else: + middle_args = (*output_args, post_mix, post_mix) + norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else () + return ( + *common_args, + *middle_args, + compile_key.hidden_size, + *eps_args, + *norm_eps_args, + compile_key.n_splits, + compile_key.hc_mult, + ) + + def dispatch( # type: ignore[override] + self, + *, + hidden_size: int, + hc_mult: int, + n_splits: int, + is_broadcast: bool, + use_norm_weight: bool, + use_pre_gemm_splits: bool = False, + pre_gemm_k: int = 0, + sinkhorn_repeat: int, + norm_eps: float, + broadcast_norm_eps: float = 0.0, + num_tokens: int = 1, + use_fused_tilelang: bool = False, + **compile_key_fields: float, + ) -> CompileKey: + pre_gemm_n_splits = ( + compute_num_split(64, pre_gemm_k, (num_tokens + 63) // 64) + if use_pre_gemm_splits + else n_splits + ) + fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) + actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits + actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps + actual_use_norm_weight = use_norm_weight or is_broadcast + return self.CompileKey( + **compile_key_fields, + hidden_size=hidden_size, + hc_mult=hc_mult, + n_splits=actual_n_splits, + use_norm_weight=actual_use_norm_weight, + is_broadcast=is_broadcast, + sinkhorn_repeat=sinkhorn_repeat, + norm_eps=actual_norm_eps if actual_use_norm_weight else 0.0, + ) + + def get_warmup_keys( + self, + vllm_config: Any, + *, + hidden_size: int, + hc_mult: int, + use_norm_weight: bool, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + norm_eps: float | tuple[float, ...], + broadcast_norm_eps: float | tuple[float, ...] | None = None, + include_pre_gemm_splits: bool = False, + include_broadcast_splits: bool = False, + ) -> list[CompileKey]: + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + if max_tokens <= 0: + return [] + + warmup_cases = zip_inputs( + dict( + is_broadcast=False, + use_pre_gemm_splits=False, + pre_gemm_k=0, + use_fused_tilelang=False, + ), + dict( + is_broadcast=False, + use_pre_gemm_splits=include_pre_gemm_splits, + pre_gemm_k=hc_mult * hidden_size, + use_fused_tilelang=False, + ), + dict( + is_broadcast=include_broadcast_splits, + use_pre_gemm_splits=include_broadcast_splits, + pre_gemm_k=hidden_size, + use_fused_tilelang=False, + ), + dict( + is_broadcast=False, + use_pre_gemm_splits=False, + pre_gemm_k=0, + use_fused_tilelang=True, + ), + ) + return self._trace_dispatch(self.dispatch)( + warmup_cases, + num_tokens=WarmupIntRange( + 1, + max_tokens + 1, + advance=lambda value: 8 if value < 8 else cdiv(value, 64) * 64 + 1, + ), + n_splits=1, + hidden_size=hidden_size, + hc_mult=hc_mult, + use_norm_weight=use_norm_weight, + rms_eps=rms_eps, + hc_pre_eps=hc_pre_eps, + hc_sinkhorn_eps=hc_sinkhorn_eps, + hc_post_mult_value=hc_post_mult_value, + sinkhorn_repeat=sinkhorn_repeat, + norm_eps=norm_eps, + broadcast_norm_eps=broadcast_norm_eps + if broadcast_norm_eps is not None + else norm_eps, + _when=lambda *, use_fused_tilelang, num_tokens: ( + not use_fused_tilelang or num_tokens <= 16 + ), + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + hidden_size = compile_key.hidden_size + hc_mult = compile_key.hc_mult + hc_mult3 = hc_mult * (2 + hc_mult) + num_tokens = 1 + + gemm_out_mul = make_tilelang_warmup_tensor( + torch.float32, compile_key.n_splits, num_tokens, hc_mult3 + ) + gemm_out_sqrsum = make_tilelang_warmup_tensor( + torch.float32, compile_key.n_splits, num_tokens + ) + hc_scale = make_tilelang_warmup_tensor(torch.float32, 3) + hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult3) + post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult) + comb_mix = make_tilelang_warmup_tensor( + torch.float32, num_tokens, hc_mult * hc_mult + ) + layer_input = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hidden_size + ) + residual_shape = ( + (num_tokens, hidden_size) + if compile_key.is_broadcast + else (num_tokens, hc_mult, hidden_size) + ) + residual = make_tilelang_warmup_tensor(torch.bfloat16, *residual_shape) + residual_out = ( + make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hc_mult, hidden_size + ) + if compile_key.is_broadcast + else None + ) + norm_weight = ( + make_tilelang_warmup_tensor(torch.bfloat16, hidden_size) + if compile_key.is_broadcast or compile_key.use_norm_weight + else None + ) + return dict( + gemm_out_mul=gemm_out_mul, + gemm_out_sqrsum=gemm_out_sqrsum, + hc_scale=hc_scale, + hc_base=hc_base, + residual=residual, + post_mix=post_mix, + comb_mix=comb_mix, + layer_input=layer_input, + rms_eps=compile_key.rms_eps, + hc_pre_eps=compile_key.hc_pre_eps, + hc_sinkhorn_eps=compile_key.hc_sinkhorn_eps, + hc_post_mult_value=compile_key.hc_post_mult_value, + sinkhorn_repeat=compile_key.sinkhorn_repeat, + residual_out=residual_out, + norm_weight=norm_weight, + norm_eps=compile_key.norm_eps, + ) + + @kernel_launcher + def __call__( + self, + gemm_out_mul: Any, + gemm_out_sqrsum: Any, + hc_scale: Any, + hc_base: Any, + residual: Any, + post_mix: Any, + comb_mix: Any, + layer_input: Any, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + *, + residual_out: Any | None = None, + norm_weight: Any | None = None, + norm_eps: float = 0.0, + ) -> TileLangLaunchSpec: + is_broadcast = residual_out is not None + use_norm_weight = norm_weight is not None + n_splits = gemm_out_mul.shape[0] + if is_broadcast: + assert residual_out is not None + hidden_size = residual.shape[-1] + hc_mult = residual_out.shape[-2] + else: + hc_mult = residual.shape[-2] + hidden_size = residual.shape[-1] + compile_key = self.dispatch( + hidden_size=hidden_size, + hc_mult=hc_mult, + n_splits=n_splits, + is_broadcast=is_broadcast, + use_norm_weight=use_norm_weight, + rms_eps=rms_eps, + hc_pre_eps=hc_pre_eps, + hc_sinkhorn_eps=hc_sinkhorn_eps, + hc_post_mult_value=hc_post_mult_value, + sinkhorn_repeat=sinkhorn_repeat, + norm_eps=norm_eps, + broadcast_norm_eps=norm_eps, + ) + return ( + (compile_key,), + self._kernel_args( + compile_key, + gemm_out_mul=gemm_out_mul, + gemm_out_sqrsum=gemm_out_sqrsum, + hc_scale=hc_scale, + hc_base=hc_base, + residual=residual, + residual_out=residual_out, + post_mix=post_mix, + comb_mix=comb_mix, + layer_input=layer_input, + norm_weight=norm_weight, + ), + ) + + +class MhcPostTileLangKernel(VllmTileLangJitKernel["MhcPostTileLangKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + hidden_size: int + hc_mult: int + + @staticmethod + def kernel() -> Any: + return mhc_post_tilelang + + def dispatch( # type: ignore[override] + self, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey(**compile_key_fields) + + def get_warmup_keys( + self, + *, + hidden_size: int, + hc_mult: int, + ) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)( + hidden_size=hidden_size, + hc_mult=hc_mult, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + num_tokens = 1 + hidden_size = compile_key.hidden_size + hc_mult = compile_key.hc_mult + comb_mix = make_tilelang_warmup_tensor( + torch.float32, num_tokens, hc_mult, hc_mult + ) + residual = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hc_mult, hidden_size + ) + post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult) + layer_input = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hidden_size + ) + out = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hc_mult, hidden_size + ) + return dict( + comb_mix=comb_mix, + residual=residual, + post_mix=post_mix, + layer_input=layer_input, + out=out, + hc_mult=hc_mult, + hidden_size=hidden_size, + ) + + @kernel_launcher + def __call__( + self, + comb_mix: Any, + residual: Any, + post_mix: Any, + layer_input: Any, + out: Any, + hc_mult: int, + hidden_size: int, + ) -> TileLangLaunchSpec: + return (), ( + comb_mix, + residual, + post_mix, + layer_input, + out, + hc_mult, + hidden_size, + ) + + +class MhcFusedTileLangKernel( + VllmTileLangJitKernel["MhcFusedTileLangKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + hidden_size: int + hc_mult: int + n_splits: int + tile_n: int + + @staticmethod + def kernel() -> Any: + return mhc_fused_tilelang + + def dispatch( # type: ignore[override] + self, + *, + num_tokens: int, + hidden_size: int, + hc_mult: int, + ) -> CompileKey: + # TODO(gnovack): investigate autotuning these heuristics + tile_n = 2 if num_tokens < 8 else 3 + n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) + return self.CompileKey( + hidden_size=hidden_size, + hc_mult=hc_mult, + n_splits=n_splits, + tile_n=tile_n, + ) + + def get_warmup_keys( + self, + vllm_config: Any, + *, + hidden_size: int, + hc_mult: int, + ) -> list[CompileKey]: + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + return self._trace_dispatch(self.dispatch)( + num_tokens=WarmupIntRange(1, max_tokens + 1), + hidden_size=hidden_size, + hc_mult=hc_mult, + _when=lambda *, num_tokens: num_tokens <= 16, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + num_tokens = 1 if compile_key.tile_n == 2 else 8 + hidden_size = compile_key.hidden_size + hc_mult = compile_key.hc_mult + hc_mult3 = hc_mult * (2 + hc_mult) + comb_mix = make_tilelang_warmup_tensor( + torch.float32, num_tokens, hc_mult, hc_mult + ) + residual_in = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hc_mult, hidden_size + ) + post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult) + x_in = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size) + weight_t = make_tilelang_warmup_tensor( + torch.float32, hc_mult3, hc_mult, hidden_size + ) + return dict( + comb_mix=comb_mix, + residual_in=residual_in, + post_mix=post_mix, + x_in=x_in, + weight_t=weight_t, + hc_mult=hc_mult, + hidden_size=hidden_size, + hc_mult3=hc_mult3, + ) + + @kernel_launcher + def __call__( + self, + comb_mix: Any, + residual_in: Any, + post_mix: Any, + x_in: Any, + weight_t: Any, + hc_mult: int, + hidden_size: int, + hc_mult3: int, + ) -> TileLangLaunchSpec: + num_tokens = residual_in.shape[0] + tile_n = 2 if num_tokens < 8 else 3 + n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) + yp_out = residual_in.new_empty( + (n_splits, num_tokens, hc_mult3), dtype=torch.float32 + ) + rp_out = residual_in.new_empty((n_splits, num_tokens), dtype=torch.float32) + residual_out = residual_in.new_empty(residual_in.shape) + return ( + (), + ( + comb_mix, + residual_in, + post_mix, + x_in, + weight_t, + yp_out, + rp_out, + residual_out, + hc_mult, + hidden_size, + hc_mult3, + ), + dict(tile_n=tile_n, split_k=n_splits), + (yp_out, rp_out, residual_out), + ) + + +class HcHeadFusedTileLangKernel( + VllmTileLangJitKernel["HcHeadFusedTileLangKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + hidden_size: int + hc_mult: int + rms_eps: float + hc_eps: float + + @staticmethod + def kernel() -> Any: + return hc_head_fuse_tilelang + + def dispatch( # type: ignore[override] + self, + *, + hidden_size: int, + hc_mult: int, + rms_eps: float, + hc_eps: float, + ) -> CompileKey: + return self.CompileKey( + hidden_size=hidden_size, + hc_mult=hc_mult, + rms_eps=rms_eps, + hc_eps=hc_eps, + ) + + def get_warmup_keys( + self, + *, + hidden_size: int, + hc_mult: int, + rms_eps: float, + hc_eps: float, + ) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)( + hidden_size=hidden_size, + hc_mult=hc_mult, + rms_eps=rms_eps, + hc_eps=hc_eps, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + num_tokens = 1 + hidden_size = compile_key.hidden_size + hc_mult = compile_key.hc_mult + residual = make_tilelang_warmup_tensor( + torch.bfloat16, num_tokens, hc_mult, hidden_size + ) + fn = make_tilelang_warmup_tensor(torch.float32, hc_mult, hc_mult * hidden_size) + hc_scale = make_tilelang_warmup_tensor(torch.float32, 1) + hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult) + out = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size) + return dict( + residual=residual, + fn=fn, + hc_scale=hc_scale, + hc_base=hc_base, + out=out, + hidden_size=hidden_size, + rms_eps=compile_key.rms_eps, + hc_eps=compile_key.hc_eps, + hc_mult=hc_mult, + ) + + @kernel_launcher + def __call__( + self, + residual: Any, + fn: Any, + hc_scale: Any, + hc_base: Any, + out: Any, + hidden_size: int, + rms_eps: float, + hc_eps: float, + hc_mult: int, + ) -> TileLangLaunchSpec: + return (), ( + residual, + fn, + hc_scale, + hc_base, + out, + hidden_size, + rms_eps, + hc_eps, + hc_mult, + ) + + +_HC_PRENORM_GEMM_TILELANG_KERNEL = HcPrenormGemmTileLangKernel() +_MHC_PRE_BIG_FUSE_TILELANG_KERNEL = MhcPreBigFuseTileLangKernel() +_MHC_POST_TILELANG_KERNEL = MhcPostTileLangKernel() +_MHC_FUSED_TILELANG_KERNEL = MhcFusedTileLangKernel() +_HC_HEAD_FUSED_TILELANG_KERNEL = HcHeadFusedTileLangKernel() diff --git a/vllm/model_executor/warmup/deepseek_v4_mhc_warmup.py b/vllm/model_executor/warmup/deepseek_v4_mhc_warmup.py deleted file mode 100644 index fc7731509d..0000000000 --- a/vllm/model_executor/warmup/deepseek_v4_mhc_warmup.py +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Warm up DeepSeek V4 mHC TileLang kernels before serving requests. - -Ported from lucifer1004/vllm-jasl with the two env-var knobs removed -(`VLLM_ENABLE_DEEPSEEK_V4_MHC_WARMUP`, `VLLM_DEEPSEEK_V4_MHC_WARMUP_TOKEN_SIZES`). -Gating is intrinsic: non-DSv4 models and layers without hc_* attributes -return early, so the warmup is a no-op except where it's needed. -""" - -import time -from collections.abc import Iterable - -import torch - -from vllm.logger import init_logger -from vllm.tracing import instrument - -logger = init_logger(__name__) - -_AUTO_WARMUP_MAX_TOKENS = 16_384 -_DEFAULT_TOKEN_SIZE_CANDIDATES = ( - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1024, - 2048, - 4096, - 8192, - 16_384, -) - - -def _normalize_token_sizes( - token_sizes: Iterable[int], - *, - max_tokens: int, -) -> list[int]: - return sorted({size for size in token_sizes if 1 <= size <= max_tokens}) - - -def _select_mhc_warmup_token_sizes( - *, - max_tokens: int, - cudagraph_capture_sizes: list[int], -) -> list[int]: - if max_tokens <= 0: - return [] - - max_auto_tokens = min(max_tokens, _AUTO_WARMUP_MAX_TOKENS) - candidates = list(_DEFAULT_TOKEN_SIZE_CANDIDATES) - candidates.extend(cudagraph_capture_sizes) - candidates.append(max_auto_tokens) - return _normalize_token_sizes(candidates, max_tokens=max_auto_tokens) - - -def _find_first_mhc_layer(model: torch.nn.Module) -> torch.nn.Module | None: - for module in model.modules(): - if module.__class__.__name__ != "DeepseekV4DecoderLayer": - continue - if all( - hasattr(module, attr) - for attr in ( - "hc_pre", - "hc_post", - "hc_attn_fn", - "hc_attn_scale", - "hc_attn_base", - "hc_ffn_fn", - "hc_ffn_scale", - "hc_ffn_base", - ) - ): - return module - return None - - -def _find_deepseek_v4_model(model: torch.nn.Module) -> torch.nn.Module | None: - for module in model.modules(): - if module.__class__.__name__ != "DeepseekV4Model": - continue - if all( - hasattr(module, attr) - for attr in ("hc_head_fn", "hc_head_scale", "hc_head_base") - ): - return module - return None - - -def _warmup_layer_mhc( - layer: torch.nn.Module, - token_sizes: list[int], -) -> None: - max_tokens = max(token_sizes) - hidden_size = int(layer.hidden_size) - hc_mult = int(layer.hc_mult) - device = layer.hc_attn_fn.device - residual = torch.zeros( - max_tokens, - hc_mult, - hidden_size, - dtype=torch.bfloat16, - device=device, - ) - - for size in token_sizes: - residual_slice = residual[:size] - for fn, scale, base in ( - (layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base), - (layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base), - ): - layer_input, post_mix, comb_mix = layer.hc_pre( - residual_slice, - fn, - scale, - base, - ) - layer.hc_post(layer_input, residual_slice, post_mix, comb_mix) - - -def _warmup_hc_head( - model: torch.nn.Module, - token_sizes: list[int], -) -> None: - # Upstream a8887c208 ("[DSV4] aiter mhc support (ROCm)") refactored - # ``hc_head`` from a free function into the ``HCHeadOp`` CustomOp - # instance attached to the model as ``hc_head_op``. We call through - # that instance so the warmup exercises the same dispatched - # implementation as the inference path. - hc_head_op = getattr(model, "hc_head_op", None) - if hc_head_op is None: - return - - max_tokens = max(token_sizes) - hidden_size = int(model.config.hidden_size) - hc_mult = int(model.hc_mult) - device = model.hc_head_fn.device - hidden_states = torch.zeros( - max_tokens, - hc_mult, - hidden_size, - dtype=torch.bfloat16, - device=device, - ) - - for size in token_sizes: - hc_head_op( - hidden_states[:size], - model.hc_head_fn, - model.hc_head_scale, - model.hc_head_base, - model.rms_norm_eps, - model.hc_eps, - ) - - -@instrument(span_name="DeepSeek V4 mHC warmup") -def deepseek_v4_mhc_warmup( - model: torch.nn.Module, - *, - max_tokens: int, - cudagraph_capture_sizes: list[int] | None = None, -) -> None: - # Cheap model-type gate before walking ``model.modules()``. The class - # walk below is O(num_layers) and shows up in startup time on very - # large checkpoints; bail out for any model that is not DeepSeek V4. - config = getattr(model, "config", None) - model_type = getattr(config, "model_type", None) if config is not None else None - if model_type is not None and model_type not in ( - "deepseek_v4", - "deepseek_v41", - ): - return - - layer = _find_first_mhc_layer(model) - if layer is None: - return - - device = layer.hc_attn_fn.device - if device.type != "cuda": - return - - deepseek_model = _find_deepseek_v4_model(model) - token_sizes = _select_mhc_warmup_token_sizes( - max_tokens=max_tokens, - cudagraph_capture_sizes=cudagraph_capture_sizes or [], - ) - if not token_sizes: - return - - started = time.perf_counter() - logger.info( - "Warming up DeepSeek V4 mHC TileLang kernels for token sizes: %s", - token_sizes, - ) - with torch.inference_mode(): - _warmup_layer_mhc(layer, token_sizes) - if deepseek_model is not None: - _warmup_hc_head(deepseek_model, token_sizes) - torch.accelerator.synchronize() - logger.info( - "DeepSeek V4 mHC TileLang warmup finished in %.2f seconds.", - time.perf_counter() - started, - ) diff --git a/vllm/model_executor/warmup/jit_warmup_tilelang_helper.py b/vllm/model_executor/warmup/jit_warmup_tilelang_helper.py new file mode 100644 index 0000000000..056ca51805 --- /dev/null +++ b/vllm/model_executor/warmup/jit_warmup_tilelang_helper.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compile-only helpers for TileLang JIT warmup.""" + +from __future__ import annotations + +import logging +from abc import abstractmethod +from collections.abc import Callable, Iterator, Mapping, MutableMapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from functools import wraps +from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast + +import torch + +from vllm.model_executor.warmup.jit_warmup import VllmJitKernel + +CompileKeyT = TypeVar("CompileKeyT") +# (kernel_args, launch_args), optionally followed by keywords and a result. +TileLangLaunchSpec: TypeAlias = ( + tuple[tuple[Any, ...], tuple[Any, ...]] + | tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None] + | tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None, Any] +) + + +@contextmanager +def _quiet_tilelang_warmup_logs() -> Iterator[None]: + loggers = [ + logging.getLogger("tilelang.cache.kernel_cache"), + logging.getLogger("tilelang.jit.kernel"), + ] + previous_levels = [logger.level for logger in loggers] + try: + for logger in loggers: + logger.setLevel(logging.ERROR) + yield + finally: + for logger, level in zip(loggers, previous_levels): + logger.setLevel(level) + + +@dataclass(frozen=True) +class TileLangWarmupTensor: + """Minimal tensor-like object accepted by TileLang compile(). + + TileLang builds its cache key and TIR from tensor dtype, shape and stride. + This object deliberately has no storage, so compile-only warmup does not + allocate GPU memory and does not launch the kernel. + """ + + dtype: torch.dtype + shape: tuple[int, ...] = (1,) + strides: tuple[int, ...] | None = field(default=None) + + def stride(self) -> tuple[int, ...]: + if self.strides is not None: + return self.strides + + strides: list[int] = [] + stride = 1 + for size in reversed(self.shape): + strides.append(stride) + stride *= size + return tuple(reversed(strides)) + + def new_empty( + self, + shape: tuple[int, ...], + *, + dtype: torch.dtype | None = None, + device: Any = None, + ) -> TileLangWarmupTensor: + return TileLangWarmupTensor(dtype or self.dtype, shape) + + +def make_tilelang_warmup_tensor( + dtype: torch.dtype, + *shape: int, + strides: tuple[int, ...] | None = None, +) -> TileLangWarmupTensor: + return TileLangWarmupTensor(dtype=dtype, shape=tuple(shape), strides=strides) + + +def _tilelang_call_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]: + call_kwargs = dict(kwargs) + tune_params = call_kwargs.pop("__tune_params", {}) + if isinstance(tune_params, dict): + call_kwargs.update(tune_params) + return call_kwargs + + +def compile_tilelang(jit_impl: Any, *args: Any, **kwargs: Any) -> None: + """Compile one TileLang specialization and populate its call cache. + + TileLang's ``compile()`` materializes the kernel without launching it. + We also store the compiled kernel in ``_kernel_cache`` using the same + parsed key as ``__call__`` so runtime does not report a cache miss for an + already materialized specialization. + """ + + with _quiet_tilelang_warmup_logs(): + compiled = jit_impl.compile(*args, **kwargs) + func = getattr(jit_impl, "func", None) + parse_args = getattr(func, "parse_args", None) + cache = getattr(jit_impl, "_kernel_cache", None) + if not callable(parse_args) or not isinstance(cache, MutableMapping): + return + + key, _ = parse_args(*args, **_tilelang_call_kwargs(kwargs)) + cache[key] = compiled + + +class VllmTileLangJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): + """TileLang owner whose runtime launch specification is reused for warmup.""" + + kernel: ClassVar[Any] + _warming_key: CompileKeyT | None = None + + @abstractmethod + def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]: + """Return runtime-shaped inputs that reproduce one compile key.""" + raise NotImplementedError + + def compile(self, compile_key: CompileKeyT) -> None: + self._warming_key = compile_key + try: + cast(Callable[..., Any], self)(**self.warmup_inputs(compile_key)) + finally: + self._warming_key = None + + def launch( + self, + jit_impl: Any, + args: tuple[Any, ...], + kwargs: Mapping[str, Any] | None = None, + ) -> Any: + call_kwargs = dict(kwargs or {}) + if self._warming_key is not None: + return compile_tilelang(jit_impl, *args, **call_kwargs) + return jit_impl(*args, **call_kwargs) + + +def kernel_launcher( + call_fn: Callable[..., TileLangLaunchSpec], +) -> Callable[..., Any]: + """Launch TileLang from declarative kernel and runtime argument tuples.""" + + @wraps(call_fn) + def wrapper( + self: VllmTileLangJitKernel[Any], + *args: Any, + **kwargs: Any, + ) -> Any: + launch_spec = call_fn(self, *args, **kwargs) + kernel_args, launch_args = launch_spec[:2] + if self._warming_key is not None and kernel_args not in ( + (), + (self._warming_key,), + ): + raise RuntimeError( + "TileLang warmup inputs produced a different compile key: " + f"expected {self._warming_key!r}, got kernel arguments " + f"{kernel_args!r}" + ) + launch_kwargs = launch_spec[2] if len(launch_spec) > 2 else None + output = launch_spec[3] if len(launch_spec) > 3 else None + result = self.launch(self.kernel(*kernel_args), launch_args, launch_kwargs) + return output if len(launch_spec) > 3 else result + + return wrapper diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index e56ae50ebb..80b5160d26 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -17,9 +17,6 @@ from vllm.logger import init_logger from vllm.model_executor.warmup.b12x_warmup import b12x_warmup from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup -from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import ( - deepseek_v4_mhc_warmup, -) from vllm.model_executor.warmup.flashinfer_autotune_cache import ( resolve_flashinfer_autotune_file, write_flashinfer_autotune_cache, @@ -197,15 +194,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): compilation_config = worker.vllm_config.compilation_config cudagraph_capture_sizes = list(compilation_config.cudagraph_capture_sizes or []) - # DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder - # layer per token; warm them across token sizes first so the first real - # request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside). - deepseek_v4_mhc_warmup( - worker.get_model(), - max_tokens=worker.scheduler_config.max_num_batched_tokens, - cudagraph_capture_sizes=cudagraph_capture_sizes, - ) - # Run next so input-prep kernels JIT against pristine runner state. if enable_jit_warmup: kimi_k3_triton_warmup(worker) diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 97e7e617fe..b1379cc9eb 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -1210,6 +1210,53 @@ class DeepseekV4DecoderLayer(nn.Module): requires_grad=False, ) + if vllm_config.kernel_config.enable_jit_warmup: + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, + _MHC_FUSED_TILELANG_KERNEL, + _MHC_POST_TILELANG_KERNEL, + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, + ) + from vllm.utils.deep_gemm import is_deep_gemm_supported + + include_pre_gemm_splits = is_deep_gemm_supported() + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.hc_mult, + use_norm_weight=True, + include_pre_gemm_splits=include_pre_gemm_splits, + include_broadcast_splits=( + get_pp_group().is_first_rank and extract_layer_index(prefix) == 0 + ), + rms_eps=self.rms_norm_eps, + hc_pre_eps=self.hc_eps, + hc_sinkhorn_eps=self.hc_eps, + hc_post_mult_value=self.hc_post_alpha, + sinkhorn_repeat=self.hc_sinkhorn_iters, + norm_eps=( + self.attn_norm.variance_epsilon, + self.ffn_norm.variance_epsilon, + ), + broadcast_norm_eps=self.attn_norm.variance_epsilon, + ) + if not include_pre_gemm_splits: + _HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.hc_mult, + n_out=self.hc_mult * (2 + self.hc_mult), + ) + _MHC_POST_TILELANG_KERNEL.register_warmup( + hidden_size=self.hidden_size, + hc_mult=self.hc_mult, + ) + _MHC_FUSED_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.hc_mult, + ) + def forward( self, x: torch.Tensor, @@ -1402,6 +1449,18 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): else: self._mtp_hidden_buffer = None + if vllm_config.kernel_config.enable_jit_warmup and get_pp_group().is_last_rank: + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_HEAD_FUSED_TILELANG_KERNEL, + ) + + _HC_HEAD_FUSED_TILELANG_KERNEL.register_warmup( + hidden_size=config.hidden_size, + hc_mult=self.hc_mult, + rms_eps=self.rms_norm_eps, + hc_eps=self.hc_eps, + ) + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) diff --git a/vllm/models/glm5next/nvidia/model.py b/vllm/models/glm5next/nvidia/model.py index 8444c81416..21918bdf6b 100644 --- a/vllm/models/glm5next/nvidia/model.py +++ b/vllm/models/glm5next/nvidia/model.py @@ -403,6 +403,50 @@ class Glm5NextDecoderLayer(nn.Module): self.mhc_post_op = MHCPostOp() self.mhc_fused_post_pre_op = MHCFusedPostPreOp() + if vllm_config.kernel_config.enable_jit_warmup: + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, + _MHC_FUSED_TILELANG_KERNEL, + _MHC_POST_TILELANG_KERNEL, + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, + ) + from vllm.utils.deep_gemm import is_deep_gemm_supported + + include_pre_gemm_splits = is_deep_gemm_supported() + _MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.n, + use_norm_weight=True, + include_pre_gemm_splits=include_pre_gemm_splits, + include_broadcast_splits=False, + rms_eps=self.rms_norm_eps, + hc_pre_eps=self.hc_eps, + hc_sinkhorn_eps=self.hc_eps, + hc_post_mult_value=self.mhc_post_mult_value, + sinkhorn_repeat=self.mhc_sinkhorn_iterations, + norm_eps=( + self.input_layernorm.variance_epsilon, + self.post_attention_layernorm.variance_epsilon, + ), + ) + if not include_pre_gemm_splits: + _HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.n, + n_out=self.n * (2 + self.n), + ) + _MHC_POST_TILELANG_KERNEL.register_warmup( + hidden_size=self.hidden_size, + hc_mult=self.n, + ) + _MHC_FUSED_TILELANG_KERNEL.register_warmup( + vllm_config, + hidden_size=self.hidden_size, + hc_mult=self.n, + ) + def forward( self, positions: torch.Tensor,