[9/N][warmup][DSv4] Migrate MHC TileLang kernels (#50178)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
aed894c190
commit
e52be1a62d
@@ -0,0 +1,296 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Validate JIT dispatch against pre-contract behavior."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("NVIDIA dispatch tests require CUDA", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
HcHeadFusedTileLangKernel,
|
||||
HcPrenormGemmTileLangKernel,
|
||||
MhcFusedTileLangKernel,
|
||||
MhcPostTileLangKernel,
|
||||
MhcPreBigFuseTileLangKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "expected"),
|
||||
[
|
||||
(
|
||||
dict(
|
||||
num_tokens=64,
|
||||
hc_hidden_size=4096,
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
),
|
||||
(2048, 2, 128, 1024, 4, 1, False, 1),
|
||||
),
|
||||
(
|
||||
dict(
|
||||
num_tokens=1024,
|
||||
hc_hidden_size=4096,
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
),
|
||||
(2048, 2, 128, 512, 12, 1, True, 2),
|
||||
),
|
||||
(
|
||||
dict(
|
||||
num_tokens=64,
|
||||
hc_hidden_size=4096,
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
n_thr=256,
|
||||
tile_n=8,
|
||||
n_splits=4,
|
||||
),
|
||||
(2048, 2, 128, 256, 8, 4, False, 1),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hc_prenorm_gemm_dispatch_matches_legacy_runtime_config(
|
||||
kwargs: dict[str, Any],
|
||||
expected: tuple[int, int, int, int, int, int, bool, int],
|
||||
) -> None:
|
||||
kernel = HcPrenormGemmTileLangKernel()
|
||||
|
||||
assert kernel.dispatch(**kwargs) == kernel.CompileKey(*expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("is_broadcast", "use_norm_weight", "expected_use_norm", "expected_eps"),
|
||||
[
|
||||
(False, False, False, 0.0),
|
||||
(False, True, True, 1.0e-5),
|
||||
(True, False, True, 2.0e-5),
|
||||
],
|
||||
)
|
||||
def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
|
||||
is_broadcast: bool,
|
||||
use_norm_weight: bool,
|
||||
expected_use_norm: bool,
|
||||
expected_eps: float,
|
||||
) -> None:
|
||||
kernel = MhcPreBigFuseTileLangKernel()
|
||||
|
||||
assert kernel.dispatch(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=2,
|
||||
is_broadcast=is_broadcast,
|
||||
use_norm_weight=use_norm_weight,
|
||||
rms_eps=1.0e-6,
|
||||
hc_pre_eps=2.0e-6,
|
||||
hc_sinkhorn_eps=3.0e-6,
|
||||
hc_post_mult_value=0.5,
|
||||
sinkhorn_repeat=3,
|
||||
norm_eps=1.0e-5,
|
||||
broadcast_norm_eps=2.0e-5,
|
||||
) == kernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=2,
|
||||
use_norm_weight=expected_use_norm,
|
||||
is_broadcast=is_broadcast,
|
||||
rms_eps=1.0e-6,
|
||||
hc_pre_eps=2.0e-6,
|
||||
hc_sinkhorn_eps=3.0e-6,
|
||||
hc_post_mult_value=0.5,
|
||||
sinkhorn_repeat=3,
|
||||
norm_eps=expected_eps,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
|
||||
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
|
||||
)
|
||||
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
expected_n_splits: int,
|
||||
expected_tile_n: int,
|
||||
) -> None:
|
||||
kernel = MhcFusedTileLangKernel()
|
||||
|
||||
assert kernel.dispatch(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=4,
|
||||
) == kernel.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=4,
|
||||
n_splits=expected_n_splits,
|
||||
tile_n=expected_tile_n,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel", "compile_key"),
|
||||
[
|
||||
(
|
||||
HcPrenormGemmTileLangKernel(),
|
||||
HcPrenormGemmTileLangKernel.CompileKey(
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
n_thr=1024,
|
||||
tile_n=4,
|
||||
n_splits=1,
|
||||
use_block_m=False,
|
||||
block_m=1,
|
||||
),
|
||||
),
|
||||
(
|
||||
HcPrenormGemmTileLangKernel(),
|
||||
HcPrenormGemmTileLangKernel.CompileKey(
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
n_thr=512,
|
||||
tile_n=12,
|
||||
n_splits=1,
|
||||
use_block_m=False,
|
||||
block_m=1,
|
||||
),
|
||||
),
|
||||
(
|
||||
HcPrenormGemmTileLangKernel(),
|
||||
HcPrenormGemmTileLangKernel.CompileKey(
|
||||
hidden_size=2048,
|
||||
hc_mult=2,
|
||||
n_out=128,
|
||||
n_thr=512,
|
||||
tile_n=12,
|
||||
n_splits=1,
|
||||
use_block_m=True,
|
||||
block_m=2,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcPreBigFuseTileLangKernel(),
|
||||
MhcPreBigFuseTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=2,
|
||||
use_norm_weight=True,
|
||||
is_broadcast=False,
|
||||
rms_eps=1.0e-6,
|
||||
hc_pre_eps=2.0e-6,
|
||||
hc_sinkhorn_eps=3.0e-6,
|
||||
hc_post_mult_value=0.5,
|
||||
sinkhorn_repeat=3,
|
||||
norm_eps=1.0e-5,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcPreBigFuseTileLangKernel(),
|
||||
MhcPreBigFuseTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=1,
|
||||
use_norm_weight=False,
|
||||
is_broadcast=False,
|
||||
rms_eps=1.0e-6,
|
||||
hc_pre_eps=2.0e-6,
|
||||
hc_sinkhorn_eps=3.0e-6,
|
||||
hc_post_mult_value=0.5,
|
||||
sinkhorn_repeat=3,
|
||||
norm_eps=0.0,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcPreBigFuseTileLangKernel(),
|
||||
MhcPreBigFuseTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=2,
|
||||
use_norm_weight=True,
|
||||
is_broadcast=True,
|
||||
rms_eps=1.0e-6,
|
||||
hc_pre_eps=2.0e-6,
|
||||
hc_sinkhorn_eps=3.0e-6,
|
||||
hc_post_mult_value=0.5,
|
||||
sinkhorn_repeat=3,
|
||||
norm_eps=1.0e-5,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcPostTileLangKernel(),
|
||||
MhcPostTileLangKernel.CompileKey(hidden_size=4096, hc_mult=4),
|
||||
),
|
||||
(
|
||||
MhcFusedTileLangKernel(),
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
tile_n=3,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcFusedTileLangKernel(),
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=8,
|
||||
tile_n=2,
|
||||
),
|
||||
),
|
||||
(
|
||||
MhcFusedTileLangKernel(),
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=8192,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
tile_n=2,
|
||||
),
|
||||
),
|
||||
(
|
||||
HcHeadFusedTileLangKernel(),
|
||||
HcHeadFusedTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
rms_eps=1.0e-6,
|
||||
hc_eps=2.0e-6,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_tilelang_warmup_inputs_reproduce_compile_key(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
kernel: Any,
|
||||
compile_key: Any,
|
||||
) -> None:
|
||||
compiled: list[Any] = []
|
||||
single_kernel = isinstance(
|
||||
kernel,
|
||||
(MhcPostTileLangKernel, MhcFusedTileLangKernel, HcHeadFusedTileLangKernel),
|
||||
)
|
||||
expected_kernel = kernel.kernel() if single_kernel else kernel.kernel(compile_key)
|
||||
if single_kernel:
|
||||
monkeypatch.setattr(
|
||||
kernel,
|
||||
"dispatch",
|
||||
lambda **kwargs: pytest.fail("single-kernel launch must not dispatch"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jit_warmup_tilelang_helper,
|
||||
"compile_tilelang",
|
||||
lambda jit_impl, *args, **kwargs: compiled.append(jit_impl),
|
||||
)
|
||||
|
||||
kernel.compile(compile_key)
|
||||
|
||||
assert compiled == [expected_kernel]
|
||||
@@ -10,10 +10,12 @@ import torch.nn.functional as F
|
||||
import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_tilelang_hc_prenorm_gemm,
|
||||
_torch_hc_prenorm_gemm,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
|
||||
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
@@ -485,7 +487,14 @@ def test_hc_prenorm_gemm_tilelang(num_tokens, hidden_size):
|
||||
sqrsum = torch.empty_like(sqrsum_ref)
|
||||
|
||||
_torch_hc_prenorm_gemm(x, fn, out_ref, sqrsum_ref)
|
||||
_tilelang_hc_prenorm_gemm(x, fn, out, sqrsum, hidden_size, hc_mult)
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, out_ref, atol=1e-5, rtol=1e-4)
|
||||
torch.testing.assert_close(sqrsum, sqrsum_ref, atol=1e-2, rtol=1e-6)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@@ -18,73 +19,56 @@ def _torch_hc_prenorm_gemm(
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
def _tilelang_hc_prenorm_gemm(
|
||||
def _hc_prenorm_gemm_outputs(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
tile_n: int = 12,
|
||||
n_thr: int = 512,
|
||||
n_splits: int = 1,
|
||||
) -> None:
|
||||
use_tilelang_fallback: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
hc_prenorm_gemm_block_m_tilelang,
|
||||
hc_prenorm_gemm_tilelang,
|
||||
compute_num_split,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
tf32_hc_prenorm_gemm,
|
||||
)
|
||||
|
||||
assert out.shape[0] == n_splits
|
||||
assert sqrsum.shape[0] == n_splits
|
||||
assert x.shape[1] == hc_mult * hidden_size
|
||||
assert x.shape[1] % n_splits == 0
|
||||
assert (x.shape[1] // n_splits) % n_thr == 0
|
||||
use_default_config = tile_n == 12 and n_thr == 512
|
||||
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
|
||||
hc_prenorm_gemm_block_m_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if (
|
||||
n_splits == 1
|
||||
and use_default_config
|
||||
and x.shape[0] < 128
|
||||
and x.shape[1] % 1024 == 0
|
||||
):
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
1024,
|
||||
4,
|
||||
n_splits,
|
||||
)
|
||||
return
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
n_splits,
|
||||
use_deep_gemm = is_deep_gemm_supported() or not use_tilelang_fallback
|
||||
num_tokens = x.shape[0]
|
||||
n_splits = (
|
||||
compute_num_split(64, x.shape[1], cdiv(num_tokens, 64)) if use_deep_gemm else 1
|
||||
)
|
||||
out = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
fn.shape[0],
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(x, fn, out, sqrsum, n_splits)
|
||||
else:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
return out, sqrsum
|
||||
|
||||
|
||||
def mhc_pre_delayed_tilelang(
|
||||
@@ -127,6 +111,7 @@ def mhc_pre_delayed_tilelang(
|
||||
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.warmup import (
|
||||
@@ -186,7 +171,14 @@ def mhc_pre_delayed_tilelang(
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(x, fn, mixes, sqrsum, n_splits)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(x, fn, mixes, sqrsum, input_size, 1)
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
x,
|
||||
fn,
|
||||
mixes,
|
||||
sqrsum,
|
||||
input_size,
|
||||
1,
|
||||
)
|
||||
if norm_weight is not None:
|
||||
assert norm_weight.shape == (hidden_size,)
|
||||
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
|
||||
@@ -298,7 +290,8 @@ def mhc_pre_tilelang(
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
n_splits: retained for the shared MHC operator API; the active GEMM
|
||||
backend selects its split factor internally.
|
||||
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
|
||||
torch.bfloat16. When provided, RMSNorm is fused into the
|
||||
layer_input write path of the big_fuse kernel.
|
||||
@@ -311,12 +304,8 @@ def mhc_pre_tilelang(
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
@@ -346,17 +335,6 @@ def mhc_pre_tilelang(
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
if use_deep_gemm:
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
@@ -367,78 +345,30 @@ def mhc_pre_tilelang(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
# Pre-mix buffers are unused in V4 mode.
|
||||
post_mix,
|
||||
post_mix,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
post_mix,
|
||||
post_mix,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
@@ -505,11 +435,10 @@ def mhc_pre_broadcast_tilelang(
|
||||
fn_broadcast: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""First-layer mHC pre for a residual broadcast from ``(T, H)``."""
|
||||
# n_splits is retained for the shared API; split selection is internal.
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_broadcast_with_norm_tilelang,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert norm_weight is not None, "broadcast mHC pre currently requires fused RMSNorm"
|
||||
assert residual.dtype == torch.bfloat16
|
||||
@@ -537,8 +466,6 @@ def mhc_pre_broadcast_tilelang(
|
||||
residual_flat = residual
|
||||
num_tokens = residual.shape[0]
|
||||
|
||||
n_splits = compute_num_split(64, hidden_size, cdiv(num_tokens, 64))
|
||||
|
||||
residual_out = torch.empty(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
@@ -551,42 +478,31 @@ def mhc_pre_broadcast_tilelang(
|
||||
layer_input = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_flat,
|
||||
fn_broadcast,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
use_tilelang_fallback=False,
|
||||
)
|
||||
mhc_pre_big_fuse_broadcast_with_norm_tilelang(
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
residual_out,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
residual_out=residual_out,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
return (
|
||||
residual_out,
|
||||
@@ -603,11 +519,11 @@ def mhc_post_tilelang(
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
mhc_post_tilelang as _mhc_post_kernel,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
out = torch.empty_like(residual)
|
||||
_mhc_post_kernel(
|
||||
_MHC_POST_TILELANG_KERNEL(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.squeeze(-1),
|
||||
@@ -644,6 +560,9 @@ def mhc_fused_post_pre_tilelang(
|
||||
RMSNorm'd activation (fused into the kernel); otherwise it is the
|
||||
raw pre-norm activation as before.
|
||||
|
||||
``n_splits`` and ``tile_n`` are retained for the shared MHC operator API.
|
||||
The TileLang path selects both values internally from the runtime shape.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
@@ -652,13 +571,10 @@ def mhc_fused_post_pre_tilelang(
|
||||
"""
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_fused_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert x.dtype == torch.bfloat16
|
||||
@@ -692,48 +608,14 @@ def mhc_fused_post_pre_tilelang(
|
||||
if not norm_weight.is_contiguous():
|
||||
norm_weight = norm_weight.contiguous()
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
x_flat = x.view(num_tokens, hidden_size)
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
use_small_fma = num_tokens <= 16
|
||||
if use_small_fma:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
if use_deep_gemm:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(
|
||||
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
|
||||
)
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
@@ -754,23 +636,36 @@ def mhc_fused_post_pre_tilelang(
|
||||
)
|
||||
|
||||
if use_small_fma:
|
||||
mhc_fused_tilelang(
|
||||
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
fn.view(hc_mult3, hc_mult, hidden_size),
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
else:
|
||||
mhc_post_tilelang(
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
_MHC_POST_TILELANG_KERNEL(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
@@ -779,52 +674,13 @@ def mhc_fused_post_pre_tilelang(
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
# Pre-mix buffers are unused in V4 mode.
|
||||
post_mix_cur,
|
||||
post_mix_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
gemm_out_mul, gemm_out_sqrsum = _hc_prenorm_gemm_outputs(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
@@ -833,18 +689,13 @@ def mhc_fused_post_pre_tilelang(
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
norm_weight,
|
||||
post_mix_cur,
|
||||
post_mix_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
return (
|
||||
@@ -926,9 +777,11 @@ def hc_head_fused_kernel_tilelang(
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import hc_head_fuse_tilelang
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
hc_head_fuse_tilelang(
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
|
||||
@@ -4,10 +4,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, zip_inputs
|
||||
from vllm.model_executor.warmup.jit_warmup_tilelang_helper import (
|
||||
TileLangLaunchSpec,
|
||||
VllmTileLangJitKernel,
|
||||
kernel_launcher,
|
||||
make_tilelang_warmup_tensor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tilelang_utils import T, tilelang, tilelang_jit
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -28,6 +37,10 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
|
||||
return split_k
|
||||
|
||||
|
||||
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
|
||||
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
|
||||
|
||||
|
||||
@tilelang_jit
|
||||
def mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
@@ -987,3 +1000,745 @@ def hc_head_fuse_tilelang(
|
||||
|
||||
if ENABLE_PDL:
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
class HcPrenormGemmTileLangKernel(
|
||||
VllmTileLangJitKernel["HcPrenormGemmTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_out: int
|
||||
n_thr: int
|
||||
tile_n: int
|
||||
n_splits: int
|
||||
use_block_m: bool
|
||||
block_m: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
if compile_key.use_block_m:
|
||||
return hc_prenorm_gemm_block_m_tilelang
|
||||
return hc_prenorm_gemm_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
num_tokens: int,
|
||||
hc_hidden_size: int,
|
||||
n_thr: int = 512,
|
||||
tile_n: int = 12,
|
||||
n_splits: int = 1,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
use_default_config = n_splits == 1 and tile_n == 12 and n_thr == 512
|
||||
use_block_m = use_default_config and num_tokens >= 1024
|
||||
use_small_tile = (
|
||||
use_default_config
|
||||
and not use_block_m
|
||||
and num_tokens < 128
|
||||
and hc_hidden_size % 1024 == 0
|
||||
)
|
||||
effective_n_thr = 1024 if use_small_tile else n_thr
|
||||
effective_tile_n = 4 if use_small_tile else tile_n
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
n_thr=effective_n_thr,
|
||||
tile_n=effective_tile_n,
|
||||
n_splits=n_splits,
|
||||
use_block_m=use_block_m,
|
||||
block_m=2 if use_block_m else 1,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
n_out: int,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
num_tokens=WarmupIntRange(
|
||||
1,
|
||||
max_tokens + 1,
|
||||
advance=lambda value: (
|
||||
128 if value < 128 else 1024 if value < 1024 else max_tokens + 1
|
||||
),
|
||||
),
|
||||
hc_hidden_size=hidden_size * hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_out=n_out,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 128
|
||||
if compile_key.use_block_m:
|
||||
num_tokens = 1024
|
||||
elif compile_key.n_thr == 1024:
|
||||
num_tokens = 1
|
||||
hc_hidden_size = compile_key.hidden_size * compile_key.hc_mult
|
||||
return dict(
|
||||
x=make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hc_hidden_size),
|
||||
fn=make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_out, hc_hidden_size
|
||||
),
|
||||
out=make_tilelang_warmup_tensor(
|
||||
torch.float32,
|
||||
compile_key.n_splits,
|
||||
num_tokens,
|
||||
compile_key.n_out,
|
||||
),
|
||||
sqrsum=make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens
|
||||
),
|
||||
hidden_size=compile_key.hidden_size,
|
||||
hc_mult=compile_key.hc_mult,
|
||||
tile_n=compile_key.tile_n,
|
||||
n_thr=compile_key.n_thr,
|
||||
n_splits=compile_key.n_splits,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
x: Any,
|
||||
fn: Any,
|
||||
out: Any,
|
||||
sqrsum: Any,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
tile_n: int = 12,
|
||||
n_thr: int = 512,
|
||||
n_splits: int = 1,
|
||||
) -> TileLangLaunchSpec:
|
||||
assert out.shape[0] == n_splits
|
||||
assert sqrsum.shape[0] == n_splits
|
||||
assert x.shape[1] == hc_mult * hidden_size
|
||||
assert x.shape[1] % n_splits == 0
|
||||
assert (x.shape[1] // n_splits) % n_thr == 0
|
||||
compile_key = self.dispatch(
|
||||
num_tokens=x.shape[0],
|
||||
hc_hidden_size=x.shape[1],
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_out=fn.shape[0],
|
||||
n_thr=n_thr,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
kernel_arg = (
|
||||
compile_key.block_m if compile_key.use_block_m else compile_key.n_splits
|
||||
)
|
||||
return (compile_key,), (
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
compile_key.n_thr,
|
||||
compile_key.tile_n,
|
||||
kernel_arg,
|
||||
)
|
||||
|
||||
|
||||
class MhcPreBigFuseTileLangKernel(
|
||||
VllmTileLangJitKernel["MhcPreBigFuseTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_splits: int
|
||||
use_norm_weight: bool
|
||||
is_broadcast: bool
|
||||
rms_eps: float
|
||||
hc_pre_eps: float
|
||||
hc_sinkhorn_eps: float
|
||||
hc_post_mult_value: float
|
||||
sinkhorn_repeat: int
|
||||
norm_eps: float
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
if compile_key.is_broadcast:
|
||||
return mhc_pre_big_fuse_broadcast_with_norm_tilelang
|
||||
if compile_key.use_norm_weight:
|
||||
return mhc_pre_big_fuse_with_norm_tilelang
|
||||
return mhc_pre_big_fuse_tilelang
|
||||
|
||||
@staticmethod
|
||||
def _kernel_args(
|
||||
compile_key: CompileKey,
|
||||
*,
|
||||
gemm_out_mul: Any,
|
||||
gemm_out_sqrsum: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
residual: Any,
|
||||
residual_out: Any | None,
|
||||
post_mix: Any,
|
||||
comb_mix: Any,
|
||||
layer_input: Any,
|
||||
norm_weight: Any | None,
|
||||
) -> tuple[Any, ...]:
|
||||
common_args = (
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
)
|
||||
output_args = (post_mix, comb_mix, layer_input)
|
||||
eps_args = (
|
||||
compile_key.rms_eps,
|
||||
compile_key.hc_pre_eps,
|
||||
compile_key.hc_sinkhorn_eps,
|
||||
compile_key.hc_post_mult_value,
|
||||
compile_key.sinkhorn_repeat,
|
||||
)
|
||||
middle_args: tuple[Any, ...]
|
||||
if compile_key.is_broadcast:
|
||||
assert residual_out is not None
|
||||
assert norm_weight is not None
|
||||
middle_args = (residual_out, *output_args, norm_weight)
|
||||
elif compile_key.use_norm_weight:
|
||||
assert norm_weight is not None
|
||||
middle_args = (*output_args, norm_weight, post_mix, post_mix)
|
||||
else:
|
||||
middle_args = (*output_args, post_mix, post_mix)
|
||||
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
|
||||
return (
|
||||
*common_args,
|
||||
*middle_args,
|
||||
compile_key.hidden_size,
|
||||
*eps_args,
|
||||
*norm_eps_args,
|
||||
compile_key.n_splits,
|
||||
compile_key.hc_mult,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
n_splits: int,
|
||||
is_broadcast: bool,
|
||||
use_norm_weight: bool,
|
||||
use_pre_gemm_splits: bool = False,
|
||||
pre_gemm_k: int = 0,
|
||||
sinkhorn_repeat: int,
|
||||
norm_eps: float,
|
||||
broadcast_norm_eps: float = 0.0,
|
||||
num_tokens: int = 1,
|
||||
use_fused_tilelang: bool = False,
|
||||
**compile_key_fields: float,
|
||||
) -> CompileKey:
|
||||
pre_gemm_n_splits = (
|
||||
compute_num_split(64, pre_gemm_k, (num_tokens + 63) // 64)
|
||||
if use_pre_gemm_splits
|
||||
else n_splits
|
||||
)
|
||||
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
|
||||
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
|
||||
actual_use_norm_weight = use_norm_weight or is_broadcast
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=actual_n_splits,
|
||||
use_norm_weight=actual_use_norm_weight,
|
||||
is_broadcast=is_broadcast,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=actual_norm_eps if actual_use_norm_weight else 0.0,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
use_norm_weight: bool,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
norm_eps: float | tuple[float, ...],
|
||||
broadcast_norm_eps: float | tuple[float, ...] | None = None,
|
||||
include_pre_gemm_splits: bool = False,
|
||||
include_broadcast_splits: bool = False,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
|
||||
warmup_cases = zip_inputs(
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=False,
|
||||
pre_gemm_k=0,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=include_pre_gemm_splits,
|
||||
pre_gemm_k=hc_mult * hidden_size,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=include_broadcast_splits,
|
||||
use_pre_gemm_splits=include_broadcast_splits,
|
||||
pre_gemm_k=hidden_size,
|
||||
use_fused_tilelang=False,
|
||||
),
|
||||
dict(
|
||||
is_broadcast=False,
|
||||
use_pre_gemm_splits=False,
|
||||
pre_gemm_k=0,
|
||||
use_fused_tilelang=True,
|
||||
),
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
warmup_cases,
|
||||
num_tokens=WarmupIntRange(
|
||||
1,
|
||||
max_tokens + 1,
|
||||
advance=lambda value: 8 if value < 8 else cdiv(value, 64) * 64 + 1,
|
||||
),
|
||||
n_splits=1,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
use_norm_weight=use_norm_weight,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
broadcast_norm_eps=broadcast_norm_eps
|
||||
if broadcast_norm_eps is not None
|
||||
else norm_eps,
|
||||
_when=lambda *, use_fused_tilelang, num_tokens: (
|
||||
not use_fused_tilelang or num_tokens <= 16
|
||||
),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
num_tokens = 1
|
||||
|
||||
gemm_out_mul = make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens, hc_mult3
|
||||
)
|
||||
gemm_out_sqrsum = make_tilelang_warmup_tensor(
|
||||
torch.float32, compile_key.n_splits, num_tokens
|
||||
)
|
||||
hc_scale = make_tilelang_warmup_tensor(torch.float32, 3)
|
||||
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult3)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult * hc_mult
|
||||
)
|
||||
layer_input = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hidden_size
|
||||
)
|
||||
residual_shape = (
|
||||
(num_tokens, hidden_size)
|
||||
if compile_key.is_broadcast
|
||||
else (num_tokens, hc_mult, hidden_size)
|
||||
)
|
||||
residual = make_tilelang_warmup_tensor(torch.bfloat16, *residual_shape)
|
||||
residual_out = (
|
||||
make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
if compile_key.is_broadcast
|
||||
else None
|
||||
)
|
||||
norm_weight = (
|
||||
make_tilelang_warmup_tensor(torch.bfloat16, hidden_size)
|
||||
if compile_key.is_broadcast or compile_key.use_norm_weight
|
||||
else None
|
||||
)
|
||||
return dict(
|
||||
gemm_out_mul=gemm_out_mul,
|
||||
gemm_out_sqrsum=gemm_out_sqrsum,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
residual=residual,
|
||||
post_mix=post_mix,
|
||||
comb_mix=comb_mix,
|
||||
layer_input=layer_input,
|
||||
rms_eps=compile_key.rms_eps,
|
||||
hc_pre_eps=compile_key.hc_pre_eps,
|
||||
hc_sinkhorn_eps=compile_key.hc_sinkhorn_eps,
|
||||
hc_post_mult_value=compile_key.hc_post_mult_value,
|
||||
sinkhorn_repeat=compile_key.sinkhorn_repeat,
|
||||
residual_out=residual_out,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=compile_key.norm_eps,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gemm_out_mul: Any,
|
||||
gemm_out_sqrsum: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
residual: Any,
|
||||
post_mix: Any,
|
||||
comb_mix: Any,
|
||||
layer_input: Any,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
*,
|
||||
residual_out: Any | None = None,
|
||||
norm_weight: Any | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> TileLangLaunchSpec:
|
||||
is_broadcast = residual_out is not None
|
||||
use_norm_weight = norm_weight is not None
|
||||
n_splits = gemm_out_mul.shape[0]
|
||||
if is_broadcast:
|
||||
assert residual_out is not None
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual_out.shape[-2]
|
||||
else:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
compile_key = self.dispatch(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=n_splits,
|
||||
is_broadcast=is_broadcast,
|
||||
use_norm_weight=use_norm_weight,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
broadcast_norm_eps=norm_eps,
|
||||
)
|
||||
return (
|
||||
(compile_key,),
|
||||
self._kernel_args(
|
||||
compile_key,
|
||||
gemm_out_mul=gemm_out_mul,
|
||||
gemm_out_sqrsum=gemm_out_sqrsum,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
residual=residual,
|
||||
residual_out=residual_out,
|
||||
post_mix=post_mix,
|
||||
comb_mix=comb_mix,
|
||||
layer_input=layer_input,
|
||||
norm_weight=norm_weight,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MhcPostTileLangKernel(VllmTileLangJitKernel["MhcPostTileLangKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return mhc_post_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult, hc_mult
|
||||
)
|
||||
residual = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
layer_input = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hidden_size
|
||||
)
|
||||
out = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
return dict(
|
||||
comb_mix=comb_mix,
|
||||
residual=residual,
|
||||
post_mix=post_mix,
|
||||
layer_input=layer_input,
|
||||
out=out,
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
comb_mix: Any,
|
||||
residual: Any,
|
||||
post_mix: Any,
|
||||
layer_input: Any,
|
||||
out: Any,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
return (), (
|
||||
comb_mix,
|
||||
residual,
|
||||
post_mix,
|
||||
layer_input,
|
||||
out,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
)
|
||||
|
||||
|
||||
class MhcFusedTileLangKernel(
|
||||
VllmTileLangJitKernel["MhcFusedTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
n_splits: int
|
||||
tile_n: int
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return mhc_fused_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> CompileKey:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
return self.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=n_splits,
|
||||
tile_n=tile_n,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> list[CompileKey]:
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
num_tokens=WarmupIntRange(1, max_tokens + 1),
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
_when=lambda *, num_tokens: num_tokens <= 16,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1 if compile_key.tile_n == 2 else 8
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
comb_mix = make_tilelang_warmup_tensor(
|
||||
torch.float32, num_tokens, hc_mult, hc_mult
|
||||
)
|
||||
residual_in = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
post_mix = make_tilelang_warmup_tensor(torch.float32, num_tokens, hc_mult)
|
||||
x_in = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
|
||||
weight_t = make_tilelang_warmup_tensor(
|
||||
torch.float32, hc_mult3, hc_mult, hidden_size
|
||||
)
|
||||
return dict(
|
||||
comb_mix=comb_mix,
|
||||
residual_in=residual_in,
|
||||
post_mix=post_mix,
|
||||
x_in=x_in,
|
||||
weight_t=weight_t,
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
hc_mult3=hc_mult3,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
comb_mix: Any,
|
||||
residual_in: Any,
|
||||
post_mix: Any,
|
||||
x_in: Any,
|
||||
weight_t: Any,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_mult3: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
num_tokens = residual_in.shape[0]
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
yp_out = residual_in.new_empty(
|
||||
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
|
||||
)
|
||||
rp_out = residual_in.new_empty((n_splits, num_tokens), dtype=torch.float32)
|
||||
residual_out = residual_in.new_empty(residual_in.shape)
|
||||
return (
|
||||
(),
|
||||
(
|
||||
comb_mix,
|
||||
residual_in,
|
||||
post_mix,
|
||||
x_in,
|
||||
weight_t,
|
||||
yp_out,
|
||||
rp_out,
|
||||
residual_out,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
),
|
||||
dict(tile_n=tile_n, split_k=n_splits),
|
||||
(yp_out, rp_out, residual_out),
|
||||
)
|
||||
|
||||
|
||||
class HcHeadFusedTileLangKernel(
|
||||
VllmTileLangJitKernel["HcHeadFusedTileLangKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
hc_mult: int
|
||||
rms_eps: float
|
||||
hc_eps: float
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
return hc_head_fuse_tilelang
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
rms_eps=rms_eps,
|
||||
hc_eps=hc_eps,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
rms_eps=rms_eps,
|
||||
hc_eps=hc_eps,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
residual = make_tilelang_warmup_tensor(
|
||||
torch.bfloat16, num_tokens, hc_mult, hidden_size
|
||||
)
|
||||
fn = make_tilelang_warmup_tensor(torch.float32, hc_mult, hc_mult * hidden_size)
|
||||
hc_scale = make_tilelang_warmup_tensor(torch.float32, 1)
|
||||
hc_base = make_tilelang_warmup_tensor(torch.float32, hc_mult)
|
||||
out = make_tilelang_warmup_tensor(torch.bfloat16, num_tokens, hidden_size)
|
||||
return dict(
|
||||
residual=residual,
|
||||
fn=fn,
|
||||
hc_scale=hc_scale,
|
||||
hc_base=hc_base,
|
||||
out=out,
|
||||
hidden_size=hidden_size,
|
||||
rms_eps=compile_key.rms_eps,
|
||||
hc_eps=compile_key.hc_eps,
|
||||
hc_mult=hc_mult,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
residual: Any,
|
||||
fn: Any,
|
||||
hc_scale: Any,
|
||||
hc_base: Any,
|
||||
out: Any,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
return (), (
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL = HcPrenormGemmTileLangKernel()
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL = MhcPreBigFuseTileLangKernel()
|
||||
_MHC_POST_TILELANG_KERNEL = MhcPostTileLangKernel()
|
||||
_MHC_FUSED_TILELANG_KERNEL = MhcFusedTileLangKernel()
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL = HcHeadFusedTileLangKernel()
|
||||
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up DeepSeek V4 mHC TileLang kernels before serving requests.
|
||||
|
||||
Ported from lucifer1004/vllm-jasl with the two env-var knobs removed
|
||||
(`VLLM_ENABLE_DEEPSEEK_V4_MHC_WARMUP`, `VLLM_DEEPSEEK_V4_MHC_WARMUP_TOKEN_SIZES`).
|
||||
Gating is intrinsic: non-DSv4 models and layers without hc_* attributes
|
||||
return early, so the warmup is a no-op except where it's needed.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tracing import instrument
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_AUTO_WARMUP_MAX_TOKENS = 16_384
|
||||
_DEFAULT_TOKEN_SIZE_CANDIDATES = (
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
8,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
4096,
|
||||
8192,
|
||||
16_384,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_token_sizes(
|
||||
token_sizes: Iterable[int],
|
||||
*,
|
||||
max_tokens: int,
|
||||
) -> list[int]:
|
||||
return sorted({size for size in token_sizes if 1 <= size <= max_tokens})
|
||||
|
||||
|
||||
def _select_mhc_warmup_token_sizes(
|
||||
*,
|
||||
max_tokens: int,
|
||||
cudagraph_capture_sizes: list[int],
|
||||
) -> list[int]:
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
|
||||
max_auto_tokens = min(max_tokens, _AUTO_WARMUP_MAX_TOKENS)
|
||||
candidates = list(_DEFAULT_TOKEN_SIZE_CANDIDATES)
|
||||
candidates.extend(cudagraph_capture_sizes)
|
||||
candidates.append(max_auto_tokens)
|
||||
return _normalize_token_sizes(candidates, max_tokens=max_auto_tokens)
|
||||
|
||||
|
||||
def _find_first_mhc_layer(model: torch.nn.Module) -> torch.nn.Module | None:
|
||||
for module in model.modules():
|
||||
if module.__class__.__name__ != "DeepseekV4DecoderLayer":
|
||||
continue
|
||||
if all(
|
||||
hasattr(module, attr)
|
||||
for attr in (
|
||||
"hc_pre",
|
||||
"hc_post",
|
||||
"hc_attn_fn",
|
||||
"hc_attn_scale",
|
||||
"hc_attn_base",
|
||||
"hc_ffn_fn",
|
||||
"hc_ffn_scale",
|
||||
"hc_ffn_base",
|
||||
)
|
||||
):
|
||||
return module
|
||||
return None
|
||||
|
||||
|
||||
def _find_deepseek_v4_model(model: torch.nn.Module) -> torch.nn.Module | None:
|
||||
for module in model.modules():
|
||||
if module.__class__.__name__ != "DeepseekV4Model":
|
||||
continue
|
||||
if all(
|
||||
hasattr(module, attr)
|
||||
for attr in ("hc_head_fn", "hc_head_scale", "hc_head_base")
|
||||
):
|
||||
return module
|
||||
return None
|
||||
|
||||
|
||||
def _warmup_layer_mhc(
|
||||
layer: torch.nn.Module,
|
||||
token_sizes: list[int],
|
||||
) -> None:
|
||||
max_tokens = max(token_sizes)
|
||||
hidden_size = int(layer.hidden_size)
|
||||
hc_mult = int(layer.hc_mult)
|
||||
device = layer.hc_attn_fn.device
|
||||
residual = torch.zeros(
|
||||
max_tokens,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for size in token_sizes:
|
||||
residual_slice = residual[:size]
|
||||
for fn, scale, base in (
|
||||
(layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base),
|
||||
(layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base),
|
||||
):
|
||||
layer_input, post_mix, comb_mix = layer.hc_pre(
|
||||
residual_slice,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
)
|
||||
layer.hc_post(layer_input, residual_slice, post_mix, comb_mix)
|
||||
|
||||
|
||||
def _warmup_hc_head(
|
||||
model: torch.nn.Module,
|
||||
token_sizes: list[int],
|
||||
) -> None:
|
||||
# Upstream a8887c208 ("[DSV4] aiter mhc support (ROCm)") refactored
|
||||
# ``hc_head`` from a free function into the ``HCHeadOp`` CustomOp
|
||||
# instance attached to the model as ``hc_head_op``. We call through
|
||||
# that instance so the warmup exercises the same dispatched
|
||||
# implementation as the inference path.
|
||||
hc_head_op = getattr(model, "hc_head_op", None)
|
||||
if hc_head_op is None:
|
||||
return
|
||||
|
||||
max_tokens = max(token_sizes)
|
||||
hidden_size = int(model.config.hidden_size)
|
||||
hc_mult = int(model.hc_mult)
|
||||
device = model.hc_head_fn.device
|
||||
hidden_states = torch.zeros(
|
||||
max_tokens,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for size in token_sizes:
|
||||
hc_head_op(
|
||||
hidden_states[:size],
|
||||
model.hc_head_fn,
|
||||
model.hc_head_scale,
|
||||
model.hc_head_base,
|
||||
model.rms_norm_eps,
|
||||
model.hc_eps,
|
||||
)
|
||||
|
||||
|
||||
@instrument(span_name="DeepSeek V4 mHC warmup")
|
||||
def deepseek_v4_mhc_warmup(
|
||||
model: torch.nn.Module,
|
||||
*,
|
||||
max_tokens: int,
|
||||
cudagraph_capture_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
# Cheap model-type gate before walking ``model.modules()``. The class
|
||||
# walk below is O(num_layers) and shows up in startup time on very
|
||||
# large checkpoints; bail out for any model that is not DeepSeek V4.
|
||||
config = getattr(model, "config", None)
|
||||
model_type = getattr(config, "model_type", None) if config is not None else None
|
||||
if model_type is not None and model_type not in (
|
||||
"deepseek_v4",
|
||||
"deepseek_v41",
|
||||
):
|
||||
return
|
||||
|
||||
layer = _find_first_mhc_layer(model)
|
||||
if layer is None:
|
||||
return
|
||||
|
||||
device = layer.hc_attn_fn.device
|
||||
if device.type != "cuda":
|
||||
return
|
||||
|
||||
deepseek_model = _find_deepseek_v4_model(model)
|
||||
token_sizes = _select_mhc_warmup_token_sizes(
|
||||
max_tokens=max_tokens,
|
||||
cudagraph_capture_sizes=cudagraph_capture_sizes or [],
|
||||
)
|
||||
if not token_sizes:
|
||||
return
|
||||
|
||||
started = time.perf_counter()
|
||||
logger.info(
|
||||
"Warming up DeepSeek V4 mHC TileLang kernels for token sizes: %s",
|
||||
token_sizes,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
_warmup_layer_mhc(layer, token_sizes)
|
||||
if deepseek_model is not None:
|
||||
_warmup_hc_head(deepseek_model, token_sizes)
|
||||
torch.accelerator.synchronize()
|
||||
logger.info(
|
||||
"DeepSeek V4 mHC TileLang warmup finished in %.2f seconds.",
|
||||
time.perf_counter() - started,
|
||||
)
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compile-only helpers for TileLang JIT warmup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Iterator, Mapping, MutableMapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from functools import wraps
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
# (kernel_args, launch_args), optionally followed by keywords and a result.
|
||||
TileLangLaunchSpec: TypeAlias = (
|
||||
tuple[tuple[Any, ...], tuple[Any, ...]]
|
||||
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None]
|
||||
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None, Any]
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _quiet_tilelang_warmup_logs() -> Iterator[None]:
|
||||
loggers = [
|
||||
logging.getLogger("tilelang.cache.kernel_cache"),
|
||||
logging.getLogger("tilelang.jit.kernel"),
|
||||
]
|
||||
previous_levels = [logger.level for logger in loggers]
|
||||
try:
|
||||
for logger in loggers:
|
||||
logger.setLevel(logging.ERROR)
|
||||
yield
|
||||
finally:
|
||||
for logger, level in zip(loggers, previous_levels):
|
||||
logger.setLevel(level)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TileLangWarmupTensor:
|
||||
"""Minimal tensor-like object accepted by TileLang compile().
|
||||
|
||||
TileLang builds its cache key and TIR from tensor dtype, shape and stride.
|
||||
This object deliberately has no storage, so compile-only warmup does not
|
||||
allocate GPU memory and does not launch the kernel.
|
||||
"""
|
||||
|
||||
dtype: torch.dtype
|
||||
shape: tuple[int, ...] = (1,)
|
||||
strides: tuple[int, ...] | None = field(default=None)
|
||||
|
||||
def stride(self) -> tuple[int, ...]:
|
||||
if self.strides is not None:
|
||||
return self.strides
|
||||
|
||||
strides: list[int] = []
|
||||
stride = 1
|
||||
for size in reversed(self.shape):
|
||||
strides.append(stride)
|
||||
stride *= size
|
||||
return tuple(reversed(strides))
|
||||
|
||||
def new_empty(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype | None = None,
|
||||
device: Any = None,
|
||||
) -> TileLangWarmupTensor:
|
||||
return TileLangWarmupTensor(dtype or self.dtype, shape)
|
||||
|
||||
|
||||
def make_tilelang_warmup_tensor(
|
||||
dtype: torch.dtype,
|
||||
*shape: int,
|
||||
strides: tuple[int, ...] | None = None,
|
||||
) -> TileLangWarmupTensor:
|
||||
return TileLangWarmupTensor(dtype=dtype, shape=tuple(shape), strides=strides)
|
||||
|
||||
|
||||
def _tilelang_call_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
|
||||
call_kwargs = dict(kwargs)
|
||||
tune_params = call_kwargs.pop("__tune_params", {})
|
||||
if isinstance(tune_params, dict):
|
||||
call_kwargs.update(tune_params)
|
||||
return call_kwargs
|
||||
|
||||
|
||||
def compile_tilelang(jit_impl: Any, *args: Any, **kwargs: Any) -> None:
|
||||
"""Compile one TileLang specialization and populate its call cache.
|
||||
|
||||
TileLang's ``compile()`` materializes the kernel without launching it.
|
||||
We also store the compiled kernel in ``_kernel_cache`` using the same
|
||||
parsed key as ``__call__`` so runtime does not report a cache miss for an
|
||||
already materialized specialization.
|
||||
"""
|
||||
|
||||
with _quiet_tilelang_warmup_logs():
|
||||
compiled = jit_impl.compile(*args, **kwargs)
|
||||
func = getattr(jit_impl, "func", None)
|
||||
parse_args = getattr(func, "parse_args", None)
|
||||
cache = getattr(jit_impl, "_kernel_cache", None)
|
||||
if not callable(parse_args) or not isinstance(cache, MutableMapping):
|
||||
return
|
||||
|
||||
key, _ = parse_args(*args, **_tilelang_call_kwargs(kwargs))
|
||||
cache[key] = compiled
|
||||
|
||||
|
||||
class VllmTileLangJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""TileLang owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
_warming_key: CompileKeyT | None = None
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
|
||||
"""Return runtime-shaped inputs that reproduce one compile key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
self._warming_key = compile_key
|
||||
try:
|
||||
cast(Callable[..., Any], self)(**self.warmup_inputs(compile_key))
|
||||
finally:
|
||||
self._warming_key = None
|
||||
|
||||
def launch(
|
||||
self,
|
||||
jit_impl: Any,
|
||||
args: tuple[Any, ...],
|
||||
kwargs: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
call_kwargs = dict(kwargs or {})
|
||||
if self._warming_key is not None:
|
||||
return compile_tilelang(jit_impl, *args, **call_kwargs)
|
||||
return jit_impl(*args, **call_kwargs)
|
||||
|
||||
|
||||
def kernel_launcher(
|
||||
call_fn: Callable[..., TileLangLaunchSpec],
|
||||
) -> Callable[..., Any]:
|
||||
"""Launch TileLang from declarative kernel and runtime argument tuples."""
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmTileLangJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
launch_spec = call_fn(self, *args, **kwargs)
|
||||
kernel_args, launch_args = launch_spec[:2]
|
||||
if self._warming_key is not None and kernel_args not in (
|
||||
(),
|
||||
(self._warming_key,),
|
||||
):
|
||||
raise RuntimeError(
|
||||
"TileLang warmup inputs produced a different compile key: "
|
||||
f"expected {self._warming_key!r}, got kernel arguments "
|
||||
f"{kernel_args!r}"
|
||||
)
|
||||
launch_kwargs = launch_spec[2] if len(launch_spec) > 2 else None
|
||||
output = launch_spec[3] if len(launch_spec) > 3 else None
|
||||
result = self.launch(self.kernel(*kernel_args), launch_args, launch_kwargs)
|
||||
return output if len(launch_spec) > 3 else result
|
||||
|
||||
return wrapper
|
||||
@@ -17,9 +17,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
|
||||
deepseek_v4_mhc_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
@@ -197,15 +194,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
compilation_config = worker.vllm_config.compilation_config
|
||||
cudagraph_capture_sizes = list(compilation_config.cudagraph_capture_sizes or [])
|
||||
|
||||
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
|
||||
# layer per token; warm them across token sizes first so the first real
|
||||
# request doesn't pay JIT cost. No-op for non-DSv4 models (gated inside).
|
||||
deepseek_v4_mhc_warmup(
|
||||
worker.get_model(),
|
||||
max_tokens=worker.scheduler_config.max_num_batched_tokens,
|
||||
cudagraph_capture_sizes=cudagraph_capture_sizes,
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
|
||||
@@ -1210,6 +1210,53 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
include_pre_gemm_splits = is_deep_gemm_supported()
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.hc_mult,
|
||||
use_norm_weight=True,
|
||||
include_pre_gemm_splits=include_pre_gemm_splits,
|
||||
include_broadcast_splits=(
|
||||
get_pp_group().is_first_rank and extract_layer_index(prefix) == 0
|
||||
),
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=(
|
||||
self.attn_norm.variance_epsilon,
|
||||
self.ffn_norm.variance_epsilon,
|
||||
),
|
||||
broadcast_norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
if not include_pre_gemm_splits:
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.hc_mult,
|
||||
n_out=self.hc_mult * (2 + self.hc_mult),
|
||||
)
|
||||
_MHC_POST_TILELANG_KERNEL.register_warmup(
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.hc_mult,
|
||||
)
|
||||
_MHC_FUSED_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.hc_mult,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
@@ -1402,6 +1449,18 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
self._mtp_hidden_buffer = None
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup and get_pp_group().is_last_rank:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL,
|
||||
)
|
||||
|
||||
_HC_HEAD_FUSED_TILELANG_KERNEL.register_warmup(
|
||||
hidden_size=config.hidden_size,
|
||||
hc_mult=self.hc_mult,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_eps=self.hc_eps,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
|
||||
@@ -403,6 +403,50 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
self.mhc_post_op = MHCPostOp()
|
||||
self.mhc_fused_post_pre_op = MHCFusedPostPreOp()
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
)
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
include_pre_gemm_splits = is_deep_gemm_supported()
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.n,
|
||||
use_norm_weight=True,
|
||||
include_pre_gemm_splits=include_pre_gemm_splits,
|
||||
include_broadcast_splits=False,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.mhc_post_mult_value,
|
||||
sinkhorn_repeat=self.mhc_sinkhorn_iterations,
|
||||
norm_eps=(
|
||||
self.input_layernorm.variance_epsilon,
|
||||
self.post_attention_layernorm.variance_epsilon,
|
||||
),
|
||||
)
|
||||
if not include_pre_gemm_splits:
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.n,
|
||||
n_out=self.n * (2 + self.n),
|
||||
)
|
||||
_MHC_POST_TILELANG_KERNEL.register_warmup(
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.n,
|
||||
)
|
||||
_MHC_FUSED_TILELANG_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
hidden_size=self.hidden_size,
|
||||
hc_mult=self.n,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user