[Kernel] Add mhc_pre_big_fuse_with_norm_tilelang (#43474)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-05-25 01:19:45 +00:00
committed by GitHub
parent d0a100c87a
commit b06813e872
5 changed files with 360 additions and 91 deletions
+114 -36
View File
@@ -16,6 +16,8 @@ def mhc_pre_tilelang(
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
@@ -31,6 +33,11 @@ def mhc_pre_tilelang(
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
torch.bfloat16. When provided, RMSNorm is fused into the
layer_input write path of the big_fuse kernel.
norm_eps: epsilon for the fused RMSNorm; only consulted when
norm_weight is given.
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
@@ -40,6 +47,7 @@ def mhc_pre_tilelang(
from vllm._tilelang_ops import (
compute_num_split,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
from vllm.utils.math_utils import cdiv
@@ -60,6 +68,13 @@ def mhc_pre_tilelang(
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
if norm_weight is not None:
assert norm_weight.shape == (hidden_size,)
if norm_weight.dtype != torch.bfloat16:
norm_weight = norm_weight.to(torch.bfloat16)
if not norm_weight.is_contiguous():
norm_weight = norm_weight.contiguous()
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
@@ -95,24 +110,46 @@ def mhc_pre_tilelang(
n_splits,
)
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
norm_weight,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
)
return (
post_mix.view(*outer_shape, hc_mult, 1),
@@ -132,6 +169,8 @@ def _mhc_pre_tilelang_fake(
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
@@ -198,10 +237,16 @@ def mhc_fused_post_pre_tilelang(
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Run one MHC post block followed by the next MHC pre block.
When ``norm_weight`` is provided, the layer_input_cur output is the
RMSNorm'd activation (fused into the kernel); otherwise it is the
raw pre-norm activation as before.
Returns:
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
post_mix_cur: shape (..., hc_mult, 1)
@@ -214,6 +259,7 @@ def mhc_fused_post_pre_tilelang(
mhc_fused_tilelang,
mhc_post_tilelang,
mhc_pre_big_fuse_tilelang,
mhc_pre_big_fuse_with_norm_tilelang,
)
from vllm.utils.math_utils import cdiv
@@ -242,6 +288,13 @@ def mhc_fused_post_pre_tilelang(
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
if norm_weight is not None:
assert norm_weight.shape == (hidden_size,)
if norm_weight.dtype != torch.bfloat16:
norm_weight = norm_weight.to(torch.bfloat16)
if not norm_weight.is_contiguous():
norm_weight = norm_weight.contiguous()
assert n_splits in (1, 2, 4, 8)
assert hidden_size % n_splits == 0
@@ -332,24 +385,46 @@ def mhc_fused_post_pre_tilelang(
n_splits,
)
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
else:
mhc_pre_big_fuse_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
norm_weight,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_eps,
n_splits,
hc_mult,
)
return (
residual_cur.view(*outer_shape, hc_mult, hidden_size),
@@ -373,6 +448,9 @@ def _mhc_fused_post_pre_tilelang_fake(
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]