add fused mhc_post_pre kernel (#41536)

Signed-off-by: george <george@inferact.ai>
Co-authored-by: george <george@inferact.ai>
This commit is contained in:
gnovack
2026-05-10 19:56:52 -07:00
committed by GitHub
co-authored by george
parent 879a8c3180
commit 171019ab19
3 changed files with 533 additions and 11 deletions
+343
View File
@@ -441,6 +441,131 @@ def mhc_post_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs={
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
},
)
def mhc_fused_tilelang(
comb_mix,
residual_in,
post_mix,
x_in,
weight_t,
yp_out,
rp_out,
residual_out,
hc: int,
hidden: int,
n_out: int,
n_thr: int = 256,
h_blk: int = 256,
tile_n: int = 1,
split_k: int = 1,
) -> tilelang.JITKernel:
"""Fused mhc post-mapping + pre-norm GEMM FMA"""
m = T.dynamic("num_tokens")
split_k = T.dynamic("split_k")
h = hidden
h_blk = math.gcd(hidden, h_blk)
h_per_split = h // split_k
n_tiles = n_out // tile_n
comb_mix: T.Tensor((m, hc, hc), T.float32) # type: ignore[no-redef, valid-type]
residual_in: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
post_mix: T.Tensor((m, hc), T.float32) # type: ignore[no-redef, valid-type]
x_in: T.Tensor((m, h), T.bfloat16) # type: ignore[no-redef, valid-type]
weight_t: T.Tensor((n_out, hc, h), T.float32) # type: ignore[no-redef, valid-type]
yp_out: T.Tensor((split_k, m, n_out), T.float32) # type: ignore[no-redef, valid-type]
rp_out: T.Tensor((split_k, m), T.float32) # type: ignore[no-redef, valid-type]
residual_out: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
h_iters = h_per_split // n_thr
num_warps = n_thr // 32
with T.Kernel(m, n_tiles, split_k, threads=n_thr) as (i_n, i_nt, i_ks):
tid = T.get_thread_binding()
warp_id = T.get_warp_idx()
lane = T.get_lane_idx()
s_warp = T.alloc_shared((num_warps, tile_n + 1), T.float32)
s_post = T.alloc_shared((hc,), T.float32)
s_comb = T.alloc_shared((hc, hc), T.float32)
pm = T.alloc_local((hc,), T.float32)
cm = T.alloc_local((hc, hc), T.float32)
acc = T.alloc_local((tile_n,), T.float32)
sqr = T.alloc_local((1,), T.float32)
new_r = T.alloc_local((hc,), T.float32)
T.clear(acc)
T.clear(sqr)
h_split_start = i_ks * h_per_split
T.pdl_sync()
T.copy(post_mix[i_n, 0], s_post)
T.copy(comb_mix[i_n, 0, 0], s_comb)
for j in T.unroll(hc):
pm[j] = s_post[j]
for j in T.unroll(hc):
for k in T.unroll(hc):
cm[k, j] = s_comb[k, j]
# Each thread owns h_iters elements of the k-split's h slice.
for it in T.serial(h_iters):
h_idx = h_split_start + it * n_thr + tid
# Compute new residual from layer output and past residual
for j in T.unroll(hc):
new_r[j] = pm[j] * x_in[i_n, h_idx]
for k in T.unroll(hc):
new_r[j] += cm[k, j] * residual_in[i_n, k, h_idx]
# populate residual_out and compute sqr sum
if i_nt == 0:
for j in T.unroll(hc):
residual_out[i_n, j, h_idx] = new_r[j]
sqr[0] += new_r[j] * new_r[j]
# Per-thread FMA into acc[n]
for n in T.unroll(tile_n):
for j in T.unroll(hc):
acc[n] += weight_t[i_nt * tile_n + n, j, h_idx] * new_r[j]
for n in T.unroll(tile_n):
acc[n] = T.warp_reduce_sum(acc[n])
if i_nt == 0:
sqr[0] = T.warp_reduce_sum(sqr[0])
# Cross-warp reduce via shared mem
if lane == 0:
for n in T.unroll(tile_n):
s_warp[warp_id, n] = acc[n]
if i_nt == 0:
s_warp[warp_id, tile_n] = sqr[0]
T.sync_threads()
# Warp 0 does the final cross-warp sum and writes outputs
if warp_id == 0:
if lane < tile_n:
v = T.alloc_var(T.float32, init=0.0)
for w in T.unroll(num_warps):
v += s_warp[w, lane]
yp_out[i_ks, i_n, i_nt * tile_n + lane] = v
if i_nt == 0 and lane == 0:
v2 = T.alloc_var(T.float32, init=0.0)
for w in T.unroll(num_warps):
v2 += s_warp[w, tile_n]
rp_out[i_ks, i_n] = v2
T.pdl_trigger()
def mhc_post(
x: torch.Tensor,
residual: torch.Tensor,
@@ -468,6 +593,218 @@ def mhc_post(
return out
def mhc_fused_post_pre(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Run one MHC post block followed by the next MHC pre block.
Returns:
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
post_mix_cur: shape (..., hc_mult, 1)
comb_mix_cur: shape (..., hc_mult, hc_mult)
layer_input_cur: shape (..., hidden_size)
"""
assert residual.dtype == torch.bfloat16
assert x.dtype == torch.bfloat16
assert post_layer_mix.dtype == torch.float32
assert comb_res_mix.dtype == torch.float32
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
outer_shape = residual.shape[:-2]
assert x.shape == (*outer_shape, hidden_size)
assert post_layer_mix.shape in (
(*outer_shape, hc_mult, 1),
(*outer_shape, hc_mult),
)
assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult)
assert fn.shape == (hc_mult3, hc_hidden_size)
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
assert n_splits in (1, 2, 4, 8)
assert hidden_size % n_splits == 0
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
x_flat = x.view(num_tokens, hidden_size)
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
fma_token_threshold = 16
if num_tokens <= fma_token_threshold:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
else:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
gemm_out_mul = torch.empty(
n_splits,
num_tokens,
hc_mult3,
dtype=torch.float32,
device=residual.device,
)
gemm_out_sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=residual.device,
)
residual_cur = torch.empty_like(residual_flat)
post_mix_cur = torch.empty(
num_tokens,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
comb_mix_cur = torch.empty(
num_tokens,
hc_mult2,
dtype=torch.float32,
device=residual.device,
)
layer_input_cur = torch.empty(
num_tokens,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
if num_tokens <= fma_token_threshold:
mhc_fused_tilelang(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
fn.view(hc_mult3, hc_mult, hidden_size),
gemm_out_mul,
gemm_out_sqrsum,
residual_cur,
hc_mult,
hidden_size,
hc_mult3,
tile_n=tile_n,
n_splits=n_splits,
)
else:
mhc_post_tilelang(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
residual_cur,
residual.shape[-2],
residual.shape[-1],
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
residual_cur.view(num_tokens, hc_mult * hidden_size),
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
return (
residual_cur.view(*outer_shape, hc_mult, hidden_size),
post_mix_cur.view(*outer_shape, hc_mult, 1),
comb_mix_cur.view(*outer_shape, hc_mult, hc_mult),
layer_input_cur.view(*outer_shape, hidden_size),
)
def _mhc_fused_post_pre_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
residual_cur = torch.empty_like(residual)
post_mix_cur = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix_cur = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input_cur = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
def _mhc_post_fake(
x: torch.Tensor,
residual: torch.Tensor,
@@ -489,6 +826,12 @@ direct_register_custom_op(
mutates_args=[],
fake_impl=_mhc_post_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre",
op_func=mhc_fused_post_pre,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_fake,
)
@tilelang.jit(
+48 -11
View File
@@ -1203,23 +1203,53 @@ class DeepseekV4DecoderLayer(nn.Module):
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
residual: torch.Tensor | None,
) -> torch.Tensor:
residual = x
x, post, comb = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
if residual is None:
# Run standalone hc_pre on first layer
residual = x
x, post_mix, res_mix = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
else:
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x
return x, residual, post_mix, res_mix
@support_torch_compile
@@ -1369,12 +1399,19 @@ class DeepseekV4Model(nn.Module):
if self.use_mega_moe:
input_ids = input_ids.to(torch.int64)
residual, post_mix, res_mix = None, None, None
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
input_ids,
post_mix,
res_mix,
residual,
)
else:
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})