[Perf] Intergrate Tile Kernels head_compute_mix_kernel for Deepseek-V4 (#41255)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -448,3 +448,137 @@ direct_register_custom_op(
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_fake,
|
||||
)
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
|
||||
},
|
||||
)
|
||||
def hc_head_fuse_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int = 4,
|
||||
n_thr: int = 128,
|
||||
h_blk: int = 1024,
|
||||
):
|
||||
"""Two-pass fused kernel for hc_head.
|
||||
|
||||
Pass 1: accumulate per-token squared sum and hc_mult dot-products
|
||||
(projections onto fn rows) using cross-thread reducers.
|
||||
Pass 2: apply sigmoid-gated weighted sum of residual channels to output.
|
||||
|
||||
Avoids materialising mixes / rsqrt / pre tensors to global memory.
|
||||
"""
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_dim = hc_mult * hidden_size
|
||||
h_block = math.gcd(h_blk, hidden_size)
|
||||
n_h = hidden_size // h_block
|
||||
|
||||
residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef,valid-type]
|
||||
fn: T.Tensor[[hc_mult, hc_dim], T.float32] # type: ignore[no-redef,valid-type]
|
||||
hc_scale: T.Tensor[[1], T.float32] # type: ignore[no-redef,valid-type]
|
||||
hc_base: T.Tensor[[hc_mult], T.float32] # type: ignore[no-redef,valid-type]
|
||||
out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef,valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=n_thr) as i:
|
||||
T.pdl_sync()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Pass 1 – for each residual channel m_c and h_block:
|
||||
# • accumulate squared sum (for RMS norm denominator)
|
||||
# • accumulate hc_mult dot-products with fn rows
|
||||
# ------------------------------------------------------------------
|
||||
sqrsum_r = T.alloc_reducer((1,), T.float32, replication="all")
|
||||
mixes_r = T.alloc_reducer((hc_mult,), T.float32, replication="all")
|
||||
T.fill(sqrsum_r, 0.0)
|
||||
T.fill(mixes_r, 0.0)
|
||||
|
||||
for m_c in T.serial(hc_mult):
|
||||
for i_h in T.serial(n_h):
|
||||
x_local = T.alloc_fragment(h_block, T.float32)
|
||||
T.copy(residual[i, m_c, i_h * h_block], x_local)
|
||||
|
||||
for k in T.Parallel(h_block):
|
||||
sqrsum_r[0] += x_local[k] * x_local[k]
|
||||
|
||||
for m_m in T.unroll(hc_mult):
|
||||
fn_local = T.alloc_fragment(h_block, T.float32)
|
||||
T.copy(fn[m_m, m_c * hidden_size + i_h * h_block], fn_local)
|
||||
for k in T.Parallel(h_block):
|
||||
mixes_r[m_m] += x_local[k] * fn_local[k]
|
||||
|
||||
T.finalize_reducer(sqrsum_r)
|
||||
T.finalize_reducer(mixes_r)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Compute pre_mix = sigmoid(mix * rsqrt * scale + base) + eps
|
||||
# ------------------------------------------------------------------
|
||||
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
|
||||
rsqrt_val = T.alloc_fragment(1, T.float32)
|
||||
rsqrt_val[0] = T.rsqrt(sqrsum_r[0] / hc_dim + rms_eps)
|
||||
for m in T.Parallel(hc_mult):
|
||||
pre_mix_shared[m] = (
|
||||
T.sigmoid(mixes_r[m] * rsqrt_val[0] * hc_scale[0] + hc_base[m]) + hc_eps
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Pass 2 – apply_mix: pipelined weighted sum over residual channels
|
||||
# ------------------------------------------------------------------
|
||||
for i0_h in T.Pipelined(n_h, num_stages=2):
|
||||
xs = T.alloc_shared((hc_mult, h_block), T.bfloat16)
|
||||
xl = T.alloc_fragment((hc_mult, h_block), T.float32)
|
||||
T.copy(residual[i, 0, i0_h * h_block], xs, disable_tma=True)
|
||||
T.copy(xs, xl)
|
||||
|
||||
ol = T.alloc_fragment(h_block, T.float32)
|
||||
T.clear(ol)
|
||||
for i_hc in T.serial(hc_mult):
|
||||
pre = pre_mix_shared[i_hc]
|
||||
for i1_h in T.Parallel(h_block):
|
||||
ol[i1_h] += pre * xl[i_hc, i1_h]
|
||||
|
||||
T.copy(ol, out[i, i0_h * h_block], disable_tma=True)
|
||||
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
def _hc_head_fused_kernel(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] > 0:
|
||||
hc_head_fuse_tilelang(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_fused_kernel",
|
||||
op_func=_hc_head_fused_kernel,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
|
||||
@@ -7,7 +7,6 @@ from itertools import islice
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
@@ -1456,14 +1455,25 @@ def hc_head(
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
x = hidden_states
|
||||
shape, dtype = x.size(), x.dtype
|
||||
x = x.flatten(1).float()
|
||||
rsqrt = torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_norm_eps)
|
||||
mixes = F.linear(x, hc_fn) * rsqrt
|
||||
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
|
||||
y = torch.sum(pre.unsqueeze(-1) * x.view(shape), dim=1)
|
||||
return y.to(dtype)
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = hs_flat.shape[0]
|
||||
out = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
torch.ops.vllm.hc_head_fused_kernel(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
|
||||
def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
|
||||
Reference in New Issue
Block a user