[Bugfix] [ROCm] [DSV4] [Perf] Add aiter mhc support (#41946)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
TJian
2026-05-13 21:43:15 +08:00
committed by GitHub
co-authored by Yongye Zhu
parent 0ddaf6dffa
commit a8887c208f
12 changed files with 1929 additions and 1042 deletions
-3
View File
@@ -21,6 +21,3 @@ timm>=1.0.17
# amd-quark: required for Quark quantization on ROCm
# To be consistent with test_quark.py
amd-quark>=0.8.99
# tilelang has to be installed for mhc module to be
# imported correctly.
tilelang==0.1.9
+58 -2
View File
@@ -3,7 +3,7 @@
import pytest
import torch
import vllm.model_executor.layers.mhc as mhc_ops # noqa: F401
import vllm.model_executor.kernels.mhc # noqa: F401
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -74,6 +74,23 @@ def mhc_post_ref(
return (x.float().unsqueeze(-2) * post_layer_mix + term2).bfloat16()
def hc_head_ref(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_eps: float,
) -> torch.Tensor:
residual_flat = residual.flatten(-2).float()
residual_norm = residual_flat * torch.rsqrt(
residual_flat.square().mean(dim=-1, keepdim=True) + rms_eps
)
pre_mix = torch.nn.functional.linear(residual_norm, fn)
pre_mix = torch.sigmoid(pre_mix * hc_scale + hc_base) + hc_eps
return torch.sum(pre_mix.unsqueeze(-1) * residual.float(), dim=-2).bfloat16()
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA required",
@@ -121,7 +138,7 @@ def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult):
residual_ref, post_mix_ref, res_mix_ref, layer_input_ref = run_ref()
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre_tilelang(
x,
residual,
post_layer_mix,
@@ -140,3 +157,42 @@ def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult):
torch.testing.assert_close(post_mix, post_mix_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(res_mix, res_mix_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(x, layer_input_ref, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm required",
)
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128])
@pytest.mark.parametrize("hidden_size", [4096, 7168])
@pytest.mark.parametrize("hc_mult", [4])
def test_hc_head_triton(num_tokens, hidden_size, hc_mult):
torch.set_default_device(DEVICE)
set_random_seed(0)
residual = torch.randn((num_tokens, hc_mult, hidden_size), dtype=torch.bfloat16)
fn = torch.randn((hc_mult, hc_mult * hidden_size), dtype=torch.float32) * 1e-4
hc_scale = torch.randn((1,), dtype=torch.float32) * 0.1
hc_base = torch.randn((hc_mult,), dtype=torch.float32) * 0.1
rms_eps = hc_eps = 1e-6
out = torch.empty((num_tokens, hidden_size), dtype=torch.bfloat16)
out.fill_(float("nan"))
result = torch.ops.vllm.hc_head_triton(
residual,
fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_eps,
hc_eps,
hc_mult,
)
assert result is None
assert not torch.isnan(out).any()
out_ref = hc_head_ref(residual, fn, hc_scale, hc_base, rms_eps, hc_eps)
torch.testing.assert_close(out, out_ref, atol=5e-2, rtol=1e-2)
+178
View File
@@ -2395,5 +2395,183 @@ class rocm_aiter_ops:
kv_cache_dtype=kv_cache_dtype,
)
@staticmethod
def mhc_pre(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from aiter.ops.mhc import mhc_pre
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
if num_tokens == 0:
return (
torch.empty(
num_tokens,
hc_mult,
1,
dtype=torch.float32,
device=residual_flat.device,
),
torch.empty(
num_tokens,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual_flat.device,
),
torch.empty(
num_tokens,
hidden_size,
dtype=torch.bfloat16,
device=residual_flat.device,
),
)
# AITER's Python wrapper allocates intermediate/output tensors without
# explicit device arguments, so run it under the residual tensor's device.
with torch.device(residual_flat.device):
post_mix, comb_mix, layer_input = mhc_pre(
residual_flat,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
@staticmethod
def hc_head(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Run hc_head through AITER mhc_pre and write the result to out."""
assert hs_flat.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
assert hs_flat.shape[-2:] == (hc_mult, hidden_size)
assert fn.shape == (hc_mult, hc_mult * hidden_size)
assert hc_scale.shape == (1,)
assert hc_base.shape == (hc_mult,)
num_tokens = hs_flat.shape[0]
if num_tokens == 0:
return
hc_mult3 = hc_mult * 2 + hc_mult * hc_mult
full_fn = torch.zeros(
hc_mult3,
hc_mult * hidden_size,
dtype=fn.dtype,
device=fn.device,
)
full_fn[:hc_mult] = fn
full_base = torch.zeros(hc_mult3, dtype=hc_base.dtype, device=hc_base.device)
full_base[:hc_mult] = hc_base
full_scale = torch.zeros(3, dtype=hc_scale.dtype, device=hc_scale.device)
full_scale[0] = hc_scale[0]
_, _, layer_input = rocm_aiter_ops.mhc_pre(
hs_flat,
full_fn,
full_scale,
full_base,
rms_eps,
hc_eps,
0.0,
1.0,
0,
)
out.copy_(layer_input)
@staticmethod
def mhc_post(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
from aiter.ops.mhc import mhc_post
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
out = torch.empty_like(residual_flat)
mhc_post(
out,
x.view(num_tokens, hidden_size),
residual_flat,
post_layer_mix.view(num_tokens, hc_mult, 1),
comb_res_mix.view(num_tokens, hc_mult, hc_mult),
)
return out.view_as(residual)
rocm_aiter_ops.register_ops_once()
+462
View File
@@ -0,0 +1,462 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from functools import cache
from typing import TYPE_CHECKING
import torch
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_tilelang
from vllm.utils.math_utils import cdiv
# tilelang is only available on CUDA platforms
if TYPE_CHECKING or current_platform.is_cuda():
if not has_tilelang():
raise ImportError(
"tilelang is required for mhc but is not installed. Install it with "
"`pip install tilelang`."
)
import tilelang
import tilelang.language as T
else:
tilelang = None # type: ignore[assignment]
T = None # type: ignore[assignment]
@cache
def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
device_props = torch.cuda.get_device_properties(0)
n_sms = device_props.multi_processor_count
split_k = n_sms // grid_size
if k is not None:
# avoid split_k for small k
num_block_k = cdiv(k, block_k)
split_k = min(split_k, num_block_k // 4)
split_k = max(split_k, 1)
return split_k
@tilelang.jit(
pass_configs={
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
},
)
def mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual,
post_mix,
comb_mix,
layer_input,
hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 16,
hc_mult: int = 4,
):
"""Deeply fused kernels, everything other than gemm & sqrsum in mHC pre block."""
num_tokens = T.dynamic("num_tokens")
hc_mult3 = hc_mult * (2 + hc_mult)
hidden_block = math.gcd(512, hidden_size)
gemm_out_mul: T.Tensor[[n_splits, num_tokens, hc_mult3], T.float32] # type: ignore[no-redef, valid-type]
gemm_out_sqrsum: T.Tensor[[n_splits, num_tokens], T.float32] # type: ignore[no-redef, valid-type]
hc_scale: T.Tensor[[3], T.float32] # type: ignore[no-redef, valid-type]
hc_base: T.Tensor[[hc_mult3], T.float32] # type: ignore[no-redef, valid-type]
residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
# outputs
post_mix: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
comb_mix: T.Tensor[[num_tokens, hc_mult * hc_mult], T.float32] # type: ignore[no-redef, valid-type]
layer_input: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
with T.Kernel(num_tokens, threads=96) as i:
T.pdl_sync()
##################################################################
# _pre_norm_fn_fwd_norm
rms = T.alloc_fragment(1, T.float32)
mixes = T.alloc_fragment(hc_mult3, T.float32)
T.clear(mixes)
rms[0] = 0
for i_split in T.serial(n_splits):
rms[0] += gemm_out_sqrsum[i_split, i]
rms[0] = T.rsqrt(rms[0] / (hc_mult * hidden_size) + rms_eps)
for j in T.Parallel(hc_mult3):
mixes[j] = 0
for i_split in T.serial(n_splits):
mixes[j] += gemm_out_mul[i_split, i, j]
mixes[j] *= rms[0]
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
T.copy(mixes, mixes_shared)
if T.get_thread_binding() < 32:
##################################################################
# _pre_split_mixes_fwd (post & comb)
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
for j in T.Parallel(hc_mult):
post_mix[i, j] = (
T.sigmoid(
mixes_shared[j + hc_mult] * hc_scale[1] + hc_base[j + hc_mult]
)
* hc_post_mult_value
)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = (
mixes_shared[j * hc_mult + k + hc_mult * 2] * hc_scale[2]
+ hc_base[j * hc_mult + k + hc_mult * 2]
)
##################################################################
# _sinkhorn_fwd
row_sum = T.alloc_fragment(hc_mult, T.float32)
col_sum = T.alloc_fragment(hc_mult, T.float32)
# comb = comb.softmax(-1) + eps
row_max = T.alloc_fragment(hc_mult, T.float32)
T.reduce_max(cm, row_max, dim=1)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = T.exp(cm[j, k] - row_max[j])
T.reduce_sum(cm, row_sum, dim=1)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = cm[j, k] / row_sum[j] + hc_sinkhorn_eps
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(cm, col_sum, dim=0)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
for _ in T.serial(sinkhorn_repeat - 1):
# comb = comb / (comb.sum(-1) + eps)
T.reduce_sum(cm, row_sum, dim=1)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = cm[j, k] / (row_sum[j] + hc_sinkhorn_eps)
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(cm, col_sum, dim=0)
for j, k in T.Parallel(hc_mult, hc_mult):
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
# save comb_mix to global memory
for j, k in T.Parallel(hc_mult, hc_mult):
comb_mix[i, j * hc_mult + k] = cm[j, k]
else:
##################################################################
# _pre_split_mixes_fwd (pre)
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
for j in T.Parallel(hc_mult):
pre_mix_shared[j] = (
T.sigmoid(
mixes_shared[j] * hc_scale[0] + hc_base[j],
)
+ hc_pre_eps
)
###################################################################
# _pre_apply_mix_fwd
for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=2):
xs = T.alloc_shared((hc_mult, hidden_block), T.float32)
xl = T.alloc_fragment((hc_mult, hidden_block), T.float32)
T.copy(residual[i, 0, i0_h * hidden_block], xs)
T.copy(xs, xl)
ol = T.alloc_fragment(hidden_block, T.float32)
T.clear(ol)
for i_hc in T.serial(hc_mult):
pre = pre_mix_shared[i_hc]
for i1_h in T.Parallel(hidden_block):
ol[i1_h] += pre * xl[i_hc, i1_h]
T.copy(ol, layer_input[i, i0_h * hidden_block])
T.pdl_trigger()
@tilelang.jit(
pass_configs={
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
},
)
def mhc_fused_tilelang(
comb_mix,
residual_in,
post_mix,
x_in,
weight_t,
yp_out,
rp_out,
residual_out,
hc: int,
hidden: int,
n_out: int,
n_thr: int = 256,
h_blk: int = 256,
tile_n: int = 1,
split_k: int = 1,
) -> tilelang.JITKernel:
"""Fused mhc post-mapping + pre-norm GEMM FMA"""
m = T.dynamic("num_tokens")
split_k = T.dynamic("split_k")
h = hidden
h_blk = math.gcd(hidden, h_blk)
h_per_split = h // split_k
n_tiles = n_out // tile_n
comb_mix: T.Tensor((m, hc, hc), T.float32) # type: ignore[no-redef, valid-type]
residual_in: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
post_mix: T.Tensor((m, hc), T.float32) # type: ignore[no-redef, valid-type]
x_in: T.Tensor((m, h), T.bfloat16) # type: ignore[no-redef, valid-type]
weight_t: T.Tensor((n_out, hc, h), T.float32) # type: ignore[no-redef, valid-type]
yp_out: T.Tensor((split_k, m, n_out), T.float32) # type: ignore[no-redef, valid-type]
rp_out: T.Tensor((split_k, m), T.float32) # type: ignore[no-redef, valid-type]
residual_out: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
h_iters = h_per_split // n_thr
num_warps = n_thr // 32
with T.Kernel(m, n_tiles, split_k, threads=n_thr) as (i_n, i_nt, i_ks):
tid = T.get_thread_binding()
warp_id = T.get_warp_idx()
lane = T.get_lane_idx()
s_warp = T.alloc_shared((num_warps, tile_n + 1), T.float32)
s_post = T.alloc_shared((hc,), T.float32)
s_comb = T.alloc_shared((hc, hc), T.float32)
pm = T.alloc_local((hc,), T.float32)
cm = T.alloc_local((hc, hc), T.float32)
acc = T.alloc_local((tile_n,), T.float32)
sqr = T.alloc_local((1,), T.float32)
new_r = T.alloc_local((hc,), T.float32)
T.clear(acc)
T.clear(sqr)
h_split_start = i_ks * h_per_split
T.pdl_sync()
T.copy(post_mix[i_n, 0], s_post)
T.copy(comb_mix[i_n, 0, 0], s_comb)
for j in T.unroll(hc):
pm[j] = s_post[j]
for j in T.unroll(hc):
for k in T.unroll(hc):
cm[k, j] = s_comb[k, j]
# Each thread owns h_iters elements of the k-split's h slice.
for it in T.serial(h_iters):
h_idx = h_split_start + it * n_thr + tid
# Compute new residual from layer output and past residual
for j in T.unroll(hc):
new_r[j] = pm[j] * x_in[i_n, h_idx]
for k in T.unroll(hc):
new_r[j] += cm[k, j] * residual_in[i_n, k, h_idx]
# populate residual_out and compute sqr sum
if i_nt == 0:
for j in T.unroll(hc):
residual_out[i_n, j, h_idx] = new_r[j]
sqr[0] += new_r[j] * new_r[j]
# Per-thread FMA into acc[n]
for n in T.unroll(tile_n):
for j in T.unroll(hc):
acc[n] += weight_t[i_nt * tile_n + n, j, h_idx] * new_r[j]
for n in T.unroll(tile_n):
acc[n] = T.warp_reduce_sum(acc[n])
if i_nt == 0:
sqr[0] = T.warp_reduce_sum(sqr[0])
# Cross-warp reduce via shared mem
if lane == 0:
for n in T.unroll(tile_n):
s_warp[warp_id, n] = acc[n]
if i_nt == 0:
s_warp[warp_id, tile_n] = sqr[0]
T.sync_threads()
# Warp 0 does the final cross-warp sum and writes outputs
if warp_id == 0:
if lane < tile_n:
v = T.alloc_var(T.float32, init=0.0)
for w in T.unroll(num_warps):
v += s_warp[w, lane]
yp_out[i_ks, i_n, i_nt * tile_n + lane] = v
if i_nt == 0 and lane == 0:
v2 = T.alloc_var(T.float32, init=0.0)
for w in T.unroll(num_warps):
v2 += s_warp[w, tile_n]
rp_out[i_ks, i_n] = v2
T.pdl_trigger()
@tilelang.jit(
pass_configs={
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
},
)
def mhc_post_tilelang(
a,
b,
c,
d,
x,
hc: int,
hidden: int,
n_thr: int = 128,
h_blk: int = 1024,
) -> tilelang.JITKernel:
# rename for shorter code
n = T.dynamic("num_tokens")
h = hidden
h_blk = math.gcd(hidden, h_blk)
a: T.Tensor((n, hc, hc), T.float32) # type: ignore[no-redef, valid-type]
b: T.Tensor((n, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
c: T.Tensor((n, hc), T.float32) # type: ignore[no-redef, valid-type]
d: T.Tensor((n, h), T.bfloat16) # type: ignore[no-redef, valid-type]
x: T.Tensor((n, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
with T.Kernel(n, threads=n_thr) as i_n:
x_shared = T.alloc_shared((hc, h_blk), T.bfloat16)
b_shared = T.alloc_shared((hc, h_blk), T.bfloat16)
d_shared = T.alloc_shared(h_blk, T.bfloat16)
x_local = T.alloc_fragment((hc, h_blk), T.float32)
b_local = T.alloc_fragment((hc, h_blk), T.float32)
d_local = T.alloc_fragment(h_blk, T.float32)
a_local = T.alloc_fragment((hc, hc), T.float32)
c_local = T.alloc_fragment(hc, T.float32)
T.pdl_sync()
T.copy(a[i_n, 0, 0], a_local)
T.copy(c[i_n, 0], c_local)
for i0_h in T.Pipelined(T.ceildiv(h, h_blk), num_stages=2):
T.copy(b[i_n, 0, i0_h * h_blk], b_shared)
T.copy(d[i_n, i0_h * h_blk], d_shared)
T.copy(b_shared, b_local)
T.copy(d_shared, d_local)
for i_hco, i1_h in T.Parallel(hc, h_blk):
x_local[i_hco, i1_h] = c_local[i_hco] * d_local[i1_h]
for i_hci in T.serial(hc):
x_local[i_hco, i1_h] += a_local[i_hci, i_hco] * b_local[i_hci, i1_h]
T.copy(x_local, x_shared)
T.copy(x_shared, x[i_n, 0, i0_h * h_blk])
T.pdl_trigger()
@tilelang.jit(
pass_configs={
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
},
)
def hc_head_fuse_tilelang(
residual,
fn,
hc_scale,
hc_base,
out,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int = 4,
n_thr: int = 128,
h_blk: int = 1024,
):
"""Two-pass fused kernel for hc_head.
Pass 1: accumulate per-token squared sum and hc_mult dot-products
(projections onto fn rows) using cross-thread reducers.
Pass 2: apply sigmoid-gated weighted sum of residual channels to output.
Avoids materialising mixes / rsqrt / pre tensors to global memory.
"""
num_tokens = T.dynamic("num_tokens")
hc_dim = hc_mult * hidden_size
h_block = math.gcd(h_blk, hidden_size)
n_h = hidden_size // h_block
residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef,valid-type]
fn: T.Tensor[[hc_mult, hc_dim], T.float32] # type: ignore[no-redef,valid-type]
hc_scale: T.Tensor[[1], T.float32] # type: ignore[no-redef,valid-type]
hc_base: T.Tensor[[hc_mult], T.float32] # type: ignore[no-redef,valid-type]
out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef,valid-type]
with T.Kernel(num_tokens, threads=n_thr) as i:
T.pdl_sync()
# ------------------------------------------------------------------
# Pass 1 – for each residual channel m_c and h_block:
# • accumulate squared sum (for RMS norm denominator)
# • accumulate hc_mult dot-products with fn rows
# ------------------------------------------------------------------
sqrsum_r = T.alloc_reducer((1,), T.float32, replication="all")
mixes_r = T.alloc_reducer((hc_mult,), T.float32, replication="all")
T.fill(sqrsum_r, 0.0)
T.fill(mixes_r, 0.0)
for m_c in T.serial(hc_mult):
for i_h in T.serial(n_h):
x_local = T.alloc_fragment(h_block, T.float32)
T.copy(residual[i, m_c, i_h * h_block], x_local)
for k in T.Parallel(h_block):
sqrsum_r[0] += x_local[k] * x_local[k]
for m_m in T.unroll(hc_mult):
fn_local = T.alloc_fragment(h_block, T.float32)
T.copy(fn[m_m, m_c * hidden_size + i_h * h_block], fn_local)
for k in T.Parallel(h_block):
mixes_r[m_m] += x_local[k] * fn_local[k]
T.finalize_reducer(sqrsum_r)
T.finalize_reducer(mixes_r)
# ------------------------------------------------------------------
# Compute pre_mix = sigmoid(mix * rsqrt * scale + base) + eps
# ------------------------------------------------------------------
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
rsqrt_val = T.alloc_fragment(1, T.float32)
rsqrt_val[0] = T.rsqrt(sqrsum_r[0] / hc_dim + rms_eps)
for m in T.Parallel(hc_mult):
pre_mix_shared[m] = (
T.sigmoid(mixes_r[m] * rsqrt_val[0] * hc_scale[0] + hc_base[m]) + hc_eps
)
# ------------------------------------------------------------------
# Pass 2 – apply_mix: pipelined weighted sum over residual channels
# ------------------------------------------------------------------
for i0_h in T.Pipelined(n_h, num_stages=2):
xs = T.alloc_shared((hc_mult, h_block), T.bfloat16)
xl = T.alloc_fragment((hc_mult, h_block), T.float32)
T.copy(residual[i, 0, i0_h * h_block], xs, disable_tma=True)
T.copy(xs, xl)
ol = T.alloc_fragment(h_block, T.float32)
T.clear(ol)
for i_hc in T.serial(hc_mult):
pre = pre_mix_shared[i_hc]
for i1_h in T.Parallel(h_block):
ol[i1_h] += pre * xl[i_hc, i1_h]
T.copy(ol, out[i, i0_h * h_block], disable_tma=True)
T.pdl_trigger()
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .aiter import *
from .tilelang import *
from .torch import *
from .triton import *
__all__ = [
"mhc_pre_cuda",
"mhc_post_cuda",
"mhc_fused_post_pre_cuda",
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
"mhc_pre_tilelang",
"mhc_post_tilelang",
"mhc_fused_post_pre_tilelang",
"hc_head_fused_tilelang",
"mhc_pre_torch",
"mhc_post_torch",
"mhc_fused_post_pre_torch",
"hc_head_fused_torch",
"mhc_pre_triton",
"mhc_post_triton",
"mhc_fused_post_pre_triton",
"hc_head_fused_triton",
]
+138
View File
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def mhc_pre_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def _mhc_pre_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
# Create empty tensors with correct shapes for meta device / shape inference
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return post_mix, comb_mix, layer_input
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_post(
x,
residual,
post_layer_mix,
comb_res_mix,
)
def _mhc_post_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(residual)
direct_register_custom_op(
op_name="mhc_pre_aiter",
op_func=mhc_pre_aiter,
mutates_args=[],
fake_impl=_mhc_pre_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
mutates_args=[],
fake_impl=_mhc_post_aiter_fake,
)
+468
View File
@@ -0,0 +1,468 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def mhc_pre_tilelang(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from vllm._tilelang_ops import (
compute_num_split,
mhc_pre_big_fuse_tilelang,
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
# these numbers are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
post_mix = torch.empty(
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
)
comb_mix = torch.empty(
num_tokens, hc_mult2, dtype=torch.float32, device=residual.device
)
layer_input = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
gemm_out_mul = torch.empty(
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
)
gemm_out_sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
)
tf32_hc_prenorm_gemm(
residual_flat.view(num_tokens, hc_mult * hidden_size),
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_flat,
post_mix,
comb_mix,
layer_input,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
def _mhc_pre_tilelang_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
# Create empty tensors with correct shapes for meta device / shape inference
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return post_mix, comb_mix, layer_input
def mhc_post_tilelang(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
from vllm._tilelang_ops import mhc_post_tilelang as _mhc_post_kernel
out = torch.empty_like(residual)
_mhc_post_kernel(
comb_res_mix,
residual,
post_layer_mix.squeeze(-1),
x,
out,
residual.shape[-2],
residual.shape[-1],
)
return out
def mhc_fused_post_pre_tilelang(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Run one MHC post block followed by the next MHC pre block.
Returns:
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
post_mix_cur: shape (..., hc_mult, 1)
comb_mix_cur: shape (..., hc_mult, hc_mult)
layer_input_cur: shape (..., hidden_size)
"""
from vllm._tilelang_ops import (
compute_num_split,
mhc_fused_tilelang,
mhc_post_tilelang,
mhc_pre_big_fuse_tilelang,
)
from vllm.utils.math_utils import cdiv
assert residual.dtype == torch.bfloat16
assert x.dtype == torch.bfloat16
assert post_layer_mix.dtype == torch.float32
assert comb_res_mix.dtype == torch.float32
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
outer_shape = residual.shape[:-2]
assert x.shape == (*outer_shape, hidden_size)
assert post_layer_mix.shape in (
(*outer_shape, hc_mult, 1),
(*outer_shape, hc_mult),
)
assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult)
assert fn.shape == (hc_mult3, hc_hidden_size)
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
assert n_splits in (1, 2, 4, 8)
assert hidden_size % n_splits == 0
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
x_flat = x.view(num_tokens, hidden_size)
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
fma_token_threshold = 16
if num_tokens <= fma_token_threshold:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
else:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
gemm_out_mul = torch.empty(
n_splits,
num_tokens,
hc_mult3,
dtype=torch.float32,
device=residual.device,
)
gemm_out_sqrsum = torch.empty(
n_splits,
num_tokens,
dtype=torch.float32,
device=residual.device,
)
residual_cur = torch.empty_like(residual_flat)
post_mix_cur = torch.empty(
num_tokens,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
comb_mix_cur = torch.empty(
num_tokens,
hc_mult2,
dtype=torch.float32,
device=residual.device,
)
layer_input_cur = torch.empty(
num_tokens,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
if num_tokens <= fma_token_threshold:
mhc_fused_tilelang(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
fn.view(hc_mult3, hc_mult, hidden_size),
gemm_out_mul,
gemm_out_sqrsum,
residual_cur,
hc_mult,
hidden_size,
hc_mult3,
tile_n=tile_n,
n_splits=n_splits,
)
else:
mhc_post_tilelang(
comb_res_mix_flat,
residual_flat,
post_layer_mix_flat,
x_flat,
residual_cur,
residual.shape[-2],
residual.shape[-1],
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
residual_cur.view(num_tokens, hc_mult * hidden_size),
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
hc_scale,
hc_base,
residual_cur,
post_mix_cur,
comb_mix_cur,
layer_input_cur,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
hc_mult,
)
return (
residual_cur.view(*outer_shape, hc_mult, hidden_size),
post_mix_cur.view(*outer_shape, hc_mult, 1),
comb_mix_cur.view(*outer_shape, hc_mult, hc_mult),
layer_input_cur.view(*outer_shape, hidden_size),
)
def _mhc_fused_post_pre_tilelang_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
residual_cur = torch.empty_like(residual)
post_mix_cur = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix_cur = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input_cur = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
def _mhc_post_tilelang_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(residual)
def _hc_head_fused_kernel_tilelang(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
if hs_flat.shape[0] == 0:
return
from vllm._tilelang_ops import hc_head_fuse_tilelang
hc_head_fuse_tilelang(
hs_flat,
fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_eps,
hc_eps,
hc_mult,
)
direct_register_custom_op(
op_name="mhc_pre_tilelang",
op_func=mhc_pre_tilelang,
mutates_args=[],
fake_impl=_mhc_pre_tilelang_fake,
)
direct_register_custom_op(
op_name="mhc_post_tilelang",
op_func=mhc_post_tilelang,
mutates_args=[],
fake_impl=_mhc_post_tilelang_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_tilelang",
op_func=mhc_fused_post_pre_tilelang,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_tilelang_fake,
)
direct_register_custom_op(
op_name="hc_head_fused_kernel_tilelang",
op_func=_hc_head_fused_kernel_tilelang,
mutates_args=["out"],
)
+106
View File
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
def mhc_pre_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
fn_flat = fn
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
mixes = torch.matmul(x, fn_flat.t())
sqrsum = x.square().sum(dim=-1, keepdim=True)
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
post_logits = (
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
)
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
2
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = torch.sum(
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
).to(torch.bfloat16)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
def mhc_post_torch(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
mixed_residual = torch.einsum(
"...ij,...ih->...jh",
comb_res_mix.to(torch.float32),
residual.to(torch.float32),
)
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
return (mixed_residual + post_term).to(residual.dtype)
+174
View File
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _rmsnorm_nw_kernel(
x_ptr,
out_ptr,
stride_row,
D,
eps,
RBLOCK: tl.constexpr,
):
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
row = tl.program_id(0)
cols = tl.arange(0, RBLOCK)
mask = cols < D
x = tl.load(
x_ptr + row * stride_row + cols,
mask=mask,
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
var = tl.sum(x * x, 0) / D
rstd = tl.rsqrt(var + eps)
out = (x * rstd).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
"""Weight-free RMSNorm over the last dimension.
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
Returns a contiguous tensor with the same shape and dtype as *x*.
"""
orig_shape = x.shape
D = orig_shape[-1]
x_2d = x.reshape(-1, D)
num_rows = x_2d.shape[0]
out = torch.empty_like(x_2d)
RBLOCK = triton.next_power_of_2(D)
_rmsnorm_nw_kernel[(num_rows,)](
x_2d,
out,
x_2d.stride(0),
D,
eps,
RBLOCK=RBLOCK,
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
)
return out.view(orig_shape)
@triton.jit
def _hc_head_reduce_store_kernel(
pre_ptr,
x_ptr,
out_ptr,
hidden_size: tl.constexpr,
hc_mult: tl.constexpr,
pre_stride_t: tl.constexpr,
pre_stride_m: tl.constexpr,
x_stride_t: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_h: tl.constexpr,
out_stride_t: tl.constexpr,
out_stride_h: tl.constexpr,
BLOCK_H: tl.constexpr,
):
token_idx = tl.program_id(0)
block_idx = tl.program_id(1)
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offsets < hidden_size
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
for mix_idx in tl.static_range(0, hc_mult):
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
tl.float32
)
x = tl.load(
x_ptr
+ token_idx * x_stride_t
+ mix_idx * x_stride_m
+ offsets * x_stride_h,
mask=mask,
other=0.0,
).to(tl.float32)
acc += pre * x
tl.store(
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
acc,
mask=mask,
)
def hc_head_reduce_triton_kernel(
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
norm_eps: float,
hc_eps: float,
) -> None:
x_flat = x.flatten(-2)
x_normed = rmsnorm_nw(x_flat, norm_eps)
mixes = F.linear(x_normed.float(), hc_fn)
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
hidden_size = x.shape[-1]
hc_mult = x.shape[-2]
block_h = 1024
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
pre,
x,
out,
hidden_size,
hc_mult,
pre.stride(0),
pre.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
)
def _hc_head_triton(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
if hs_flat.shape[0] == 0:
return
hc_head_reduce_triton_kernel(
hs_flat,
fn,
hc_scale,
hc_base,
out,
rms_eps,
hc_eps,
)
return
direct_register_custom_op(
op_name="hc_head_triton",
op_func=_hc_head_triton,
mutates_args=["out"],
)
File diff suppressed because it is too large Load Diff
+59 -38
View File
@@ -35,6 +35,12 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import (
HCHeadOp,
MHCFusedPostPreOp,
MHCPostOp,
MHCPreOp,
)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
@@ -1168,6 +1174,9 @@ class DeepseekV4DecoderLayer(nn.Module):
),
requires_grad=False,
)
self.mhc_pre = MHCPreOp()
self.mhc_post = MHCPostOp()
self.mhc_fused_post_pre = MHCFusedPostPreOp()
def hc_pre(
self,
@@ -1176,7 +1185,7 @@ class DeepseekV4DecoderLayer(nn.Module):
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
):
post_mix, res_mix, layer_input = torch.ops.vllm.mhc_pre(
post_mix, res_mix, layer_input = self.mhc_pre(
residual=x,
fn=hc_fn,
hc_scale=hc_scale,
@@ -1196,9 +1205,9 @@ class DeepseekV4DecoderLayer(nn.Module):
post: torch.Tensor,
comb: torch.Tensor,
):
return torch.ops.vllm.mhc_post(x, residual, post, comb)
return self.mhc_post(x, residual, post, comb)
def forward(
def _forward_cuda(
self,
x: torch.Tensor,
positions: torch.Tensor,
@@ -1214,7 +1223,7 @@ class DeepseekV4DecoderLayer(nn.Module):
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
else:
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
@@ -1232,7 +1241,7 @@ class DeepseekV4DecoderLayer(nn.Module):
x = self.attn_norm(x)
x = self.attn(positions, x, None)
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
post_mix,
@@ -1251,6 +1260,48 @@ class DeepseekV4DecoderLayer(nn.Module):
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
def _forward_rocm(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
residual: torch.Tensor | None,
) -> torch.Tensor:
residual = x
x, post, comb = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
)
x = self.attn_norm(x)
x = self.attn(positions, x, None)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
)
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x, None, None, None
def forward(
self,
x: torch.Tensor,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
post_mix: torch.Tensor | None,
res_mix: torch.Tensor | None,
residual: torch.Tensor | None,
) -> torch.Tensor:
if current_platform.is_rocm():
return self._forward_rocm(
x, positions, input_ids, post_mix, res_mix, residual
)
return self._forward_cuda(x, positions, input_ids, post_mix, res_mix, residual)
@support_torch_compile
class DeepseekV4Model(nn.Module):
@@ -1340,7 +1391,7 @@ class DeepseekV4Model(nn.Module):
torch.empty(1, dtype=torch.float32),
requires_grad=False,
)
self.hc_head_op = HCHeadOp()
# Pre-hc_head residual stream buffer for the MTP draft. Stable
# address (outside the cudagraph pool) so the copy_ in forward()
# refreshes it correctly across captured shapes.
@@ -1410,7 +1461,7 @@ class DeepseekV4Model(nn.Module):
res_mix,
residual,
)
else:
if layer is not None and current_platform.is_cuda():
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
if not get_pp_group().is_last_rank:
@@ -1420,7 +1471,7 @@ class DeepseekV4Model(nn.Module):
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
hidden_states = hc_head(
hidden_states = self.hc_head_op(
hidden_states,
self.hc_head_fn,
self.hc_head_scale,
@@ -1549,36 +1600,6 @@ class DeepseekV4Model(nn.Module):
layer.ffn.finalize_mega_moe_weights()
@torch.compile(backend=current_platform.simple_compile_backend)
def hc_head(
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.view(-1, hc_mult, hidden_size)
num_tokens = hs_flat.shape[0]
out = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
)
torch.ops.vllm.hc_head_fused_kernel(
hs_flat,
hc_fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_norm_eps,
hc_eps,
hc_mult,
)
return out.view(*outer_shape, hidden_size)
def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
if expert_dtype == "fp4":
# MXFP4 experts use Mxfp4MoEMethod, which registers scales as
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
@@ -40,7 +41,6 @@ from .deepseek_mtp import SharedHead
from .deepseek_v2 import get_spec_layer_idx_from_weight_name
from .deepseek_v4 import (
DeepseekV4DecoderLayer,
hc_head,
make_deepseek_v4_expert_params_mapping,
)
from .utils import maybe_prefix
@@ -119,6 +119,8 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
aux_stream_list=aux_stream_list,
)
self.hc_head_op = HCHeadOp()
def forward(
self,
input_ids: torch.Tensor,
@@ -230,7 +232,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
hidden_states = hidden_states.view(
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
)
hidden_states = hc_head(
hidden_states = self.hc_head_op(
hidden_states,
mtp_layer.hc_head_fn,
mtp_layer.hc_head_scale,