[ROCm][DSV4] Enable Tilelang MHC replacing torch/triton mhc (#43679)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-05-28 07:05:28 +00:00
committed by GitHub
parent e1814f822d
commit 0ba46d4b11
13 changed files with 715 additions and 98 deletions
+144 -26
View File
@@ -5,6 +5,88 @@ import torch
from vllm.utils.torch_utils import direct_register_custom_op
def _torch_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
) -> None:
assert out.shape[0] == 1
assert sqrsum.shape[0] == 1
x_float = x.float()
out[0].copy_(x_float @ fn.t())
sqrsum[0].copy_(x_float.square().sum(dim=-1))
def _tilelang_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
hidden_size: int,
hc_mult: int,
tile_n: int = 12,
n_thr: int = 512,
n_splits: int = 1,
) -> None:
from vllm._tilelang_ops import (
hc_prenorm_gemm_block_m_tilelang,
hc_prenorm_gemm_tilelang,
)
assert out.shape[0] == n_splits
assert sqrsum.shape[0] == n_splits
assert x.shape[1] == hc_mult * hidden_size
assert x.shape[1] % n_splits == 0
assert (x.shape[1] // n_splits) % n_thr == 0
use_default_config = tile_n == 12 and n_thr == 512
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
hc_prenorm_gemm_block_m_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
2,
)
return
if (
n_splits == 1
and use_default_config
and x.shape[0] < 128
and x.shape[1] % 1024 == 0
):
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
1024,
4,
n_splits,
)
return
hc_prenorm_gemm_tilelang(
x,
fn,
out,
sqrsum,
hidden_size,
hc_mult,
fn.shape[0],
n_thr,
tile_n,
n_splits,
)
def mhc_pre_tilelang(
residual: torch.Tensor,
fn: torch.Tensor,
@@ -80,10 +162,16 @@ def mhc_pre_tilelang(
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
# these numbers are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
if use_deep_gemm:
# these numbers are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
else:
n_splits = 1
post_mix = torch.empty(
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
@@ -102,13 +190,24 @@ def mhc_pre_tilelang(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
)
tf32_hc_prenorm_gemm(
residual_flat.view(num_tokens, hc_mult * hidden_size),
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
tf32_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
@@ -304,16 +403,24 @@ def mhc_fused_post_pre_tilelang(
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
fma_token_threshold = 16
if num_tokens <= fma_token_threshold:
from vllm.utils.deep_gemm import is_deep_gemm_supported
use_deep_gemm = is_deep_gemm_supported()
use_small_fma = num_tokens <= 16
if use_small_fma:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
else:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
if use_deep_gemm:
# these number are from deepgemm kernel impl
block_k = 64
block_m = 64
n_splits = compute_num_split(
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
)
else:
n_splits = 1
gemm_out_mul = torch.empty(
n_splits,
@@ -348,7 +455,7 @@ def mhc_fused_post_pre_tilelang(
device=residual.device,
)
if num_tokens <= fma_token_threshold:
if use_small_fma:
mhc_fused_tilelang(
comb_res_mix_flat,
residual_flat,
@@ -375,15 +482,26 @@ def mhc_fused_post_pre_tilelang(
residual.shape[-1],
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
if use_deep_gemm:
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
tf32_hc_prenorm_gemm(
residual_cur.view(num_tokens, hc_mult * hidden_size),
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
tf32_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
n_splits,
)
else:
_tilelang_hc_prenorm_gemm(
residual_cur_2d,
fn,
gemm_out_mul,
gemm_out_sqrsum,
hidden_size,
hc_mult,
)
if norm_weight is None:
mhc_pre_big_fuse_tilelang(
+126 -20
View File
@@ -3,8 +3,12 @@
import torch
# this import will also register the custom ops
# import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.kernels.mhc as mhc_kernels
from vllm.model_executor.custom_op import CustomOp
from vllm.utils.import_utils import has_tilelang
HAS_TILELANG = has_tilelang()
# --8<-- [start:mhc_pre]
@@ -85,6 +89,52 @@ class MHCPreOp(CustomOp):
# sinkhorn_repeat,
# )
# else:
if HAS_TILELANG:
return torch.ops.vllm.mhc_pre_tilelang(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
norm_weight,
norm_eps,
)
else:
return self.forward_native(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
norm_weight,
norm_eps,
)
def forward_native(
self,
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return mhc_kernels.mhc_pre_torch(
residual,
fn,
@@ -97,9 +147,6 @@ class MHCPreOp(CustomOp):
sinkhorn_repeat,
)
def forward_native(self, *args, **kwargs):
raise NotImplementedError("Native implementation of mhc_pre is not available")
# --8<-- [start:mhc_post]
@CustomOp.register("mhc_post")
@@ -147,6 +194,20 @@ class MHCPostOp(CustomOp):
# comb_res_mix,
# )
# else:
if HAS_TILELANG:
return torch.ops.vllm.mhc_post_tilelang(
x, residual, post_layer_mix, comb_res_mix
)
else:
return self.forward_native(x, residual, post_layer_mix, comb_res_mix)
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return mhc_kernels.mhc_post_torch(
x,
residual,
@@ -154,9 +215,6 @@ class MHCPostOp(CustomOp):
comb_res_mix,
)
def forward_native(self, *args, **kwargs):
raise NotImplementedError("Native implementation of mhc_post is not available")
# --8<-- [start:hc_head]
@CustomOp.register("hc_head")
@@ -220,17 +278,32 @@ class HCHeadOp(CustomOp):
out = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
)
torch.ops.vllm.hc_head_triton(
hs_flat,
hc_fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_norm_eps,
hc_eps,
hc_mult,
)
if HAS_TILELANG:
torch.ops.vllm.hc_head_fused_kernel_tilelang(
hs_flat,
hc_fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_norm_eps,
hc_eps,
hc_mult,
)
else:
torch.ops.vllm.hc_head_triton(
hs_flat,
hc_fn,
hc_scale,
hc_base,
out,
hidden_size,
rms_norm_eps,
hc_eps,
hc_mult,
)
return out.view(*outer_shape, hidden_size)
def forward_native(self, *args, **kwargs):
@@ -290,9 +363,42 @@ class MHCFusedPostPreOp(CustomOp):
norm_eps,
)
def forward_hip(self, *args, **kwargs):
raise NotImplementedError(
"Hip implementation of mhc_fused_post_pre is not available"
def forward_hip(
self,
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return torch.ops.vllm.mhc_fused_post_pre_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
tile_n,
norm_weight,
norm_eps,
)
def forward_native(self, *args, **kwargs):