[ROCm][DSV4] Enable Tilelang MHC replacing torch/triton mhc (#43679)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -5,6 +5,88 @@ import torch
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
def _tilelang_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
tile_n: int = 12,
|
||||
n_thr: int = 512,
|
||||
n_splits: int = 1,
|
||||
) -> None:
|
||||
from vllm._tilelang_ops import (
|
||||
hc_prenorm_gemm_block_m_tilelang,
|
||||
hc_prenorm_gemm_tilelang,
|
||||
)
|
||||
|
||||
assert out.shape[0] == n_splits
|
||||
assert sqrsum.shape[0] == n_splits
|
||||
assert x.shape[1] == hc_mult * hidden_size
|
||||
assert x.shape[1] % n_splits == 0
|
||||
assert (x.shape[1] // n_splits) % n_thr == 0
|
||||
use_default_config = tile_n == 12 and n_thr == 512
|
||||
if n_splits == 1 and use_default_config and x.shape[0] >= 1024:
|
||||
hc_prenorm_gemm_block_m_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if (
|
||||
n_splits == 1
|
||||
and use_default_config
|
||||
and x.shape[0] < 128
|
||||
and x.shape[1] % 1024 == 0
|
||||
):
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
1024,
|
||||
4,
|
||||
n_splits,
|
||||
)
|
||||
return
|
||||
hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
out,
|
||||
sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
fn.shape[0],
|
||||
n_thr,
|
||||
tile_n,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
|
||||
def mhc_pre_tilelang(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
@@ -80,10 +162,16 @@ def mhc_pre_tilelang(
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
if use_deep_gemm:
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
@@ -102,13 +190,24 @@ def mhc_pre_tilelang(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
residual_2d = residual_flat.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
@@ -304,16 +403,24 @@ def mhc_fused_post_pre_tilelang(
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
fma_token_threshold = 16
|
||||
if num_tokens <= fma_token_threshold:
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
use_small_fma = num_tokens <= 16
|
||||
if use_small_fma:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
if use_deep_gemm:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(
|
||||
block_k, hc_hidden_size, cdiv(num_tokens, block_m)
|
||||
)
|
||||
else:
|
||||
n_splits = 1
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
@@ -348,7 +455,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
if num_tokens <= fma_token_threshold:
|
||||
if use_small_fma:
|
||||
mhc_fused_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
@@ -375,15 +482,26 @@ def mhc_fused_post_pre_tilelang(
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
residual_cur_2d = residual_cur.view(num_tokens, hc_mult * hidden_size)
|
||||
if use_deep_gemm:
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
else:
|
||||
_tilelang_hc_prenorm_gemm(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
import torch
|
||||
|
||||
# this import will also register the custom ops
|
||||
# import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.kernels.mhc as mhc_kernels
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
|
||||
HAS_TILELANG = has_tilelang()
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre]
|
||||
@@ -85,6 +89,52 @@ class MHCPreOp(CustomOp):
|
||||
# sinkhorn_repeat,
|
||||
# )
|
||||
# else:
|
||||
if HAS_TILELANG:
|
||||
return torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
else:
|
||||
return self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return mhc_kernels.mhc_pre_torch(
|
||||
residual,
|
||||
fn,
|
||||
@@ -97,9 +147,6 @@ class MHCPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_pre is not available")
|
||||
|
||||
|
||||
# --8<-- [start:mhc_post]
|
||||
@CustomOp.register("mhc_post")
|
||||
@@ -147,6 +194,20 @@ class MHCPostOp(CustomOp):
|
||||
# comb_res_mix,
|
||||
# )
|
||||
# else:
|
||||
if HAS_TILELANG:
|
||||
return torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
else:
|
||||
return self.forward_native(x, residual, post_layer_mix, comb_res_mix)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return mhc_kernels.mhc_post_torch(
|
||||
x,
|
||||
residual,
|
||||
@@ -154,9 +215,6 @@ class MHCPostOp(CustomOp):
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_post is not available")
|
||||
|
||||
|
||||
# --8<-- [start:hc_head]
|
||||
@CustomOp.register("hc_head")
|
||||
@@ -220,17 +278,32 @@ class HCHeadOp(CustomOp):
|
||||
out = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
torch.ops.vllm.hc_head_triton(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if HAS_TILELANG:
|
||||
torch.ops.vllm.hc_head_fused_kernel_tilelang(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
torch.ops.vllm.hc_head_triton(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
@@ -290,9 +363,42 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"Hip implementation of mhc_fused_post_pre is not available"
|
||||
def forward_hip(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.mhc_fused_post_pre_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
tile_n,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user