[Bugfix] [ROCm] [DSV4] [Perf] Add aiter mhc support (#41946)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -0,0 +1,468 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def mhc_pre_tilelang(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
from vllm._tilelang_ops import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
num_tokens, hc_mult2, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_tilelang_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_tilelang(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm._tilelang_ops import mhc_post_tilelang as _mhc_post_kernel
|
||||
|
||||
out = torch.empty_like(residual)
|
||||
_mhc_post_kernel(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.squeeze(-1),
|
||||
x,
|
||||
out,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def mhc_fused_post_pre_tilelang(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run one MHC post block followed by the next MHC pre block.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
comb_mix_cur: shape (..., hc_mult, hc_mult)
|
||||
layer_input_cur: shape (..., hidden_size)
|
||||
"""
|
||||
|
||||
from vllm._tilelang_ops import (
|
||||
compute_num_split,
|
||||
mhc_fused_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert post_layer_mix.dtype == torch.float32
|
||||
assert comb_res_mix.dtype == torch.float32
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
assert x.shape == (*outer_shape, hidden_size)
|
||||
assert post_layer_mix.shape in (
|
||||
(*outer_shape, hc_mult, 1),
|
||||
(*outer_shape, hc_mult),
|
||||
)
|
||||
assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult)
|
||||
assert fn.shape == (hc_mult3, hc_hidden_size)
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
x_flat = x.view(num_tokens, hidden_size)
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
fma_token_threshold = 16
|
||||
if num_tokens <= fma_token_threshold:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult2,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
if num_tokens <= fma_token_threshold:
|
||||
mhc_fused_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
fn.view(hc_mult3, hc_mult, hidden_size),
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
else:
|
||||
mhc_post_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
residual_cur,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
residual_cur.view(*outer_shape, hc_mult, hidden_size),
|
||||
post_mix_cur.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix_cur.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input_cur.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_tilelang_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_cur = torch.empty_like(residual)
|
||||
post_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
|
||||
|
||||
|
||||
def _mhc_post_tilelang_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
def _hc_head_fused_kernel_tilelang(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
from vllm._tilelang_ops import hc_head_fuse_tilelang
|
||||
|
||||
hc_head_fuse_tilelang(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_tilelang",
|
||||
op_func=mhc_pre_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_tilelang_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_tilelang",
|
||||
op_func=mhc_post_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_tilelang_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_tilelang",
|
||||
op_func=mhc_fused_post_pre_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_tilelang_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_fused_kernel_tilelang",
|
||||
op_func=_hc_head_fused_kernel_tilelang,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
Reference in New Issue
Block a user