[Bugfix] [ROCm] [DSV4] [Perf] Add aiter mhc support (#41946)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -2395,5 +2395,183 @@ class rocm_aiter_ops:
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mhc_pre(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
from aiter.ops.mhc import mhc_pre
|
||||
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
|
||||
num_tokens = residual_flat.shape[0]
|
||||
if num_tokens == 0:
|
||||
return (
|
||||
torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual_flat.device,
|
||||
),
|
||||
torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual_flat.device,
|
||||
),
|
||||
torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual_flat.device,
|
||||
),
|
||||
)
|
||||
|
||||
# AITER's Python wrapper allocates intermediate/output tensors without
|
||||
# explicit device arguments, so run it under the residual tensor's device.
|
||||
with torch.device(residual_flat.device):
|
||||
post_mix, comb_mix, layer_input = mhc_pre(
|
||||
residual_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def hc_head(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Run hc_head through AITER mhc_pre and write the result to out."""
|
||||
assert hs_flat.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
assert hs_flat.shape[-2:] == (hc_mult, hidden_size)
|
||||
assert fn.shape == (hc_mult, hc_mult * hidden_size)
|
||||
assert hc_scale.shape == (1,)
|
||||
assert hc_base.shape == (hc_mult,)
|
||||
|
||||
num_tokens = hs_flat.shape[0]
|
||||
if num_tokens == 0:
|
||||
return
|
||||
|
||||
hc_mult3 = hc_mult * 2 + hc_mult * hc_mult
|
||||
|
||||
full_fn = torch.zeros(
|
||||
hc_mult3,
|
||||
hc_mult * hidden_size,
|
||||
dtype=fn.dtype,
|
||||
device=fn.device,
|
||||
)
|
||||
full_fn[:hc_mult] = fn
|
||||
|
||||
full_base = torch.zeros(hc_mult3, dtype=hc_base.dtype, device=hc_base.device)
|
||||
full_base[:hc_mult] = hc_base
|
||||
|
||||
full_scale = torch.zeros(3, dtype=hc_scale.dtype, device=hc_scale.device)
|
||||
full_scale[0] = hc_scale[0]
|
||||
|
||||
_, _, layer_input = rocm_aiter_ops.mhc_pre(
|
||||
hs_flat,
|
||||
full_fn,
|
||||
full_scale,
|
||||
full_base,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
0.0,
|
||||
1.0,
|
||||
0,
|
||||
)
|
||||
out.copy_(layer_input)
|
||||
|
||||
@staticmethod
|
||||
def mhc_post(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from aiter.ops.mhc import mhc_post
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
out = torch.empty_like(residual_flat)
|
||||
mhc_post(
|
||||
out,
|
||||
x.view(num_tokens, hidden_size),
|
||||
residual_flat,
|
||||
post_layer_mix.view(num_tokens, hc_mult, 1),
|
||||
comb_res_mix.view(num_tokens, hc_mult, hc_mult),
|
||||
)
|
||||
return out.view_as(residual)
|
||||
|
||||
|
||||
rocm_aiter_ops.register_ops_once()
|
||||
|
||||
Reference in New Issue
Block a user