[Bugfix] [ROCm] [DSV4] [Perf] Add aiter mhc support (#41946)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .aiter import *
|
||||
from .tilelang import *
|
||||
from .torch import *
|
||||
from .triton import *
|
||||
|
||||
__all__ = [
|
||||
"mhc_pre_cuda",
|
||||
"mhc_post_cuda",
|
||||
"mhc_fused_post_pre_cuda",
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
"mhc_pre_tilelang",
|
||||
"mhc_post_tilelang",
|
||||
"mhc_fused_post_pre_tilelang",
|
||||
"hc_head_fused_tilelang",
|
||||
"mhc_pre_torch",
|
||||
"mhc_post_torch",
|
||||
"mhc_fused_post_pre_torch",
|
||||
"hc_head_fused_torch",
|
||||
"mhc_pre_triton",
|
||||
"mhc_post_triton",
|
||||
"mhc_fused_post_pre_triton",
|
||||
"hc_head_fused_triton",
|
||||
]
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def mhc_pre_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_post(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_post_aiter_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_aiter",
|
||||
op_func=mhc_pre_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_aiter_fake,
|
||||
)
|
||||
@@ -0,0 +1,468 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def mhc_pre_tilelang(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
from vllm._tilelang_ops import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
|
||||
# these numbers are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
num_tokens, hc_mult2, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits, num_tokens, hc_mult3, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_tilelang_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_tilelang(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm._tilelang_ops import mhc_post_tilelang as _mhc_post_kernel
|
||||
|
||||
out = torch.empty_like(residual)
|
||||
_mhc_post_kernel(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.squeeze(-1),
|
||||
x,
|
||||
out,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def mhc_fused_post_pre_tilelang(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run one MHC post block followed by the next MHC pre block.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
comb_mix_cur: shape (..., hc_mult, hc_mult)
|
||||
layer_input_cur: shape (..., hidden_size)
|
||||
"""
|
||||
|
||||
from vllm._tilelang_ops import (
|
||||
compute_num_split,
|
||||
mhc_fused_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert post_layer_mix.dtype == torch.float32
|
||||
assert comb_res_mix.dtype == torch.float32
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
assert x.shape == (*outer_shape, hidden_size)
|
||||
assert post_layer_mix.shape in (
|
||||
(*outer_shape, hc_mult, 1),
|
||||
(*outer_shape, hc_mult),
|
||||
)
|
||||
assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult)
|
||||
assert fn.shape == (hc_mult3, hc_hidden_size)
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
x_flat = x.view(num_tokens, hidden_size)
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
fma_token_threshold = 16
|
||||
if num_tokens <= fma_token_threshold:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult2,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
if num_tokens <= fma_token_threshold:
|
||||
mhc_fused_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
fn.view(hc_mult3, hc_mult, hidden_size),
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
else:
|
||||
mhc_post_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
residual_cur,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
residual_cur.view(*outer_shape, hc_mult, hidden_size),
|
||||
post_mix_cur.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix_cur.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input_cur.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_tilelang_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_cur = torch.empty_like(residual)
|
||||
post_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
|
||||
|
||||
|
||||
def _mhc_post_tilelang_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
def _hc_head_fused_kernel_tilelang(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
from vllm._tilelang_ops import hc_head_fuse_tilelang
|
||||
|
||||
hc_head_fuse_tilelang(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_tilelang",
|
||||
op_func=mhc_pre_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_tilelang_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_tilelang",
|
||||
op_func=mhc_post_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_tilelang_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_tilelang",
|
||||
op_func=mhc_fused_post_pre_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_tilelang_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_fused_kernel_tilelang",
|
||||
op_func=_hc_head_fused_kernel_tilelang,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
|
||||
def mhc_pre_torch(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
fn_flat = fn
|
||||
|
||||
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
|
||||
mixes = torch.matmul(x, fn_flat.t())
|
||||
sqrsum = x.square().sum(dim=-1, keepdim=True)
|
||||
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
|
||||
|
||||
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
|
||||
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
|
||||
|
||||
post_logits = (
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
|
||||
|
||||
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
|
||||
2
|
||||
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
|
||||
layer_input = torch.sum(
|
||||
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
|
||||
).to(torch.bfloat16)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_torch(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
mixed_residual = torch.einsum(
|
||||
"...ij,...ih->...jh",
|
||||
comb_res_mix.to(torch.float32),
|
||||
residual.to(torch.float32),
|
||||
)
|
||||
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
|
||||
return (mixed_residual + post_term).to(residual.dtype)
|
||||
@@ -0,0 +1,174 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_nw_kernel(
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
stride_row,
|
||||
D,
|
||||
eps,
|
||||
RBLOCK: tl.constexpr,
|
||||
):
|
||||
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
|
||||
row = tl.program_id(0)
|
||||
cols = tl.arange(0, RBLOCK)
|
||||
mask = cols < D
|
||||
|
||||
x = tl.load(
|
||||
x_ptr + row * stride_row + cols,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
|
||||
var = tl.sum(x * x, 0) / D
|
||||
rstd = tl.rsqrt(var + eps)
|
||||
|
||||
out = (x * rstd).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
|
||||
|
||||
|
||||
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
|
||||
"""Weight-free RMSNorm over the last dimension.
|
||||
|
||||
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
|
||||
Returns a contiguous tensor with the same shape and dtype as *x*.
|
||||
"""
|
||||
orig_shape = x.shape
|
||||
D = orig_shape[-1]
|
||||
x_2d = x.reshape(-1, D)
|
||||
num_rows = x_2d.shape[0]
|
||||
|
||||
out = torch.empty_like(x_2d)
|
||||
RBLOCK = triton.next_power_of_2(D)
|
||||
|
||||
_rmsnorm_nw_kernel[(num_rows,)](
|
||||
x_2d,
|
||||
out,
|
||||
x_2d.stride(0),
|
||||
D,
|
||||
eps,
|
||||
RBLOCK=RBLOCK,
|
||||
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
|
||||
)
|
||||
return out.view(orig_shape)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_head_reduce_store_kernel(
|
||||
pre_ptr,
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
hidden_size: tl.constexpr,
|
||||
hc_mult: tl.constexpr,
|
||||
pre_stride_t: tl.constexpr,
|
||||
pre_stride_m: tl.constexpr,
|
||||
x_stride_t: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_h: tl.constexpr,
|
||||
out_stride_t: tl.constexpr,
|
||||
out_stride_h: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offsets < hidden_size
|
||||
|
||||
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
for mix_idx in tl.static_range(0, hc_mult):
|
||||
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
|
||||
tl.float32
|
||||
)
|
||||
x = tl.load(
|
||||
x_ptr
|
||||
+ token_idx * x_stride_t
|
||||
+ mix_idx * x_stride_m
|
||||
+ offsets * x_stride_h,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += pre * x
|
||||
|
||||
tl.store(
|
||||
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
|
||||
acc,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def hc_head_reduce_triton_kernel(
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> None:
|
||||
x_flat = x.flatten(-2)
|
||||
x_normed = rmsnorm_nw(x_flat, norm_eps)
|
||||
mixes = F.linear(x_normed.float(), hc_fn)
|
||||
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
|
||||
|
||||
hidden_size = x.shape[-1]
|
||||
hc_mult = x.shape[-2]
|
||||
block_h = 1024
|
||||
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
|
||||
pre,
|
||||
x,
|
||||
out,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
pre.stride(0),
|
||||
pre.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_H=block_h,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def _hc_head_triton(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
|
||||
hc_head_reduce_triton_kernel(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_triton",
|
||||
op_func=_hc_head_triton,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
+253
-997
File diff suppressed because it is too large
Load Diff
@@ -35,6 +35,12 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HCHeadOp,
|
||||
MHCFusedPostPreOp,
|
||||
MHCPostOp,
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
QuantizationMethods,
|
||||
@@ -1168,6 +1174,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.mhc_pre = MHCPreOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
self.mhc_fused_post_pre = MHCFusedPostPreOp()
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
@@ -1176,7 +1185,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
):
|
||||
post_mix, res_mix, layer_input = torch.ops.vllm.mhc_pre(
|
||||
post_mix, res_mix, layer_input = self.mhc_pre(
|
||||
residual=x,
|
||||
fn=hc_fn,
|
||||
hc_scale=hc_scale,
|
||||
@@ -1196,9 +1205,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
):
|
||||
return torch.ops.vllm.mhc_post(x, residual, post, comb)
|
||||
return self.mhc_post(x, residual, post, comb)
|
||||
|
||||
def forward(
|
||||
def _forward_cuda(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -1214,7 +1223,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
else:
|
||||
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
@@ -1232,7 +1241,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
|
||||
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
@@ -1251,6 +1260,48 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
def _forward_rocm(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
residual: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x, None, None, None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
residual: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if current_platform.is_rocm():
|
||||
return self._forward_rocm(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
|
||||
return self._forward_cuda(x, positions, input_ids, post_mix, res_mix, residual)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class DeepseekV4Model(nn.Module):
|
||||
@@ -1340,7 +1391,7 @@ class DeepseekV4Model(nn.Module):
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
# Pre-hc_head residual stream buffer for the MTP draft. Stable
|
||||
# address (outside the cudagraph pool) so the copy_ in forward()
|
||||
# refreshes it correctly across captured shapes.
|
||||
@@ -1410,7 +1461,7 @@ class DeepseekV4Model(nn.Module):
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
else:
|
||||
if layer is not None and current_platform.is_cuda():
|
||||
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
@@ -1420,7 +1471,7 @@ class DeepseekV4Model(nn.Module):
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
|
||||
hidden_states = hc_head(
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
self.hc_head_fn,
|
||||
self.hc_head_scale,
|
||||
@@ -1549,36 +1600,6 @@ class DeepseekV4Model(nn.Module):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
|
||||
@torch.compile(backend=current_platform.simple_compile_backend)
|
||||
def hc_head(
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = hs_flat.shape[0]
|
||||
out = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
torch.ops.vllm.hc_head_fused_kernel(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
|
||||
def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
if expert_dtype == "fp4":
|
||||
# MXFP4 experts use Mxfp4MoEMethod, which registers scales as
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
@@ -40,7 +41,6 @@ from .deepseek_mtp import SharedHead
|
||||
from .deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from .deepseek_v4 import (
|
||||
DeepseekV4DecoderLayer,
|
||||
hc_head,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
from .utils import maybe_prefix
|
||||
@@ -119,6 +119,8 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -230,7 +232,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
hidden_states = hidden_states.view(
|
||||
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
|
||||
)
|
||||
hidden_states = hc_head(
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
mtp_layer.hc_head_fn,
|
||||
mtp_layer.hc_head_scale,
|
||||
|
||||
Reference in New Issue
Block a user