add fused mhc_post_pre kernel (#41536)
Signed-off-by: george <george@inferact.ai> Co-authored-by: george <george@inferact.ai>
This commit is contained in:
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.mhc as mhc_ops # noqa: F401
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def sinkhorn_normalize_ref(x: torch.Tensor, repeat: int, eps: float) -> torch.Tensor:
|
||||
x = x.softmax(-1) + eps
|
||||
x = x / (x.sum(-2, keepdim=True) + eps)
|
||||
for _ in range(repeat - 1):
|
||||
x = x / (x.sum(-1, keepdim=True) + eps)
|
||||
x = x / (x.sum(-2, keepdim=True) + eps)
|
||||
return x
|
||||
|
||||
|
||||
def mhc_pre_ref(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_pre.py#L303"""
|
||||
hc_mult = residual.shape[-2]
|
||||
|
||||
residual_flat = residual.flatten(-2, -1).float()
|
||||
sqrsum = residual_flat.square().sum(-1)
|
||||
mixes = (
|
||||
residual_flat @ fn.T * (sqrsum.unsqueeze(-1) / fn.shape[-1] + rms_eps).rsqrt()
|
||||
)
|
||||
|
||||
hc_scale = torch.cat(
|
||||
[
|
||||
hc_scale[0].expand(hc_mult),
|
||||
hc_scale[1].expand(hc_mult),
|
||||
hc_scale[2].expand(hc_mult * hc_mult),
|
||||
],
|
||||
)
|
||||
mixes = mixes * hc_scale + hc_base
|
||||
|
||||
pre_mix = mixes[:, :hc_mult].sigmoid().unsqueeze(-1) + hc_pre_eps
|
||||
post_mix = (
|
||||
mixes[:, hc_mult : 2 * hc_mult].sigmoid() * hc_post_mult_value
|
||||
).unsqueeze(-1)
|
||||
res_mix = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult)
|
||||
|
||||
res_mix = sinkhorn_normalize_ref(
|
||||
res_mix, repeat=sinkhorn_repeat, eps=hc_sinkhorn_eps
|
||||
)
|
||||
|
||||
layer_input = (residual * pre_mix).sum(-2).bfloat16()
|
||||
|
||||
return post_mix, res_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_ref(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""mHC post reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_post.py#L68"""
|
||||
term2 = torch.bmm(comb_res_mix.mT, residual.float())
|
||||
return (x.float().unsqueeze(-2) * post_layer_mix + term2).bfloat16()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
@pytest.mark.parametrize("hc_mult", [4])
|
||||
def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult):
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
|
||||
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16)
|
||||
residual = torch.randn((num_tokens, hc_mult, hidden_size), dtype=torch.bfloat16)
|
||||
post_layer_mix = torch.randn((num_tokens, hc_mult, 1), dtype=torch.float32)
|
||||
comb_res_mix = torch.randn((num_tokens, hc_mult, hc_mult), dtype=torch.float32)
|
||||
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
fn = (
|
||||
torch.randn((hc_mult3, hc_mult, hidden_size), dtype=torch.float)
|
||||
* 1e-4
|
||||
* (1 + torch.arange(hc_mult).mul(0.01).view(1, -1, 1))
|
||||
).flatten(1, 2)
|
||||
hc_scale = torch.randn((3,), dtype=torch.float) * 0.1
|
||||
hc_base = torch.randn((hc_mult3,), dtype=torch.float) * 0.1
|
||||
|
||||
hc_sinkhorn_eps = hc_pre_eps = rms_eps = 1e-6
|
||||
sinkhorn_repeat = 20
|
||||
hc_post_alpha = 1.0
|
||||
|
||||
def run_ref():
|
||||
residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix)
|
||||
post_mix_ref, res_mix_ref, layer_input_ref = mhc_pre_ref(
|
||||
residual_ref,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
return residual_ref, post_mix_ref, res_mix_ref, layer_input_ref
|
||||
|
||||
residual_ref, post_mix_ref, res_mix_ref, layer_input_ref = run_ref()
|
||||
|
||||
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(residual, residual_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(post_mix, post_mix_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(res_mix, res_mix_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(x, layer_input_ref, atol=1e-2, rtol=1e-2)
|
||||
@@ -441,6 +441,131 @@ def mhc_post_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
|
||||
},
|
||||
)
|
||||
def mhc_fused_tilelang(
|
||||
comb_mix,
|
||||
residual_in,
|
||||
post_mix,
|
||||
x_in,
|
||||
weight_t,
|
||||
yp_out,
|
||||
rp_out,
|
||||
residual_out,
|
||||
hc: int,
|
||||
hidden: int,
|
||||
n_out: int,
|
||||
n_thr: int = 256,
|
||||
h_blk: int = 256,
|
||||
tile_n: int = 1,
|
||||
split_k: int = 1,
|
||||
) -> tilelang.JITKernel:
|
||||
"""Fused mhc post-mapping + pre-norm GEMM FMA"""
|
||||
m = T.dynamic("num_tokens")
|
||||
split_k = T.dynamic("split_k")
|
||||
h = hidden
|
||||
h_blk = math.gcd(hidden, h_blk)
|
||||
h_per_split = h // split_k
|
||||
n_tiles = n_out // tile_n
|
||||
|
||||
comb_mix: T.Tensor((m, hc, hc), T.float32) # type: ignore[no-redef, valid-type]
|
||||
residual_in: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
post_mix: T.Tensor((m, hc), T.float32) # type: ignore[no-redef, valid-type]
|
||||
x_in: T.Tensor((m, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
weight_t: T.Tensor((n_out, hc, h), T.float32) # type: ignore[no-redef, valid-type]
|
||||
yp_out: T.Tensor((split_k, m, n_out), T.float32) # type: ignore[no-redef, valid-type]
|
||||
rp_out: T.Tensor((split_k, m), T.float32) # type: ignore[no-redef, valid-type]
|
||||
residual_out: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
|
||||
h_iters = h_per_split // n_thr
|
||||
num_warps = n_thr // 32
|
||||
|
||||
with T.Kernel(m, n_tiles, split_k, threads=n_thr) as (i_n, i_nt, i_ks):
|
||||
tid = T.get_thread_binding()
|
||||
warp_id = T.get_warp_idx()
|
||||
lane = T.get_lane_idx()
|
||||
|
||||
s_warp = T.alloc_shared((num_warps, tile_n + 1), T.float32)
|
||||
s_post = T.alloc_shared((hc,), T.float32)
|
||||
s_comb = T.alloc_shared((hc, hc), T.float32)
|
||||
|
||||
pm = T.alloc_local((hc,), T.float32)
|
||||
cm = T.alloc_local((hc, hc), T.float32)
|
||||
acc = T.alloc_local((tile_n,), T.float32)
|
||||
sqr = T.alloc_local((1,), T.float32)
|
||||
new_r = T.alloc_local((hc,), T.float32)
|
||||
|
||||
T.clear(acc)
|
||||
T.clear(sqr)
|
||||
h_split_start = i_ks * h_per_split
|
||||
|
||||
T.pdl_sync()
|
||||
|
||||
T.copy(post_mix[i_n, 0], s_post)
|
||||
T.copy(comb_mix[i_n, 0, 0], s_comb)
|
||||
|
||||
for j in T.unroll(hc):
|
||||
pm[j] = s_post[j]
|
||||
for j in T.unroll(hc):
|
||||
for k in T.unroll(hc):
|
||||
cm[k, j] = s_comb[k, j]
|
||||
|
||||
# Each thread owns h_iters elements of the k-split's h slice.
|
||||
for it in T.serial(h_iters):
|
||||
h_idx = h_split_start + it * n_thr + tid
|
||||
|
||||
# Compute new residual from layer output and past residual
|
||||
for j in T.unroll(hc):
|
||||
new_r[j] = pm[j] * x_in[i_n, h_idx]
|
||||
for k in T.unroll(hc):
|
||||
new_r[j] += cm[k, j] * residual_in[i_n, k, h_idx]
|
||||
|
||||
# populate residual_out and compute sqr sum
|
||||
if i_nt == 0:
|
||||
for j in T.unroll(hc):
|
||||
residual_out[i_n, j, h_idx] = new_r[j]
|
||||
sqr[0] += new_r[j] * new_r[j]
|
||||
|
||||
# Per-thread FMA into acc[n]
|
||||
for n in T.unroll(tile_n):
|
||||
for j in T.unroll(hc):
|
||||
acc[n] += weight_t[i_nt * tile_n + n, j, h_idx] * new_r[j]
|
||||
|
||||
for n in T.unroll(tile_n):
|
||||
acc[n] = T.warp_reduce_sum(acc[n])
|
||||
if i_nt == 0:
|
||||
sqr[0] = T.warp_reduce_sum(sqr[0])
|
||||
|
||||
# Cross-warp reduce via shared mem
|
||||
if lane == 0:
|
||||
for n in T.unroll(tile_n):
|
||||
s_warp[warp_id, n] = acc[n]
|
||||
if i_nt == 0:
|
||||
s_warp[warp_id, tile_n] = sqr[0]
|
||||
T.sync_threads()
|
||||
|
||||
# Warp 0 does the final cross-warp sum and writes outputs
|
||||
if warp_id == 0:
|
||||
if lane < tile_n:
|
||||
v = T.alloc_var(T.float32, init=0.0)
|
||||
for w in T.unroll(num_warps):
|
||||
v += s_warp[w, lane]
|
||||
yp_out[i_ks, i_n, i_nt * tile_n + lane] = v
|
||||
|
||||
if i_nt == 0 and lane == 0:
|
||||
v2 = T.alloc_var(T.float32, init=0.0)
|
||||
for w in T.unroll(num_warps):
|
||||
v2 += s_warp[w, tile_n]
|
||||
rp_out[i_ks, i_n] = v2
|
||||
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
def mhc_post(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -468,6 +593,218 @@ def mhc_post(
|
||||
return out
|
||||
|
||||
|
||||
def mhc_fused_post_pre(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run one MHC post block followed by the next MHC pre block.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
comb_mix_cur: shape (..., hc_mult, hc_mult)
|
||||
layer_input_cur: shape (..., hidden_size)
|
||||
"""
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert post_layer_mix.dtype == torch.float32
|
||||
assert comb_res_mix.dtype == torch.float32
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
assert x.shape == (*outer_shape, hidden_size)
|
||||
assert post_layer_mix.shape in (
|
||||
(*outer_shape, hc_mult, 1),
|
||||
(*outer_shape, hc_mult),
|
||||
)
|
||||
assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult)
|
||||
assert fn.shape == (hc_mult3, hc_hidden_size)
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
x_flat = x.view(num_tokens, hidden_size)
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
fma_token_threshold = 16
|
||||
if num_tokens <= fma_token_threshold:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4
|
||||
else:
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
residual_cur = torch.empty_like(residual_flat)
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult2,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
if num_tokens <= fma_token_threshold:
|
||||
mhc_fused_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
fn.view(hc_mult3, hc_mult, hidden_size),
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
tile_n=tile_n,
|
||||
n_splits=n_splits,
|
||||
)
|
||||
else:
|
||||
mhc_post_tilelang(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
post_layer_mix_flat,
|
||||
x_flat,
|
||||
residual_cur,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_cur.view(num_tokens, hc_mult * hidden_size),
|
||||
fn,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
residual_cur.view(*outer_shape, hc_mult, hidden_size),
|
||||
post_mix_cur.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix_cur.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input_cur.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_cur = torch.empty_like(residual)
|
||||
post_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input_cur = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
|
||||
|
||||
|
||||
def _mhc_post_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -489,6 +826,12 @@ direct_register_custom_op(
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre",
|
||||
op_func=mhc_fused_post_pre,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_fake,
|
||||
)
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
|
||||
@@ -1203,23 +1203,53 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
post_mix: torch.Tensor | None,
|
||||
res_mix: torch.Tensor | None,
|
||||
residual: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
if residual is None:
|
||||
# Run standalone hc_pre on first layer
|
||||
residual = x
|
||||
x, post_mix, res_mix = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
)
|
||||
else:
|
||||
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
|
||||
x = self.attn_norm(x)
|
||||
x = self.attn(positions, x, None)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
|
||||
residual = x
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
)
|
||||
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
@@ -1369,12 +1399,19 @@ class DeepseekV4Model(nn.Module):
|
||||
|
||||
if self.use_mega_moe:
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
else:
|
||||
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
Reference in New Issue
Block a user