[ROCm][DSv4.1][Perf] Run the delayed mHC seams through aiter's fused Triton kernel (#58655)
Signed-off-by: Ahmed <Muhammad.Ahmed@amd.com>
This commit is contained in:
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_AITER_MHC_FUSED,
|
||||
HAS_AITER_MHC_FUSED_NORM,
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
|
||||
HAS_AITER_MHC_PRE_NORM,
|
||||
HAS_TILELANG_MHC,
|
||||
MHCFusedPostPreOp,
|
||||
@@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM),
|
||||
reason="AITER fused delayed-seam kernel required (gfx950)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
|
||||
@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"])
|
||||
def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam):
|
||||
"""One fused kernel must match the whole reference seam: the post block
|
||||
(the previous seam's post/comb gates fold ``sublayer_out`` into the
|
||||
streams), the delayed pre block (projection + Sinkhorn producing the next
|
||||
seam's post/comb/pre gates, plus the collapse with the carried pre-mix),
|
||||
and the RMSNorm folded onto the collapse.
|
||||
|
||||
``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers
|
||||
the three call shapes the model makes: a regular seam (post folded in), an
|
||||
Engram seam (post already applied), and the draft entry seam
|
||||
(``pre_mix=None`` = identity pre-mix).
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
|
||||
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
args = (
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
1.0,
|
||||
20,
|
||||
)
|
||||
|
||||
pre_mix = (
|
||||
None
|
||||
if seam == "identity_pre"
|
||||
else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
)
|
||||
post_kwargs = {}
|
||||
residual_ref = residual
|
||||
if seam == "post":
|
||||
sublayer_out = torch.randn(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
post_layer_mix = 2 * torch.rand(
|
||||
num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
comb_res_mix = torch.softmax(
|
||||
torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1
|
||||
)
|
||||
post_kwargs = dict(
|
||||
sublayer_out=sublayer_out,
|
||||
post_layer_mix=post_layer_mix,
|
||||
comb_res_mix=comb_res_mix,
|
||||
)
|
||||
residual_ref = mhc_post_torch(
|
||||
sublayer_out, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
|
||||
expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix)
|
||||
expected_layer_input = F.rms_norm(
|
||||
expected[2], (hidden_size,), norm_weight, norm_eps
|
||||
)
|
||||
|
||||
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
|
||||
*args,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
**post_kwargs,
|
||||
)
|
||||
|
||||
if seam == "post":
|
||||
torch.testing.assert_close(residual_out, residual_ref)
|
||||
else:
|
||||
# No post was requested, so the residual comes straight back.
|
||||
assert residual_out is residual
|
||||
# Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference).
|
||||
for i in (0, 1, 3):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3)
|
||||
# The collapse is bf16 and the kernel projects the once-rounded R', so
|
||||
# near-zero values need an absolute tolerance (same bound as aiter's UT).
|
||||
torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
"""The broadcast seam and a fused norm must not take the AITER path.
|
||||
"""The broadcast seam must not take the AITER path; a fused norm must not
|
||||
take the unfused AITER op.
|
||||
|
||||
Neither is expressible with AITER's pre kernels: the broadcast projects a
|
||||
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
|
||||
asserted here is the routing decision, which is what this gate owns; the
|
||||
numerics of whichever fallback it lands on are covered by
|
||||
``test_deepseek_v41_mhc_pre_delayed``.
|
||||
The broadcast projects a narrower ``x``, which no AITER pre kernel takes;
|
||||
``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the
|
||||
fused seam kernel where it exists (gfx950) and away from AITER otherwise.
|
||||
What is asserted here is the routing decision, which is what this gate
|
||||
owns; the numerics of the paths are covered by
|
||||
``test_deepseek_v41_mhc_pre_delayed`` and
|
||||
``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
@@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
|
||||
|
||||
fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter
|
||||
took_fused = False
|
||||
|
||||
def fused_spy(*spy_args, **spy_kwargs):
|
||||
nonlocal took_fused
|
||||
took_fused = True
|
||||
return fused_op(*spy_args, **spy_kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy
|
||||
)
|
||||
|
||||
x = residual[:, 0].contiguous()
|
||||
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
|
||||
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
@@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
)
|
||||
assert residual_out is broadcast_residual
|
||||
assert not took_aiter, "the broadcast seam must not reach AITER"
|
||||
assert not took_fused, "the broadcast seam must not reach AITER"
|
||||
for i in range(4):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
|
||||
# A fused norm must not reach the unfused AITER op; where the fused seam
|
||||
# kernel exists (gfx950) it routes there instead.
|
||||
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
|
||||
assert not took_aiter, "a fused norm must not reach AITER"
|
||||
assert not took_aiter, "a fused norm must not reach the unfused AITER op"
|
||||
assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
|
||||
# Positive control: the same inputs without a norm do take the AITER path,
|
||||
# so the two declines above are the gate discriminating rather than the
|
||||
|
||||
@@ -4235,6 +4235,57 @@ class rocm_aiter_ops:
|
||||
next_pre_mix.view(*outer_shape, hc_mult),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mhc_fused_post_pre_delayed_rms_norm(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":meth:`mhc_pre_delayed` with the RMSNorm of the collapse folded in.
|
||||
|
||||
Runs aiter's fused Triton kernel ``mhc_fused_post_pre_delayed_rmsnorm``;
|
||||
the returned ``layer_input`` is already normalised with ``norm_weight`` /
|
||||
``norm_eps``.
|
||||
"""
|
||||
from aiter.ops.triton.fusions.mhc_fused_post_pre_delayed_rmsnorm import (
|
||||
mhc_fused_post_pre_delayed_rmsnorm,
|
||||
)
|
||||
|
||||
_, post_mix, comb_mix, layer_input, next_pre_mix = (
|
||||
mhc_fused_post_pre_delayed_rmsnorm(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out=residual_out,
|
||||
)
|
||||
)
|
||||
return post_mix, comb_mix, layer_input, next_pre_mix
|
||||
|
||||
@staticmethod
|
||||
def mhc_fused_post_pre_delayed_prefers_unfused(num_tokens: int) -> bool:
|
||||
"""True when AITER's own heuristic favours separate post and pre.
|
||||
|
||||
@@ -13,6 +13,7 @@ __all__ = [
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_pre_delayed_aiter",
|
||||
"mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
|
||||
@@ -207,6 +207,79 @@ def _mhc_pre_delayed_aiter_fake(
|
||||
)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
|
||||
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return _mhc_pre_delayed_aiter_fake(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -335,6 +408,12 @@ direct_register_custom_op(
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
|
||||
@@ -59,9 +59,21 @@ def _aiter_mhc_op_accepts_norm(op_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _has_aiter_mhc_fused_post_pre_delayed_rms_norm() -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
# The fused delayed seam kernel is only validated on gfx950.
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused()
|
||||
HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre")
|
||||
HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre")
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM = (
|
||||
_has_aiter_mhc_fused_post_pre_delayed_rms_norm()
|
||||
)
|
||||
|
||||
|
||||
def _aiter_mhc_supported(
|
||||
@@ -388,10 +400,42 @@ class MHCPreDelayedOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
|
||||
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
|
||||
# (which projects a narrower `x`) or a fused norm. Both are handled by
|
||||
# TileLang, or by the reference when TileLang is unavailable.
|
||||
# aiter's fused delayed seam: post-mix, gate projection, and the
|
||||
# collapse with the carried pre-mix and its RMSNorm. It always applies
|
||||
# the norm and projects `residual` itself.
|
||||
if (
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
and x is None
|
||||
and norm_weight is not None
|
||||
and _aiter_mhc_supported(residual, norm_weight, supports_norm=True)
|
||||
):
|
||||
residual_out = (
|
||||
torch.empty_like(residual) if sublayer_out is not None else None
|
||||
)
|
||||
rest = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
return (residual if residual_out is None else residual_out), *rest
|
||||
# The unfused AITER delayed path drives mhc_pre_gemm_sqrsum against
|
||||
# `residual` and folds no RMSNorm, so it cannot serve the model-entry
|
||||
# broadcast (which projects a narrower `x`) or a fused norm the branch
|
||||
# above did not take. Both are handled by TileLang, or by the reference
|
||||
# when TileLang is unavailable.
|
||||
if (
|
||||
x is None
|
||||
and norm_weight is None
|
||||
|
||||
@@ -22,7 +22,11 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
|
||||
MHCPostOp,
|
||||
MHCPreDelayedOp,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -251,6 +255,10 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
)
|
||||
self.mhc_pre_delayed = MHCPreDelayedOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
# Where aiter's fused seam kernel runs (gfx950), it folds the following
|
||||
# attn_norm / ffn_norm into its collapse, so the separate norms are
|
||||
# skipped for the seams it takes.
|
||||
self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
|
||||
@staticmethod
|
||||
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
|
||||
@@ -269,6 +277,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Layer 0's attention seam projects the 2-D embedding with the folded
|
||||
# hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn.
|
||||
# The fused kernel only takes the latter, so that seam keeps the
|
||||
# separate attn_norm.
|
||||
fuse_attn_norm = self.fuse_seam_norm and not (residual is None and x.dim() == 2)
|
||||
# The reference collapses each sublayer's input with the *previous*
|
||||
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
|
||||
# for the first layer), the FFN uses this layer's attention pre-mix.
|
||||
@@ -303,6 +316,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
pre_args = (
|
||||
@@ -328,7 +343,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_mask,
|
||||
)
|
||||
residual, post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
|
||||
residual, *pre_args, pre_mix=pre_mix
|
||||
residual,
|
||||
*pre_args,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
(
|
||||
@@ -344,8 +363,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
sublayer_out=x,
|
||||
post_layer_mix=post_mix,
|
||||
comb_res_mix=res_mix,
|
||||
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
x = self.attn_norm(x)
|
||||
if not fuse_attn_norm:
|
||||
x = self.attn_norm(x)
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_all_gather(x)[: positions.shape[0]]
|
||||
@@ -368,8 +390,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
sublayer_out=x,
|
||||
post_layer_mix=post_mix,
|
||||
comb_res_mix=res_mix,
|
||||
norm_weight=self.ffn_norm.weight if self.fuse_seam_norm else None,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
if not self.fuse_seam_norm:
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix, ffn_pre
|
||||
|
||||
|
||||
Reference in New Issue
Block a user