[ROCm][Perf] Fuse DeepSeek-V4 mHC post/pre and RMSNorm with AITER (#52737)

Signed-off-by: shen-shanshan <467638484@qq.com>
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-08-20 04:51:04 +00:00
committed by GitHub
parent cd5035379c
commit d626108b18
5 changed files with 272 additions and 10 deletions
+2 -2
View File
@@ -9,8 +9,8 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
attention + MHC CustomOp path) instead of the nvidia one;
* route the MHC head through the ``HCHeadOp`` CustomOp dispatcher (aiter /
tilelang / triton / torch) instead of calling the tilelang kernels directly,
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (False on the aiter
path, where the decoder layer already applies hc_post in-layer);
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
or TileLang fused MHC is available; False only on the torch fallback);
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping`` /
``use_mega_moe`` / ``finalize_mega_moe_weights`` do not exist in amd/model.py).
+44 -6
View File
@@ -305,6 +305,10 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
# Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm.
_AITER_MHC_FUSED_RMSNORM_SIZES = frozenset({1280, 2560, 4096, 7168})
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
@@ -384,9 +388,18 @@ class DeepseekV4DecoderLayer(nn.Module):
self.mhc_pre = MHCPreOp()
self.mhc_post = MHCPostOp()
self.mhc_fused_post_pre = MHCFusedPostPreOp()
self.use_fused_mhc = HAS_TILELANG_MHC and not (
HAS_AITER_MHC and self.hidden_size % 256 == 0
# AITER mhc kernels (pre/post/fused) require hc_mult == 4.
use_aiter_mhc = (
HAS_AITER_MHC and self.hidden_size % 256 == 0 and self.hc_mult == 4
)
# Prefer AITER fused post+pre when eligible; otherwise TileLang.
self.use_fused_mhc = use_aiter_mhc or HAS_TILELANG_MHC
# Fold attn/ffn RMSNorm into MHC only when the active backend's
# fused-rmsnorm path supports this hidden size.
if use_aiter_mhc:
self.fuse_mhc_rmsnorm = self.hidden_size in _AITER_MHC_FUSED_RMSNORM_SIZES
else:
self.fuse_mhc_rmsnorm = HAS_TILELANG_MHC and self.use_fused_mhc
def hc_pre(
self,
@@ -394,7 +407,13 @@ class DeepseekV4DecoderLayer(nn.Module):
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
):
"""Reduce HC residual streams to the next sub-layer input.
When ``norm_weight`` is set, RMSNorm is fused into the pre kernel.
"""
post_mix, res_mix, layer_input = self.mhc_pre(
residual=x,
fn=hc_fn,
@@ -405,6 +424,8 @@ class DeepseekV4DecoderLayer(nn.Module):
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
return layer_input, post_mix, res_mix
@@ -426,11 +447,20 @@ class DeepseekV4DecoderLayer(nn.Module):
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
attn_norm_weight = self.attn_norm.weight if self.fuse_mhc_rmsnorm else None
attn_norm_eps = (
self.attn_norm.variance_epsilon if self.fuse_mhc_rmsnorm else 0.0
)
if residual is None:
# Run standalone hc_pre on first layer
residual = x
x, post_mix, res_mix = self.hc_pre(
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
x,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
norm_weight=attn_norm_weight,
norm_eps=attn_norm_eps,
)
else:
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
@@ -446,11 +476,16 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
norm_weight=attn_norm_weight,
norm_eps=attn_norm_eps,
)
x = self.attn_norm(x)
if not self.fuse_mhc_rmsnorm:
x = self.attn_norm(x)
x = self.attn(positions, x, None)
ffn_norm_weight = self.ffn_norm.weight if self.fuse_mhc_rmsnorm else None
ffn_norm_eps = self.ffn_norm.variance_epsilon if self.fuse_mhc_rmsnorm else 0.0
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
x,
residual,
@@ -464,8 +499,11 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
norm_weight=ffn_norm_weight,
norm_eps=ffn_norm_eps,
)
x = self.ffn_norm(x)
if not self.fuse_mhc_rmsnorm:
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
@@ -678,7 +716,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
residual,
)
if (idx + 1) in self.aux_hidden_state_layers:
# On the unfused (aiter) path the layer already applied hc_post,
# On the unfused path the layer already applied hc_post,
# so hidden_states is the reconstructed stream; on the fused
# path reconstruct it via hc_post before averaging.
if layer.use_fused_mhc: