From b06813e87207e15b133e903d641e03f237d85b17 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Mon, 25 May 2026 09:19:45 +0800 Subject: [PATCH] [Kernel] Add mhc_pre_big_fuse_with_norm_tilelang (#43474) Signed-off-by: Jee Jee Li --- vllm/_tilelang_ops.py | 167 ++++++++++++++++++++ vllm/model_executor/kernels/mhc/tilelang.py | 150 +++++++++++++----- vllm/model_executor/layers/mhc.py | 10 ++ vllm/models/deepseek_v4/nvidia/model.py | 113 +++++++------ vllm/models/deepseek_v4/nvidia/mtp.py | 11 +- 5 files changed, 360 insertions(+), 91 deletions(-) diff --git a/vllm/_tilelang_ops.py b/vllm/_tilelang_ops.py index 1a073e58d1..aa742fe503 100644 --- a/vllm/_tilelang_ops.py +++ b/vllm/_tilelang_ops.py @@ -177,6 +177,173 @@ def mhc_pre_big_fuse_tilelang( T.pdl_trigger() +# Copied from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/mhc.py#L478 + + +@tilelang.jit( + pass_configs={ + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10, + }, +) +def mhc_pre_big_fuse_with_norm_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual, + post_mix, + comb_mix, + layer_input, + norm_weight, + hidden_size: int, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + norm_eps: float, + n_splits: int = 16, + hc_mult: int = 4, + gemm_last_dim: int = -1, +): + num_tokens = T.dynamic("num_tokens") + hc_mult3 = hc_mult * (2 + hc_mult) + if gemm_last_dim < 0: + gemm_last_dim = hc_mult3 + hidden_block = math.gcd(1024, hidden_size) + + gemm_out_mul: T.Tensor[[n_splits, num_tokens, gemm_last_dim], T.float32] # type: ignore[no-redef, valid-type] + gemm_out_sqrsum: T.Tensor[[n_splits, num_tokens], T.float32] # type: ignore[no-redef, valid-type] + hc_scale: T.Tensor[[3], T.float32] # type: ignore[no-redef, valid-type] + hc_base: T.Tensor[[hc_mult3], T.float32] # type: ignore[no-redef, valid-type] + residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] + post_mix: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type] + comb_mix: T.Tensor[[num_tokens, hc_mult * hc_mult], T.float32] # type: ignore[no-redef, valid-type] + layer_input: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] + norm_weight: T.Tensor[[hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] + + with T.Kernel(num_tokens, threads=96) as i: + rms = T.alloc_fragment(1, T.float32) + mixes = T.alloc_fragment(hc_mult3, T.float32) + T.clear(mixes) + rms[0] = 0 + + T.pdl_sync() + + for i_split in T.serial(n_splits): + rms[0] += gemm_out_sqrsum[i_split, i] + rms[0] = T.rsqrt(rms[0] / (hc_mult * hidden_size) + rms_eps) + for j in T.Parallel(hc_mult3): + mixes[j] = 0 + for i_split in T.serial(n_splits): + mixes[j] += gemm_out_mul[i_split, i, j] + mixes[j] *= rms[0] + mixes_shared = T.alloc_shared(hc_mult3, T.float32) + T.copy(mixes, mixes_shared) + + if T.get_thread_binding() < 32: + cm = T.alloc_fragment((hc_mult, hc_mult), T.float32) + for j in T.Parallel(hc_mult): + post_mix[i, j] = ( + T.sigmoid( + mixes_shared[j + hc_mult] * hc_scale[1] + hc_base[j + hc_mult] + ) + * hc_post_mult_value + ) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = ( + mixes_shared[j * hc_mult + k + hc_mult * 2] * hc_scale[2] + + hc_base[j * hc_mult + k + hc_mult * 2] + ) + + row_sum = T.alloc_fragment(hc_mult, T.float32) + col_sum = T.alloc_fragment(hc_mult, T.float32) + + row_max = T.alloc_fragment(hc_mult, T.float32) + T.reduce_max(cm, row_max, dim=1) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = T.exp(cm[j, k] - row_max[j]) + T.reduce_sum(cm, row_sum, dim=1) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = cm[j, k] / row_sum[j] + hc_sinkhorn_eps + + T.reduce_sum(cm, col_sum, dim=0) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps) + + for _ in T.serial(sinkhorn_repeat - 1): + T.reduce_sum(cm, row_sum, dim=1) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = cm[j, k] / (row_sum[j] + hc_sinkhorn_eps) + + T.reduce_sum(cm, col_sum, dim=0) + for j, k in T.Parallel(hc_mult, hc_mult): + cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps) + + for j, k in T.Parallel(hc_mult, hc_mult): + comb_mix[i, j * hc_mult + k] = cm[j, k] + else: + pre_mix_shared = T.alloc_shared(hc_mult, T.float32) + for j in T.Parallel(hc_mult): + pre_mix_shared[j] = ( + T.sigmoid( + mixes_shared[j] * hc_scale[0] + hc_base[j], + ) + + hc_pre_eps + ) + + # Pass 1: stash unnormalized weighted-sum output in shared memory + # as bf16 (matches the rounding that RMSNorm would see) while + # accumulating the per-position squared sum. + output_shared = T.alloc_shared(hidden_size, T.bfloat16) + sumsq_per_pos = T.alloc_fragment(hidden_block, T.float32) + T.clear(sumsq_per_pos) + + for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=3): + xs = T.alloc_shared((hc_mult, hidden_block), T.bfloat16) + xl = T.alloc_fragment((hc_mult, hidden_block), T.float32) + T.copy(residual[i, 0, i0_h * hidden_block], xs) + T.copy(xs, xl) + + ol = T.alloc_fragment(hidden_block, T.float32) + T.clear(ol) + + for i_hc in T.serial(hc_mult): + pre = pre_mix_shared[i_hc] + for i1_h in T.Parallel(hidden_block): + ol[i1_h] += pre * xl[i_hc, i1_h] + + for i1_h in T.Parallel(hidden_block): + sumsq_per_pos[i1_h] += ol[i1_h] * ol[i1_h] + output_shared[i0_h * hidden_block + i1_h] = T.bfloat16(ol[i1_h]) + + sumsq = T.alloc_fragment(1, T.float32) + T.reduce_sum(sumsq_per_pos, sumsq, dim=0) + rsqrt_norm = T.alloc_fragment(1, T.float32) + rsqrt_norm[0] = T.rsqrt(sumsq[0] / hidden_size + norm_eps) + + # Pass 2: scale by rsqrt * norm_weight and write the result to HBM. + for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=2): + w_shared = T.alloc_shared(hidden_block, T.bfloat16) + w_local = T.alloc_fragment(hidden_block, T.float32) + T.copy(norm_weight[i0_h * hidden_block], w_shared) + T.copy(w_shared, w_local) + + ol = T.alloc_fragment(hidden_block, T.float32) + for i1_h in T.Parallel(hidden_block): + ol[i1_h] = ( + output_shared[i0_h * hidden_block + i1_h] + * rsqrt_norm[0] + * w_local[i1_h] + ) + + T.copy(ol, layer_input[i, i0_h * hidden_block]) + + T.pdl_trigger() + + @tilelang.jit( pass_configs={ tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, diff --git a/vllm/model_executor/kernels/mhc/tilelang.py b/vllm/model_executor/kernels/mhc/tilelang.py index 9780d653eb..a4d05ef245 100644 --- a/vllm/model_executor/kernels/mhc/tilelang.py +++ b/vllm/model_executor/kernels/mhc/tilelang.py @@ -16,6 +16,8 @@ def mhc_pre_tilelang( hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Forward pass for mHC pre block. @@ -31,6 +33,11 @@ def mhc_pre_tilelang( hc_post_mult_value: post-mix multiplier value sinkhorn_repeat: number of sinkhorn iterations n_splits: split-k factor; + norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype + torch.bfloat16. When provided, RMSNorm is fused into the + layer_input write path of the big_fuse kernel. + norm_eps: epsilon for the fused RMSNorm; only consulted when + norm_weight is given. Returns: post_mix: shape (..., hc_mult), dtype torch.float32 @@ -40,6 +47,7 @@ def mhc_pre_tilelang( from vllm._tilelang_ops import ( compute_num_split, mhc_pre_big_fuse_tilelang, + mhc_pre_big_fuse_with_norm_tilelang, ) from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm from vllm.utils.math_utils import cdiv @@ -60,6 +68,13 @@ def mhc_pre_tilelang( assert hc_scale.shape == (3,) assert hc_base.shape == (hc_mult3,) + if norm_weight is not None: + assert norm_weight.shape == (hidden_size,) + if norm_weight.dtype != torch.bfloat16: + norm_weight = norm_weight.to(torch.bfloat16) + if not norm_weight.is_contiguous(): + norm_weight = norm_weight.contiguous() + outer_shape = residual.shape[:-2] residual_flat = residual.view(-1, hc_mult, hidden_size) @@ -95,24 +110,46 @@ def mhc_pre_tilelang( n_splits, ) - mhc_pre_big_fuse_tilelang( - gemm_out_mul, - gemm_out_sqrsum, - hc_scale, - hc_base, - residual_flat, - post_mix, - comb_mix, - layer_input, - hidden_size, - rms_eps, - hc_pre_eps, - hc_sinkhorn_eps, - hc_post_mult_value, - sinkhorn_repeat, - n_splits, - hc_mult, - ) + if norm_weight is None: + mhc_pre_big_fuse_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_flat, + post_mix, + comb_mix, + layer_input, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + n_splits, + hc_mult, + ) + else: + mhc_pre_big_fuse_with_norm_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_flat, + post_mix, + comb_mix, + layer_input, + norm_weight, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + norm_eps, + n_splits, + hc_mult, + ) return ( post_mix.view(*outer_shape, hc_mult, 1), @@ -132,6 +169,8 @@ def _mhc_pre_tilelang_fake( hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: hc_mult = residual.shape[-2] hidden_size = residual.shape[-1] @@ -198,10 +237,16 @@ def mhc_fused_post_pre_tilelang( sinkhorn_repeat: int, n_splits: int = 1, tile_n: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ Run one MHC post block followed by the next MHC pre block. + When ``norm_weight`` is provided, the layer_input_cur output is the + RMSNorm'd activation (fused into the kernel); otherwise it is the + raw pre-norm activation as before. + Returns: residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size) post_mix_cur: shape (..., hc_mult, 1) @@ -214,6 +259,7 @@ def mhc_fused_post_pre_tilelang( mhc_fused_tilelang, mhc_post_tilelang, mhc_pre_big_fuse_tilelang, + mhc_pre_big_fuse_with_norm_tilelang, ) from vllm.utils.math_utils import cdiv @@ -242,6 +288,13 @@ def mhc_fused_post_pre_tilelang( assert hc_scale.shape == (3,) assert hc_base.shape == (hc_mult3,) + if norm_weight is not None: + assert norm_weight.shape == (hidden_size,) + if norm_weight.dtype != torch.bfloat16: + norm_weight = norm_weight.to(torch.bfloat16) + if not norm_weight.is_contiguous(): + norm_weight = norm_weight.contiguous() + assert n_splits in (1, 2, 4, 8) assert hidden_size % n_splits == 0 @@ -332,24 +385,46 @@ def mhc_fused_post_pre_tilelang( n_splits, ) - mhc_pre_big_fuse_tilelang( - gemm_out_mul, - gemm_out_sqrsum, - hc_scale, - hc_base, - residual_cur, - post_mix_cur, - comb_mix_cur, - layer_input_cur, - hidden_size, - rms_eps, - hc_pre_eps, - hc_sinkhorn_eps, - hc_post_mult_value, - sinkhorn_repeat, - n_splits, - hc_mult, - ) + if norm_weight is None: + mhc_pre_big_fuse_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_cur, + post_mix_cur, + comb_mix_cur, + layer_input_cur, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + n_splits, + hc_mult, + ) + else: + mhc_pre_big_fuse_with_norm_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_cur, + post_mix_cur, + comb_mix_cur, + layer_input_cur, + norm_weight, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + norm_eps, + n_splits, + hc_mult, + ) return ( residual_cur.view(*outer_shape, hc_mult, hidden_size), @@ -373,6 +448,9 @@ def _mhc_fused_post_pre_tilelang_fake( hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, + tile_n: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: hc_mult = residual.shape[-2] hidden_size = residual.shape[-1] diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index a9f84ffaf0..cb3965f764 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -34,6 +34,8 @@ class MHCPreOp(CustomOp): hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return torch.ops.vllm.mhc_pre_tilelang( residual, @@ -46,6 +48,8 @@ class MHCPreOp(CustomOp): hc_post_mult_value, sinkhorn_repeat, n_splits, + norm_weight, + norm_eps, ) def forward_hip( @@ -60,6 +64,8 @@ class MHCPreOp(CustomOp): hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: # TODO: Reenable aiter after we are at the aiter # version that has this bugfix @@ -262,6 +268,8 @@ class MHCFusedPostPreOp(CustomOp): sinkhorn_repeat: int, n_splits: int = 1, tile_n: int = 1, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: return torch.ops.vllm.mhc_fused_post_pre_tilelang( x, @@ -278,6 +286,8 @@ class MHCFusedPostPreOp(CustomOp): sinkhorn_repeat, n_splits, tile_n, + norm_weight, + norm_eps, ) def forward_hip(self, *args, **kwargs): diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index adce3e5295..6c4f058cfb 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -22,9 +22,7 @@ from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( fused_topk_bias, ) -from vllm.model_executor.layers.fused_moe.router.norm_gate_linear import ( - NormGateLinear, -) +from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -655,23 +653,23 @@ class DeepseekV4MoE(nn.Module): "deep_gemm_mega_moe for this checkpoint." ) - # Fused RMSNorm + gate: owns both ffn_norm and the gate matmul. - self.norm_gate = NormGateLinear( - hidden_size=config.hidden_size, - num_experts=config.n_routed_experts, - rms_eps=config.rms_norm_eps, - prefix=f"{prefix}.norm_gate", + self.gate = GateLinear( + input_size=config.hidden_size, + output_size=config.n_routed_experts, + bias=False, + out_dtype=torch.float32, + prefix=f"{prefix}.gate", ) - # Routing-side tensors live on ``norm_gate`` directly (not on the - # inner gate); they are initialized to None in NormGatedLinear and - # populated below depending on the MoE variant. + + self.gate.e_score_correction_bias = None + self.gate.tid2eid = None is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32 if is_hash_moe: # hash MoE doesn't use e_score_correction_bias # Use randint instead of empty to avoid garbage values causing # invalid memory access in dummy mode (--load-format="dummy") - self.norm_gate.tid2eid = nn.Parameter( + self.gate.tid2eid = nn.Parameter( torch.randint( 0, config.n_routed_experts, @@ -681,7 +679,7 @@ class DeepseekV4MoE(nn.Module): requires_grad=False, ) elif getattr(config, "topk_method", None) == "noaux_tc": - self.norm_gate.e_score_correction_bias = nn.Parameter( + self.gate.e_score_correction_bias = nn.Parameter( torch.empty(config.n_routed_experts, dtype=torch.float32), requires_grad=False, ) @@ -744,9 +742,10 @@ class DeepseekV4MoE(nn.Module): self.n_local_experts = config.n_routed_experts // self.tp_size self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts - # We don't pass `gate` into FusedMoE + self.experts = FusedMoE( shared_experts=self.shared_experts, + gate=self.gate, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, @@ -756,8 +755,8 @@ class DeepseekV4MoE(nn.Module): prefix=f"{prefix}.experts", scoring_func=self.scoring_func, routed_scaling_factor=self.routed_scaling_factor, - e_score_correction_bias=self.norm_gate.e_score_correction_bias, - hash_indices_table=self.norm_gate.tid2eid, + e_score_correction_bias=self.gate.e_score_correction_bias, + hash_indices_table=self.gate.tid2eid, swiglu_limit=self.swiglu_limit, router_logits_dtype=torch.float32, ) @@ -765,40 +764,40 @@ class DeepseekV4MoE(nn.Module): def forward( self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None ) -> torch.Tensor: - if self.norm_gate.tid2eid is not None and input_ids is None: + if self.gate.tid2eid is not None and input_ids is None: raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.") if not self.use_mega_moe: return self._forward_fused_moe(hidden_states, input_ids) org_shape = hidden_states.shape - normed_x, router_logits = self.norm_gate(hidden_states) + router_logits, _ = self.gate(hidden_states) topk_weights, topk_ids = fused_topk_bias( - hidden_states=normed_x, + hidden_states=hidden_states, gating_output=router_logits, scoring_func=self.scoring_func, - e_score_correction_bias=self.norm_gate.e_score_correction_bias.data - if self.norm_gate.e_score_correction_bias is not None + e_score_correction_bias=self.gate.e_score_correction_bias.data + if self.gate.e_score_correction_bias is not None else None, topk=self.n_activated_experts, renormalize=self.renormalize, indices_type=self.hash_indices_dtype, input_tokens=input_ids, - hash_indices_table=self.norm_gate.tid2eid, + hash_indices_table=self.gate.tid2eid, routed_scaling_factor=self.routed_scaling_factor, ) activation_clamp = ( float(self.swiglu_limit) if self.swiglu_limit is not None else None ) final_hidden_states = self.experts( - normed_x, + hidden_states, topk_weights, topk_ids, activation_clamp=activation_clamp, ) if self.shared_experts is not None: - shared_output = self.shared_experts(normed_x) + shared_output = self.shared_experts(hidden_states) final_hidden_states += shared_output return final_hidden_states.view(org_shape) @@ -806,14 +805,21 @@ class DeepseekV4MoE(nn.Module): def _forward_fused_moe( self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None ) -> torch.Tensor: - assert not self.experts.is_internal_router org_shape = hidden_states.shape - normed_x, router_logits = self.norm_gate(hidden_states) - final_hidden_states = self.experts( - hidden_states=normed_x, - router_logits=router_logits, - input_ids=input_ids, - ) + if self.experts.is_internal_router: + # In this case, the gate/router runs inside the FusedMoE class + final_hidden_states = self.experts( + hidden_states=hidden_states, + router_logits=hidden_states, + input_ids=input_ids, + ) + else: + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts( + hidden_states=hidden_states, + router_logits=router_logits, + input_ids=input_ids, + ) return final_hidden_states.view(org_shape) @@ -1024,8 +1030,7 @@ class DeepseekV4DecoderLayer(nn.Module): self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn") self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) - # ``ffn_norm`` is owned by ``self.ffn.norm_gate`` (fused with the - # router gate matmul); see ``NormGatedLinear``. + self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.hc_mult = config.hc_mult self.hc_sinkhorn_iters = config.hc_sinkhorn_iters self.hc_eps = config.hc_eps @@ -1084,6 +1089,8 @@ class DeepseekV4DecoderLayer(nn.Module): hc_fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, ): post_mix, res_mix, layer_input = self.mhc_pre( residual=x, @@ -1095,6 +1102,8 @@ class DeepseekV4DecoderLayer(nn.Module): hc_sinkhorn_eps=self.hc_eps, hc_post_mult_value=self.hc_post_alpha, sinkhorn_repeat=self.hc_sinkhorn_iters, + norm_weight=norm_weight, + norm_eps=norm_eps, ) return layer_input, post_mix, res_mix @@ -1116,11 +1125,18 @@ class DeepseekV4DecoderLayer(nn.Module): res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + attn_norm_weight = self.attn_norm.weight.data + attn_norm_eps = self.attn_norm.variance_epsilon if residual is None: # Run standalone hc_pre on first layer residual = x x, post_mix, res_mix = self.hc_pre( - x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base + x, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + norm_weight=attn_norm_weight, + norm_eps=attn_norm_eps, ) else: residual, post_mix, res_mix, x = self.mhc_fused_post_pre( @@ -1136,11 +1152,17 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, + n_splits=1, + tile_n=1, + norm_weight=attn_norm_weight, + norm_eps=attn_norm_eps, ) - x = self.attn_norm(x) + # attn_norm is fused into hc_pre / mhc_fused_post_pre above. x = self.attn(positions, x, None) + ffn_norm_weight = self.ffn_norm.weight.data + ffn_norm_eps = self.ffn_norm.variance_epsilon residual, post_mix, res_mix, x = self.mhc_fused_post_pre( x, residual, @@ -1154,9 +1176,13 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, + n_splits=1, + tile_n=1, + norm_weight=ffn_norm_weight, + norm_eps=ffn_norm_eps, ) - # ffn_norm is now folded into self.ffn.norm_gate; ffn() takes - # the pre-norm activation directly. + # ffn_norm is fused into mhc_fused_post_pre above; ffn() takes the + # already-normed activation directly. x = self.ffn(x, input_ids) return x, residual, post_mix, res_mix @@ -1183,8 +1209,7 @@ class DeepseekV4DecoderLayer(nn.Module): x, post, comb = self.hc_pre( x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base ) - # ffn_norm is now folded into self.ffn.norm_gate; ffn() takes - # the pre-norm activation directly. + x = self.ffn_norm(x) x = self.ffn(x, input_ids) x = self.hc_post(x, residual, post, comb) return x, None, None, None @@ -1534,13 +1559,7 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: orig_to_new_suffix={ "head.weight": "lm_head.weight", "embed.weight": "embed_tokens.weight", - # Pre-MoE norm + gate are now owned by ``DeepseekV4MoE.norm_gate`` - # (see NormGatedLinear). - ".ffn_norm.weight": ".ffn.norm_gate.norm.weight", - ".ffn.gate.weight": ".ffn.norm_gate.gate.weight", - ".ffn.gate.bias": ".ffn.norm_gate.e_score_correction_bias", - # Hash MoE table also moved off the inner gate. - ".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid", + ".ffn.gate.bias": ".ffn.gate.e_score_correction_bias", }, orig_to_new_substr={ ".attn.compressor.": ".attn.mla_attn.compressor.", diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index 071abe2f4a..9d0ac43530 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -292,11 +292,6 @@ class DeepSeekV4MTP(nn.Module): ".emb.tok_emb.weight": ".embed_tokens.weight", ".head.weight": ".shared_head.head.weight", ".norm.weight": ".shared_head.norm.weight", - # Pre-MoE norm + gate are now owned by - # ``DeepseekV4MoE.norm_gate`` (see NormGatedLinear). - ".ffn_norm.weight": ".ffn.norm_gate.norm.weight", - ".ffn.gate.weight": ".ffn.norm_gate.gate.weight", - ".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid", } def _remap_weight_name(name: str) -> str: @@ -444,11 +439,11 @@ class DeepSeekV4MTP(nn.Module): ".shared_experts.w2", ".shared_experts.down_proj" ) if name.endswith(".ffn.gate.bias"): - # ``e_score_correction_bias`` lives on - # ``norm_gate`` directly (not on the inner gate). + # ``e_score_correction_bias`` lives on the gate + # under a different attribute name. name = name.replace( ".ffn.gate.bias", - ".ffn.norm_gate.e_score_correction_bias", + ".ffn.gate.e_score_correction_bias", ) param = params_dict[name] weight_loader = getattr(