[DSV4] Fuse norm and router for low latency scenario (#41263)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: jeejeelee <jeejeelee@verda-b300-05.datacrunch.io>
Co-authored-by: jeejeelee <jeejeelee@verda-b300-05.datacrunch.io>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jee Jee Li
2026-05-14 05:11:02 -07:00
committed by GitHub
co-authored by jeejeelee mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1ea9401364
commit 0a65d46628
11 changed files with 815 additions and 43 deletions
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused RMSNorm + GateLinear for DeepSeek V4 MoE routing."""
import torch
from torch import nn
import vllm._custom_ops as ops
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.utils.torch_utils import direct_register_custom_op
DSV4_PRO_NUM_EXPERTS = 384
DSV4_PRO_HIDDEN_SIZE = 7168
DSV4_PRO_MAX_NUM_TOKENS = 16
def _dsv4_pro_norm_gate(
x: torch.Tensor,
norm_weight: torch.Tensor,
gate_weight: torch.Tensor,
rms_eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Runtime dispatcher: fused ``dsv4_norm_router_gemm`` (M<=16) vs the
unfused ``rms_norm + dsv3_router_gemm`` fallback (M>16).
"""
if x.shape[0] <= DSV4_PRO_MAX_NUM_TOKENS:
return ops.dsv4_norm_router_gemm(x, norm_weight, gate_weight, rms_eps)
normed = torch.empty_like(x)
# Call `_C::rms_norm` here to avoid select the path of native rms
torch.ops._C.rms_norm(normed, x, norm_weight, rms_eps)
logits = torch.mm(normed, gate_weight.t(), out_dtype=torch.float32)
return normed, logits
def _dsv4_pro_norm_gate_fake(
x: torch.Tensor,
norm_weight: torch.Tensor,
gate_weight: torch.Tensor,
rms_eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = x.shape[0]
num_experts = gate_weight.shape[0]
return (
torch.empty_like(x),
torch.empty(num_tokens, num_experts, dtype=torch.float32, device=x.device),
)
direct_register_custom_op(
op_name="dsv4_pro_norm_gate",
op_func=_dsv4_pro_norm_gate,
mutates_args=[],
fake_impl=_dsv4_pro_norm_gate_fake,
)
@PluggableLayer.register("norm_gated_linear")
class NormGateLinear(nn.Module):
"""RMSNorm + GateLinear, fused on DSV4-Pro only."""
def __init__(
self,
hidden_size: int,
num_experts: int,
rms_eps: float = 1e-6,
params_dtype: torch.dtype | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.num_experts = num_experts
self.rms_eps = rms_eps
self.norm = RMSNorm(hidden_size, eps=rms_eps, dtype=params_dtype)
self.gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=torch.float32, # DSV4 router output is fp32
params_dtype=params_dtype,
prefix=f"{prefix}.gate" if prefix else "gate",
)
self.e_score_correction_bias = None
self.tid2eid = None
self._fused_kernel_supported = (
hidden_size == DSV4_PRO_HIDDEN_SIZE
and num_experts == DSV4_PRO_NUM_EXPERTS
and self.gate.allow_dsv3_router_gemm # cuda platform
)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if self._fused_kernel_supported:
assert x.shape[1] == DSV4_PRO_HIDDEN_SIZE
assert self.gate.weight.shape == (
DSV4_PRO_NUM_EXPERTS,
DSV4_PRO_HIDDEN_SIZE,
)
# This must be wrapped in a custom op because our torch.compile integration
# does not support runtime dispatching on num_tokens.
return torch.ops.vllm.dsv4_pro_norm_gate(
x, self.norm.weight, self.gate.weight, self.rms_eps
)
# Non-Pro fallback (e.g. DSV4-Flash with hidden_size=4096):
normed_x = self.norm(x)
logits, _ = self.gate(normed_x)
return normed_x, logits
+44 -42
View File
@@ -23,11 +23,14 @@ from vllm.model_executor.layers.deepseek_v4_attention import (
DeepseekV4MLAModules,
DeepseekV4MultiHeadLatentAttentionWrapper,
)
from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.model_executor.layers.fused_moe.router.norm_gate_linear import (
NormGateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -755,23 +758,23 @@ class DeepseekV4MoE(nn.Module):
"deep_gemm_mega_moe for this checkpoint."
)
self.gate = GateLinear(
config.hidden_size,
config.n_routed_experts,
out_dtype=torch.float32,
bias=False,
prefix=f"{prefix}.gate",
# Fused RMSNorm + gate: owns both ffn_norm and the gate matmul.
self.norm_gate = NormGateLinear(
hidden_size=config.hidden_size,
num_experts=config.n_routed_experts,
rms_eps=config.rms_norm_eps,
prefix=f"{prefix}.norm_gate",
)
self.gate.e_score_correction_bias = None
self.gate.tid2eid = None
# Routing-side tensors live on ``norm_gate`` directly (not on the
# inner gate); they are initialized to None in NormGatedLinear and
# populated below depending on the MoE variant.
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
# Use randint instead of empty to avoid garbage values causing
# invalid memory access in dummy mode (--load-format="dummy")
self.gate.tid2eid = nn.Parameter(
self.norm_gate.tid2eid = nn.Parameter(
torch.randint(
0,
config.n_routed_experts,
@@ -781,7 +784,7 @@ class DeepseekV4MoE(nn.Module):
requires_grad=False,
)
elif getattr(config, "topk_method", None) == "noaux_tc":
self.gate.e_score_correction_bias = nn.Parameter(
self.norm_gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.n_routed_experts, dtype=torch.float32),
requires_grad=False,
)
@@ -844,10 +847,9 @@ class DeepseekV4MoE(nn.Module):
self.n_local_experts = config.n_routed_experts // self.tp_size
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
# We don't pass `gate` into FusedMoE
self.experts = FusedMoE(
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
@@ -857,8 +859,8 @@ class DeepseekV4MoE(nn.Module):
prefix=f"{prefix}.experts",
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
e_score_correction_bias=self.gate.e_score_correction_bias,
hash_indices_table=self.gate.tid2eid,
e_score_correction_bias=self.norm_gate.e_score_correction_bias,
hash_indices_table=self.norm_gate.tid2eid,
swiglu_limit=self.swiglu_limit,
router_logits_dtype=torch.float32,
)
@@ -866,40 +868,40 @@ class DeepseekV4MoE(nn.Module):
def forward(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
) -> torch.Tensor:
if self.gate.tid2eid is not None and input_ids is None:
if self.norm_gate.tid2eid is not None and input_ids is None:
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
if not self.use_mega_moe:
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
router_logits, _ = self.gate(hidden_states)
normed_x, router_logits = self.norm_gate(hidden_states)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
hidden_states=normed_x,
gating_output=router_logits,
scoring_func=self.scoring_func,
e_score_correction_bias=self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
e_score_correction_bias=self.norm_gate.e_score_correction_bias.data
if self.norm_gate.e_score_correction_bias is not None
else None,
topk=self.n_activated_experts,
renormalize=self.renormalize,
indices_type=self.hash_indices_dtype,
input_tokens=input_ids,
hash_indices_table=self.gate.tid2eid,
hash_indices_table=self.norm_gate.tid2eid,
routed_scaling_factor=self.routed_scaling_factor,
)
activation_clamp = (
float(self.swiglu_limit) if self.swiglu_limit is not None else None
)
final_hidden_states = self.experts(
hidden_states,
normed_x,
topk_weights,
topk_ids,
activation_clamp=activation_clamp,
)
if self.shared_experts is not None:
shared_output = self.shared_experts(hidden_states)
shared_output = self.shared_experts(normed_x)
final_hidden_states += shared_output
return final_hidden_states.view(org_shape)
@@ -907,21 +909,14 @@ class DeepseekV4MoE(nn.Module):
def _forward_fused_moe(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
) -> torch.Tensor:
assert not self.experts.is_internal_router
org_shape = hidden_states.shape
if self.experts.is_internal_router:
# In this case, the gate/router runs inside the FusedMoE class
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
input_ids=input_ids,
)
else:
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=router_logits,
input_ids=input_ids,
)
normed_x, router_logits = self.norm_gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=normed_x,
router_logits=router_logits,
input_ids=input_ids,
)
return final_hidden_states.view(org_shape)
@@ -1125,7 +1120,8 @@ class DeepseekV4DecoderLayer(nn.Module):
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
# ``ffn_norm`` is owned by ``self.ffn.norm_gate`` (fused with the
# router gate matmul); see ``NormGatedLinear``.
self.hc_mult = config.hc_mult
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
self.hc_eps = config.hc_eps
@@ -1255,8 +1251,8 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_post_alpha,
self.hc_sinkhorn_iters,
)
x = self.ffn_norm(x)
# ffn_norm is now folded into self.ffn.norm_gate; ffn() takes
# the pre-norm activation directly.
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
@@ -1629,7 +1625,13 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
orig_to_new_suffix={
"head.weight": "lm_head.weight",
"embed.weight": "embed_tokens.weight",
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
# Pre-MoE norm + gate are now owned by ``DeepseekV4MoE.norm_gate``
# (see NormGatedLinear).
".ffn_norm.weight": ".ffn.norm_gate.norm.weight",
".ffn.gate.weight": ".ffn.norm_gate.gate.weight",
".ffn.gate.bias": ".ffn.norm_gate.e_score_correction_bias",
# Hash MoE table also moved off the inner gate.
".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid",
},
orig_to_new_substr={
".attn.compressor.": ".attn.mla_attn.compressor.",
+11 -1
View File
@@ -290,6 +290,11 @@ class DeepSeekV4MTP(nn.Module):
".emb.tok_emb.weight": ".embed_tokens.weight",
".head.weight": ".shared_head.head.weight",
".norm.weight": ".shared_head.norm.weight",
# Pre-MoE norm + gate are now owned by
# ``DeepseekV4MoE.norm_gate`` (see NormGatedLinear).
".ffn_norm.weight": ".ffn.norm_gate.norm.weight",
".ffn.gate.weight": ".ffn.norm_gate.gate.weight",
".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid",
}
def _remap_weight_name(name: str) -> str:
@@ -437,7 +442,12 @@ class DeepSeekV4MTP(nn.Module):
".shared_experts.w2", ".shared_experts.down_proj"
)
if name.endswith(".ffn.gate.bias"):
name = name.replace(".bias", ".e_score_correction_bias")
# ``e_score_correction_bias`` lives on
# ``norm_gate`` directly (not on the inner gate).
name = name.replace(
".ffn.gate.bias",
".ffn.norm_gate.e_score_correction_bias",
)
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader