[Kernel] Add mhc_pre_big_fuse_with_norm_tilelang (#43474)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -177,6 +177,173 @@ def mhc_pre_big_fuse_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
# Copied from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/mhc.py#L478
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
|
||||
},
|
||||
)
|
||||
def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
norm_eps: float,
|
||||
n_splits: int = 16,
|
||||
hc_mult: int = 4,
|
||||
gemm_last_dim: int = -1,
|
||||
):
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
if gemm_last_dim < 0:
|
||||
gemm_last_dim = hc_mult3
|
||||
hidden_block = math.gcd(1024, hidden_size)
|
||||
|
||||
gemm_out_mul: T.Tensor[[n_splits, num_tokens, gemm_last_dim], T.float32] # type: ignore[no-redef, valid-type]
|
||||
gemm_out_sqrsum: T.Tensor[[n_splits, num_tokens], T.float32] # type: ignore[no-redef, valid-type]
|
||||
hc_scale: T.Tensor[[3], T.float32] # type: ignore[no-redef, valid-type]
|
||||
hc_base: T.Tensor[[hc_mult3], T.float32] # type: ignore[no-redef, valid-type]
|
||||
residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
post_mix: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
comb_mix: T.Tensor[[num_tokens, hc_mult * hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
layer_input: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
norm_weight: T.Tensor[[hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
|
||||
T.pdl_sync()
|
||||
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / (hc_mult * hidden_size) + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
|
||||
T.copy(mixes, mixes_shared)
|
||||
|
||||
if T.get_thread_binding() < 32:
|
||||
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
post_mix[i, j] = (
|
||||
T.sigmoid(
|
||||
mixes_shared[j + hc_mult] * hc_scale[1] + hc_base[j + hc_mult]
|
||||
)
|
||||
* hc_post_mult_value
|
||||
)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = (
|
||||
mixes_shared[j * hc_mult + k + hc_mult * 2] * hc_scale[2]
|
||||
+ hc_base[j * hc_mult + k + hc_mult * 2]
|
||||
)
|
||||
|
||||
row_sum = T.alloc_fragment(hc_mult, T.float32)
|
||||
col_sum = T.alloc_fragment(hc_mult, T.float32)
|
||||
|
||||
row_max = T.alloc_fragment(hc_mult, T.float32)
|
||||
T.reduce_max(cm, row_max, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = T.exp(cm[j, k] - row_max[j])
|
||||
T.reduce_sum(cm, row_sum, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / row_sum[j] + hc_sinkhorn_eps
|
||||
|
||||
T.reduce_sum(cm, col_sum, dim=0)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
|
||||
|
||||
for _ in T.serial(sinkhorn_repeat - 1):
|
||||
T.reduce_sum(cm, row_sum, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (row_sum[j] + hc_sinkhorn_eps)
|
||||
|
||||
T.reduce_sum(cm, col_sum, dim=0)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
|
||||
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
comb_mix[i, j * hc_mult + k] = cm[j, k]
|
||||
else:
|
||||
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
pre_mix_shared[j] = (
|
||||
T.sigmoid(
|
||||
mixes_shared[j] * hc_scale[0] + hc_base[j],
|
||||
)
|
||||
+ hc_pre_eps
|
||||
)
|
||||
|
||||
# Pass 1: stash unnormalized weighted-sum output in shared memory
|
||||
# as bf16 (matches the rounding that RMSNorm would see) while
|
||||
# accumulating the per-position squared sum.
|
||||
output_shared = T.alloc_shared(hidden_size, T.bfloat16)
|
||||
sumsq_per_pos = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(sumsq_per_pos)
|
||||
|
||||
for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=3):
|
||||
xs = T.alloc_shared((hc_mult, hidden_block), T.bfloat16)
|
||||
xl = T.alloc_fragment((hc_mult, hidden_block), T.float32)
|
||||
T.copy(residual[i, 0, i0_h * hidden_block], xs)
|
||||
T.copy(xs, xl)
|
||||
|
||||
ol = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(ol)
|
||||
|
||||
for i_hc in T.serial(hc_mult):
|
||||
pre = pre_mix_shared[i_hc]
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
ol[i1_h] += pre * xl[i_hc, i1_h]
|
||||
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
sumsq_per_pos[i1_h] += ol[i1_h] * ol[i1_h]
|
||||
output_shared[i0_h * hidden_block + i1_h] = T.bfloat16(ol[i1_h])
|
||||
|
||||
sumsq = T.alloc_fragment(1, T.float32)
|
||||
T.reduce_sum(sumsq_per_pos, sumsq, dim=0)
|
||||
rsqrt_norm = T.alloc_fragment(1, T.float32)
|
||||
rsqrt_norm[0] = T.rsqrt(sumsq[0] / hidden_size + norm_eps)
|
||||
|
||||
# Pass 2: scale by rsqrt * norm_weight and write the result to HBM.
|
||||
for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=2):
|
||||
w_shared = T.alloc_shared(hidden_block, T.bfloat16)
|
||||
w_local = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.copy(norm_weight[i0_h * hidden_block], w_shared)
|
||||
T.copy(w_shared, w_local)
|
||||
|
||||
ol = T.alloc_fragment(hidden_block, T.float32)
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
ol[i1_h] = (
|
||||
output_shared[i0_h * hidden_block + i1_h]
|
||||
* rsqrt_norm[0]
|
||||
* w_local[i1_h]
|
||||
)
|
||||
|
||||
T.copy(ol, layer_input[i, i0_h * hidden_block])
|
||||
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
|
||||
@@ -16,6 +16,8 @@ def mhc_pre_tilelang(
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
@@ -31,6 +33,11 @@ def mhc_pre_tilelang(
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
norm_weight: optional RMSNorm weight, shape (hidden_size,), dtype
|
||||
torch.bfloat16. When provided, RMSNorm is fused into the
|
||||
layer_input write path of the big_fuse kernel.
|
||||
norm_eps: epsilon for the fused RMSNorm; only consulted when
|
||||
norm_weight is given.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
@@ -40,6 +47,7 @@ def mhc_pre_tilelang(
|
||||
from vllm._tilelang_ops import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -60,6 +68,13 @@ def mhc_pre_tilelang(
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
if norm_weight is not None:
|
||||
assert norm_weight.shape == (hidden_size,)
|
||||
if norm_weight.dtype != torch.bfloat16:
|
||||
norm_weight = norm_weight.to(torch.bfloat16)
|
||||
if not norm_weight.is_contiguous():
|
||||
norm_weight = norm_weight.contiguous()
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
@@ -95,24 +110,46 @@ def mhc_pre_tilelang(
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
@@ -132,6 +169,8 @@ def _mhc_pre_tilelang_fake(
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
@@ -198,10 +237,16 @@ def mhc_fused_post_pre_tilelang(
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run one MHC post block followed by the next MHC pre block.
|
||||
|
||||
When ``norm_weight`` is provided, the layer_input_cur output is the
|
||||
RMSNorm'd activation (fused into the kernel); otherwise it is the
|
||||
raw pre-norm activation as before.
|
||||
|
||||
Returns:
|
||||
residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size)
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
@@ -214,6 +259,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
mhc_fused_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
@@ -242,6 +288,13 @@ def mhc_fused_post_pre_tilelang(
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
if norm_weight is not None:
|
||||
assert norm_weight.shape == (hidden_size,)
|
||||
if norm_weight.dtype != torch.bfloat16:
|
||||
norm_weight = norm_weight.to(torch.bfloat16)
|
||||
if not norm_weight.is_contiguous():
|
||||
norm_weight = norm_weight.contiguous()
|
||||
|
||||
assert n_splits in (1, 2, 4, 8)
|
||||
assert hidden_size % n_splits == 0
|
||||
|
||||
@@ -332,24 +385,46 @@ def mhc_fused_post_pre_tilelang(
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
if norm_weight is None:
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
layer_input_cur,
|
||||
norm_weight,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_eps,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return (
|
||||
residual_cur.view(*outer_shape, hc_mult, hidden_size),
|
||||
@@ -373,6 +448,9 @@ def _mhc_fused_post_pre_tilelang_fake(
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
|
||||
@@ -34,6 +34,8 @@ class MHCPreOp(CustomOp):
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
@@ -46,6 +48,8 @@ class MHCPreOp(CustomOp):
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_hip(
|
||||
@@ -60,6 +64,8 @@ class MHCPreOp(CustomOp):
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# TODO: Reenable aiter after we are at the aiter
|
||||
# version that has this bugfix
|
||||
@@ -262,6 +268,8 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.mhc_fused_post_pre_tilelang(
|
||||
x,
|
||||
@@ -278,6 +286,8 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
tile_n,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
|
||||
@@ -22,9 +22,7 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.norm_gate_linear import (
|
||||
NormGateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -655,23 +653,23 @@ class DeepseekV4MoE(nn.Module):
|
||||
"deep_gemm_mega_moe for this checkpoint."
|
||||
)
|
||||
|
||||
# Fused RMSNorm + gate: owns both ffn_norm and the gate matmul.
|
||||
self.norm_gate = NormGateLinear(
|
||||
hidden_size=config.hidden_size,
|
||||
num_experts=config.n_routed_experts,
|
||||
rms_eps=config.rms_norm_eps,
|
||||
prefix=f"{prefix}.norm_gate",
|
||||
self.gate = GateLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.n_routed_experts,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
# Routing-side tensors live on ``norm_gate`` directly (not on the
|
||||
# inner gate); they are initialized to None in NormGatedLinear and
|
||||
# populated below depending on the MoE variant.
|
||||
|
||||
self.gate.e_score_correction_bias = None
|
||||
self.gate.tid2eid = None
|
||||
is_hash_moe = extract_layer_index(prefix) < config.num_hash_layers
|
||||
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
# Use randint instead of empty to avoid garbage values causing
|
||||
# invalid memory access in dummy mode (--load-format="dummy")
|
||||
self.norm_gate.tid2eid = nn.Parameter(
|
||||
self.gate.tid2eid = nn.Parameter(
|
||||
torch.randint(
|
||||
0,
|
||||
config.n_routed_experts,
|
||||
@@ -681,7 +679,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
requires_grad=False,
|
||||
)
|
||||
elif getattr(config, "topk_method", None) == "noaux_tc":
|
||||
self.norm_gate.e_score_correction_bias = nn.Parameter(
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
@@ -744,9 +742,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.n_local_experts = config.n_routed_experts // self.tp_size
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
# We don't pass `gate` into FusedMoE
|
||||
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -756,8 +755,8 @@ class DeepseekV4MoE(nn.Module):
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.norm_gate.e_score_correction_bias,
|
||||
hash_indices_table=self.norm_gate.tid2eid,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
@@ -765,40 +764,40 @@ class DeepseekV4MoE(nn.Module):
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
if self.norm_gate.tid2eid is not None and input_ids is None:
|
||||
if self.gate.tid2eid is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
|
||||
|
||||
if not self.use_mega_moe:
|
||||
return self._forward_fused_moe(hidden_states, input_ids)
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
normed_x, router_logits = self.norm_gate(hidden_states)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=normed_x,
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
scoring_func=self.scoring_func,
|
||||
e_score_correction_bias=self.norm_gate.e_score_correction_bias.data
|
||||
if self.norm_gate.e_score_correction_bias is not None
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias.data
|
||||
if self.gate.e_score_correction_bias is not None
|
||||
else None,
|
||||
topk=self.n_activated_experts,
|
||||
renormalize=self.renormalize,
|
||||
indices_type=self.hash_indices_dtype,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=self.norm_gate.tid2eid,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
activation_clamp = (
|
||||
float(self.swiglu_limit) if self.swiglu_limit is not None else None
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
normed_x,
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation_clamp=activation_clamp,
|
||||
)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
shared_output = self.shared_experts(normed_x)
|
||||
shared_output = self.shared_experts(hidden_states)
|
||||
final_hidden_states += shared_output
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
@@ -806,14 +805,21 @@ class DeepseekV4MoE(nn.Module):
|
||||
def _forward_fused_moe(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
assert not self.experts.is_internal_router
|
||||
org_shape = hidden_states.shape
|
||||
normed_x, router_logits = self.norm_gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=normed_x,
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
else:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
@@ -1024,8 +1030,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
# ``ffn_norm`` is owned by ``self.ffn.norm_gate`` (fused with the
|
||||
# router gate matmul); see ``NormGatedLinear``.
|
||||
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_sinkhorn_iters = config.hc_sinkhorn_iters
|
||||
self.hc_eps = config.hc_eps
|
||||
@@ -1084,6 +1089,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
):
|
||||
post_mix, res_mix, layer_input = self.mhc_pre(
|
||||
residual=x,
|
||||
@@ -1095,6 +1102,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
return layer_input, post_mix, res_mix
|
||||
|
||||
@@ -1116,11 +1125,18 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
res_mix: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
attn_norm_weight = self.attn_norm.weight.data
|
||||
attn_norm_eps = self.attn_norm.variance_epsilon
|
||||
if residual is None:
|
||||
# Run standalone hc_pre on first layer
|
||||
residual = x
|
||||
x, post_mix, res_mix = self.hc_pre(
|
||||
x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
|
||||
x,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
norm_weight=attn_norm_weight,
|
||||
norm_eps=attn_norm_eps,
|
||||
)
|
||||
else:
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
@@ -1136,11 +1152,17 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
n_splits=1,
|
||||
tile_n=1,
|
||||
norm_weight=attn_norm_weight,
|
||||
norm_eps=attn_norm_eps,
|
||||
)
|
||||
|
||||
x = self.attn_norm(x)
|
||||
# attn_norm is fused into hc_pre / mhc_fused_post_pre above.
|
||||
x = self.attn(positions, x, None)
|
||||
|
||||
ffn_norm_weight = self.ffn_norm.weight.data
|
||||
ffn_norm_eps = self.ffn_norm.variance_epsilon
|
||||
residual, post_mix, res_mix, x = self.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
@@ -1154,9 +1176,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
n_splits=1,
|
||||
tile_n=1,
|
||||
norm_weight=ffn_norm_weight,
|
||||
norm_eps=ffn_norm_eps,
|
||||
)
|
||||
# ffn_norm is now folded into self.ffn.norm_gate; ffn() takes
|
||||
# the pre-norm activation directly.
|
||||
# ffn_norm is fused into mhc_fused_post_pre above; ffn() takes the
|
||||
# already-normed activation directly.
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
@@ -1183,8 +1209,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
# ffn_norm is now folded into self.ffn.norm_gate; ffn() takes
|
||||
# the pre-norm activation directly.
|
||||
x = self.ffn_norm(x)
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x, None, None, None
|
||||
@@ -1534,13 +1559,7 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
# Pre-MoE norm + gate are now owned by ``DeepseekV4MoE.norm_gate``
|
||||
# (see NormGatedLinear).
|
||||
".ffn_norm.weight": ".ffn.norm_gate.norm.weight",
|
||||
".ffn.gate.weight": ".ffn.norm_gate.gate.weight",
|
||||
".ffn.gate.bias": ".ffn.norm_gate.e_score_correction_bias",
|
||||
# Hash MoE table also moved off the inner gate.
|
||||
".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
".attn.compressor.": ".attn.mla_attn.compressor.",
|
||||
|
||||
@@ -292,11 +292,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
".emb.tok_emb.weight": ".embed_tokens.weight",
|
||||
".head.weight": ".shared_head.head.weight",
|
||||
".norm.weight": ".shared_head.norm.weight",
|
||||
# Pre-MoE norm + gate are now owned by
|
||||
# ``DeepseekV4MoE.norm_gate`` (see NormGatedLinear).
|
||||
".ffn_norm.weight": ".ffn.norm_gate.norm.weight",
|
||||
".ffn.gate.weight": ".ffn.norm_gate.gate.weight",
|
||||
".ffn.gate.tid2eid": ".ffn.norm_gate.tid2eid",
|
||||
}
|
||||
|
||||
def _remap_weight_name(name: str) -> str:
|
||||
@@ -444,11 +439,11 @@ class DeepSeekV4MTP(nn.Module):
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
# ``e_score_correction_bias`` lives on
|
||||
# ``norm_gate`` directly (not on the inner gate).
|
||||
# ``e_score_correction_bias`` lives on the gate
|
||||
# under a different attribute name.
|
||||
name = name.replace(
|
||||
".ffn.gate.bias",
|
||||
".ffn.norm_gate.e_score_correction_bias",
|
||||
".ffn.gate.e_score_correction_bias",
|
||||
)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
|
||||
Reference in New Issue
Block a user