[Model][DeepSeek-V4] Make MegaMoE shared-expert finalize independent of linear post-load order (#59927)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-05 08:39:49 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 34051ad714
commit 4a30c4cad0
2 changed files with 43 additions and 18 deletions
+27 -7
View File
@@ -9,6 +9,9 @@ import torch
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
)
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
@@ -23,6 +26,7 @@ from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
pytestmark = pytest.mark.skipif(
@@ -473,19 +477,23 @@ def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
@pytest.mark.parametrize(
"hidden_size,intermediate_size,block_size,mxfp8",
"hidden_size,intermediate_size,block_size,mxfp8,packed",
[
pytest.param(128, 512, 128, False, id="aligned-block128"),
pytest.param(128, 512, 128, True, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"),
pytest.param(128, 2304, 128, False, id="native-block128"),
pytest.param(5120, 2304, 32, True, id="deepseek-v41-flash-mxfp8"),
pytest.param(128, 512, 128, False, False, id="aligned-block128"),
pytest.param(128, 512, 128, False, True, id="aligned-block128-packed"),
pytest.param(128, 512, 128, True, False, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
pytest.param(128, 2304, 128, False, False, id="native-block128"),
pytest.param(128, 2304, 128, False, True, id="native-block128-packed"),
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash-mxfp8"),
],
)
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8, packed
):
"""Shared fusion preserves checkpoint weights and scales at native widths."""
if packed and not is_deep_gemm_supported():
pytest.skip("Packing the shared scales requires DeepGEMM")
class FakeDeepGemm:
transformed_dims: list[tuple[int, int]] = []
@@ -593,6 +601,18 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
.repeat_interleave(block_k // 32, dim=1)
)
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
if packed:
linear.weight_scale_inv = torch.nn.Parameter(
deepgemm_post_process_weight_scale_block(
scale.data.cuda().unsqueeze(0),
*linear.weight.shape,
quant_block_shape=linear.weight_block_size,
num_groups=1,
)
.squeeze(0)
.cpu(),
requires_grad=False,
)
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
+16 -11
View File
@@ -409,13 +409,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
# while the checkpoint uses coarser block-FP8 scales (usually
# 128x128). Build a dedicated, numerically equivalent scale view before
# the generic linear post-load hook replaces the raw checkpoint scales
# with its 128x128 DeepGEMM layout.
checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8)
# 128x128). Build a dedicated, numerically equivalent scale view.
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
if (
gate_up_scale.dtype in checkpoint_scale_dtypes
and down_scale.dtype in checkpoint_scale_dtypes
gate_up_scale.dtype in ue8m0_scale_dtypes
and down_scale.dtype in ue8m0_scale_dtypes
):
gate_up_scale = self._prepare_shared_expert_scale(
deep_gemm,
@@ -501,9 +499,13 @@ class DeepseekV4MegaMoEExperts(nn.Module):
return None
block_m, block_k = block_size
num_k_blocks = (k + block_k - 1) // block_k
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
packed = scale.dtype == torch.int32
expected_shape = (
(mn + block_m - 1) // block_m,
(k + block_k - 1) // block_k,
(mn, (num_k_blocks + 3) // 4)
if packed
else ((mn + block_m - 1) // block_m, num_k_blocks)
)
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
logger.warning(
@@ -518,10 +520,13 @@ class DeepseekV4MegaMoEExperts(nn.Module):
)
return None
scale_fp32 = self._ue8m0_uint8_to_float(scale.view(torch.uint8))
if packed:
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
else:
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
scale_1x32 = (
scale_fp32.repeat_interleave(block_m, dim=0)
.repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32]
self._ue8m0_uint8_to_float(row_scale)
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
.contiguous()
)
# The grouped API is used with a singleton dimension to request the