From 4a30c4cad01401d778adb03b029d9da767c59fed Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Mon, 5 Oct 2026 16:39:49 +0800 Subject: [PATCH] [Model][DeepSeek-V4] Make MegaMoE shared-expert finalize independent of linear post-load order (#59927) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- tests/models/test_deepseek_v4_mega_moe.py | 34 ++++++++++++++++++----- vllm/models/deepseek_v4/nvidia/model.py | 27 ++++++++++-------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 9ee0212fba..104ad8e895 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -9,6 +9,9 @@ import torch from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( bind_routed_experts_capturer, ) +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_weight_scale_block, +) from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4ForCausalLM, @@ -23,6 +26,7 @@ from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE from vllm.platforms import current_platform from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config +from vllm.utils.deep_gemm import is_deep_gemm_supported from vllm.utils.torch_utils import set_default_torch_dtype pytestmark = pytest.mark.skipif( @@ -473,19 +477,23 @@ def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions( @pytest.mark.parametrize( - "hidden_size,intermediate_size,block_size,mxfp8", + "hidden_size,intermediate_size,block_size,mxfp8,packed", [ - pytest.param(128, 512, 128, False, id="aligned-block128"), - pytest.param(128, 512, 128, True, id="aligned-block128-mxfp8"), - pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"), - pytest.param(128, 2304, 128, False, id="native-block128"), - pytest.param(5120, 2304, 32, True, id="deepseek-v41-flash-mxfp8"), + pytest.param(128, 512, 128, False, False, id="aligned-block128"), + pytest.param(128, 512, 128, False, True, id="aligned-block128-packed"), + pytest.param(128, 512, 128, True, False, id="aligned-block128-mxfp8"), + pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"), + pytest.param(128, 2304, 128, False, False, id="native-block128"), + pytest.param(128, 2304, 128, False, True, id="native-block128-packed"), + pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash-mxfp8"), ], ) def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights( - monkeypatch, hidden_size, intermediate_size, block_size, mxfp8 + monkeypatch, hidden_size, intermediate_size, block_size, mxfp8, packed ): """Shared fusion preserves checkpoint weights and scales at native widths.""" + if packed and not is_deep_gemm_supported(): + pytest.skip("Packing the shared scales requires DeepGEMM") class FakeDeepGemm: transformed_dims: list[tuple[int, int]] = [] @@ -593,6 +601,18 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights( .repeat_interleave(block_k // 32, dim=1) ) originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32)) + if packed: + linear.weight_scale_inv = torch.nn.Parameter( + deepgemm_post_process_weight_scale_block( + scale.data.cuda().unsqueeze(0), + *linear.weight.shape, + quant_block_shape=linear.weight_block_size, + num_groups=1, + ) + .squeeze(0) + .cpu(), + requires_grad=False, + ) monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm) original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr() diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index a004120804..b9996d007c 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -409,13 +409,11 @@ class DeepseekV4MegaMoEExperts(nn.Module): # MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row, # while the checkpoint uses coarser block-FP8 scales (usually - # 128x128). Build a dedicated, numerically equivalent scale view before - # the generic linear post-load hook replaces the raw checkpoint scales - # with its 128x128 DeepGEMM layout. - checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8) + # 128x128). Build a dedicated, numerically equivalent scale view. + ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32) if ( - gate_up_scale.dtype in checkpoint_scale_dtypes - and down_scale.dtype in checkpoint_scale_dtypes + gate_up_scale.dtype in ue8m0_scale_dtypes + and down_scale.dtype in ue8m0_scale_dtypes ): gate_up_scale = self._prepare_shared_expert_scale( deep_gemm, @@ -501,9 +499,13 @@ class DeepseekV4MegaMoEExperts(nn.Module): return None block_m, block_k = block_size + num_k_blocks = (k + block_k - 1) // block_k + # The linear hook's DeepGEMM layout packs four k-block bytes per int32. + packed = scale.dtype == torch.int32 expected_shape = ( - (mn + block_m - 1) // block_m, - (k + block_k - 1) // block_k, + (mn, (num_k_blocks + 3) // 4) + if packed + else ((mn + block_m - 1) // block_m, num_k_blocks) ) if block_k % 32 != 0 or tuple(scale.shape) != expected_shape: logger.warning( @@ -518,10 +520,13 @@ class DeepseekV4MegaMoEExperts(nn.Module): ) return None - scale_fp32 = self._ue8m0_uint8_to_float(scale.view(torch.uint8)) + if packed: + row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks] + else: + row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn] scale_1x32 = ( - scale_fp32.repeat_interleave(block_m, dim=0) - .repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32] + self._ue8m0_uint8_to_float(row_scale) + .repeat_interleave(block_k // 32, dim=1)[:, : k // 32] .contiguous() ) # The grouped API is used with a singleton dimension to request the