[Model][DeepSeek-V4] Make MegaMoE shared-expert finalize independent of linear post-load order (#59927)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
34051ad714
commit
4a30c4cad0
@@ -9,6 +9,9 @@ import torch
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
@@ -23,6 +26,7 @@ from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -473,19 +477,23 @@ def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,intermediate_size,block_size,mxfp8",
|
||||
"hidden_size,intermediate_size,block_size,mxfp8,packed",
|
||||
[
|
||||
pytest.param(128, 512, 128, False, id="aligned-block128"),
|
||||
pytest.param(128, 512, 128, True, id="aligned-block128-mxfp8"),
|
||||
pytest.param(5120, 2304, 32, False, id="deepseek-v41-flash"),
|
||||
pytest.param(128, 2304, 128, False, id="native-block128"),
|
||||
pytest.param(5120, 2304, 32, True, id="deepseek-v41-flash-mxfp8"),
|
||||
pytest.param(128, 512, 128, False, False, id="aligned-block128"),
|
||||
pytest.param(128, 512, 128, False, True, id="aligned-block128-packed"),
|
||||
pytest.param(128, 512, 128, True, False, id="aligned-block128-mxfp8"),
|
||||
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
|
||||
pytest.param(128, 2304, 128, False, False, id="native-block128"),
|
||||
pytest.param(128, 2304, 128, False, True, id="native-block128-packed"),
|
||||
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash-mxfp8"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8, packed
|
||||
):
|
||||
"""Shared fusion preserves checkpoint weights and scales at native widths."""
|
||||
if packed and not is_deep_gemm_supported():
|
||||
pytest.skip("Packing the shared scales requires DeepGEMM")
|
||||
|
||||
class FakeDeepGemm:
|
||||
transformed_dims: list[tuple[int, int]] = []
|
||||
@@ -593,6 +601,18 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
.repeat_interleave(block_k // 32, dim=1)
|
||||
)
|
||||
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
|
||||
if packed:
|
||||
linear.weight_scale_inv = torch.nn.Parameter(
|
||||
deepgemm_post_process_weight_scale_block(
|
||||
scale.data.cuda().unsqueeze(0),
|
||||
*linear.weight.shape,
|
||||
quant_block_shape=linear.weight_block_size,
|
||||
num_groups=1,
|
||||
)
|
||||
.squeeze(0)
|
||||
.cpu(),
|
||||
requires_grad=False,
|
||||
)
|
||||
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
|
||||
|
||||
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
|
||||
|
||||
@@ -409,13 +409,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
|
||||
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
|
||||
# while the checkpoint uses coarser block-FP8 scales (usually
|
||||
# 128x128). Build a dedicated, numerically equivalent scale view before
|
||||
# the generic linear post-load hook replaces the raw checkpoint scales
|
||||
# with its 128x128 DeepGEMM layout.
|
||||
checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8)
|
||||
# 128x128). Build a dedicated, numerically equivalent scale view.
|
||||
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
|
||||
if (
|
||||
gate_up_scale.dtype in checkpoint_scale_dtypes
|
||||
and down_scale.dtype in checkpoint_scale_dtypes
|
||||
gate_up_scale.dtype in ue8m0_scale_dtypes
|
||||
and down_scale.dtype in ue8m0_scale_dtypes
|
||||
):
|
||||
gate_up_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
@@ -501,9 +499,13 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
return None
|
||||
|
||||
block_m, block_k = block_size
|
||||
num_k_blocks = (k + block_k - 1) // block_k
|
||||
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
|
||||
packed = scale.dtype == torch.int32
|
||||
expected_shape = (
|
||||
(mn + block_m - 1) // block_m,
|
||||
(k + block_k - 1) // block_k,
|
||||
(mn, (num_k_blocks + 3) // 4)
|
||||
if packed
|
||||
else ((mn + block_m - 1) // block_m, num_k_blocks)
|
||||
)
|
||||
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
|
||||
logger.warning(
|
||||
@@ -518,10 +520,13 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
)
|
||||
return None
|
||||
|
||||
scale_fp32 = self._ue8m0_uint8_to_float(scale.view(torch.uint8))
|
||||
if packed:
|
||||
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
|
||||
else:
|
||||
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
|
||||
scale_1x32 = (
|
||||
scale_fp32.repeat_interleave(block_m, dim=0)
|
||||
.repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32]
|
||||
self._ue8m0_uint8_to_float(row_scale)
|
||||
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
|
||||
.contiguous()
|
||||
)
|
||||
# The grouped API is used with a singleton dimension to request the
|
||||
|
||||
Reference in New Issue
Block a user