From a8e069f81b4842723a04a84fad2cd5f757820af3 Mon Sep 17 00:00:00 2001 From: xiaozhoupy Date: Wed, 30 Sep 2026 11:04:10 +0800 Subject: [PATCH] [Bugfix][MiniMax M3] Share target embeddings with MTP under PP (#58648) Signed-off-by: Peiyuan Zhou Co-authored-by: Codex Co-authored-by: Woosuk Kwon Co-authored-by: Jee Jee Li --- .../test_spec_decode_embed_sharing_pp.py | 188 ++++++++++++++++++ vllm/model_executor/models/utils.py | 15 +- vllm/models/minimax_m3/amd/model.py | 5 +- vllm/models/minimax_m3/amd/mtp.py | 11 + vllm/models/minimax_m3/nvidia/model.py | 5 +- vllm/models/minimax_m3/nvidia/mtp.py | 11 + 6 files changed, 229 insertions(+), 6 deletions(-) diff --git a/tests/v1/worker/test_spec_decode_embed_sharing_pp.py b/tests/v1/worker/test_spec_decode_embed_sharing_pp.py index 1a1d96a6b4..36b1268ca3 100644 --- a/tests/v1/worker/test_spec_decode_embed_sharing_pp.py +++ b/tests/v1/worker/test_spec_decode_embed_sharing_pp.py @@ -1,15 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib from types import SimpleNamespace import pytest import torch import torch.nn as nn +from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, +) from vllm.model_executor.models.utils import ( PPMissingLayer, spec_decode_needs_target_embed, ) +from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils VOCAB, HIDDEN = 32, 8 @@ -108,3 +113,186 @@ def test_target_embedding_provisioning( speculative_config = None if method is None else SimpleNamespace(method=method) config = SimpleNamespace(speculative_config=speculative_config) assert spec_decode_needs_target_embed(config) is expected + + +@pytest.mark.parametrize( + "pp_size,is_last_rank,expected", + [(2, True, True), (2, False, False), (1, True, False)], +) +def test_mtp_target_embedding_requires_model_opt_in( + monkeypatch, pp_size, is_last_rank, expected +): + monkeypatch.setattr( + "vllm.distributed.parallel_state.get_pp_group", + _fake_pp(pp_size, is_last_rank), + ) + config = SimpleNamespace(speculative_config=SimpleNamespace(method="mtp")) + assert spec_decode_needs_target_embed(config, include_mtp=True) is expected + assert not spec_decode_needs_target_embed(config) + + +@pytest.fixture +def m3_modules(): + if not current_platform.is_cuda_alike(): + pytest.skip("M3 model imports require CUDA or ROCm dependencies") + backend = "amd" if current_platform.is_rocm() else "nvidia" + root = f"vllm.models.minimax_m3.{backend}" + return importlib.import_module(f"{root}.mtp"), importlib.import_module( + f"{root}.model" + ) + + +@pytest.fixture +def m3_config(): + hf_config = SimpleNamespace( + vocab_size=VOCAB, + hidden_size=HIDDEN, + num_local_experts=0, + num_hidden_layers=0, + rms_norm_eps=1e-5, + ) + return SimpleNamespace( + use_v2_model_runner=True, + load_config=SimpleNamespace(load_format="auto"), + model_config=SimpleNamespace(hf_text_config=hf_config), + speculative_config=SimpleNamespace( + method="mtp", draft_model_config=SimpleNamespace(hf_config=hf_config) + ), + quant_config=None, + cache_config=None, + ) + + +@pytest.fixture +def make_m3_draft(monkeypatch, m3_modules, m3_config): + mtp, _ = m3_modules + inner = _inner(_embed()) + inner.layers = nn.ModuleDict({"0": nn.Linear(1, 1, bias=False)}) + inner.num_mtp_layers = 1 + monkeypatch.setattr(mtp, "MiniMaxM3MultiTokenPredictor", lambda **_: inner) + monkeypatch.setattr(mtp, "ParallelLMHead", lambda *_, **kw: _embed()) + monkeypatch.setattr(mtp, "LogitsProcessor", lambda *_: None) + monkeypatch.setattr( + mtp, "fused_moe_make_expert_params_mapping", lambda *_, **kw: [] + ) + monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(2)) + monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2)) + return lambda: mtp.MiniMaxM3MTP(vllm_config=m3_config) + + +@pytest.mark.parametrize( + "key", + [None, "model.embed_tokens.weight", "language_model.model.embed_tokens.weight"], +) +def test_m3_checkpoint_embedding_ownership(make_m3_draft, key): + """Actual M3 loading determines sharing and survives an unrelated reload.""" + draft = make_m3_draft() + assert draft.has_own_embed_tokens + weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))] + if key is not None: + weights.append((key, torch.ones(VOCAB, HIDDEN))) + loaded = draft.load_weights(weights) + assert ("model.embed_tokens.weight" in loaded) is (key is not None) + with disable_mtp_completeness_check(): + draft.load_weights([("lm_head.weight", torch.zeros(VOCAB, HIDDEN))]) + assert draft.has_own_embed_tokens is (key is not None) + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + if key is None: + assert draft.model.embed_tokens is target.embed_tokens + else: + assert draft.model.embed_tokens is original + torch.testing.assert_close(original.weight, torch.ones(VOCAB, HIDDEN)) + + +@pytest.mark.parametrize( + "load_format", ["dummy", "sharded_state", "runai_streamer_sharded", "tensorizer"] +) +def test_m3_direct_loader_embedding_ownership(make_m3_draft, m3_config, load_format): + """Directly loaded embeddings survive transfers and share only when equal.""" + m3_config.load_config.load_format = load_format + draft = make_m3_draft() + draft.load_state_dict( + {key: torch.ones_like(value) for key, value in draft.state_dict().items()} + ) + with disable_mtp_completeness_check(): + draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))]) + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + assert draft.model.embed_tokens is ( + target.embed_tokens if load_format == "dummy" else original + ) + if load_format != "dummy": + equal_target = _inner(_embed(fill=1.0)) + eagle_utils.maybe_share_target_embed(draft, draft.model, equal_target) + assert draft.model.embed_tokens is equal_target.embed_tokens + + +@pytest.mark.parametrize("use_v2,pp_size", [(True, 1), (False, 1), (False, 2)]) +def test_m3_preserves_legacy_embedding_sharing( + monkeypatch, m3_modules, m3_config, make_m3_draft, use_v2, pp_size +): + mtp, _ = m3_modules + m3_config.use_v2_model_runner = use_v2 + monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(pp_size)) + draft = make_m3_draft() + draft.load_weights( + [ + ("model.mtp.layers.0.weight", torch.ones(1, 1)), + ("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)), + ] + ) + assert not hasattr(draft, "has_own_embed_tokens") + + +@pytest.mark.parametrize( + "rank,use_v2,method,expected", + [ + (0, True, "mtp", True), + (1, True, "mtp", False), + (2, True, "mtp", True), + (2, False, "mtp", False), + (2, True, None, False), + ], +) +def test_m3_target_allocates_embedding_on_required_stage( + monkeypatch, m3_modules, m3_config, rank, use_v2, method, expected +): + _, model = m3_modules + pp = lambda: SimpleNamespace( + world_size=3, is_first_rank=rank == 0, is_last_rank=rank == 2 + ) + monkeypatch.setattr(model, "get_pp_group", pp) + monkeypatch.setattr("vllm.distributed.parallel_state.get_pp_group", pp) + monkeypatch.setattr(model, "VocabParallelEmbedding", lambda *_, **kw: _embed()) + monkeypatch.setattr(model, "make_layers", lambda *_, **kw: (0, 0, nn.ModuleList())) + monkeypatch.setattr(model, "MiniMAXGemmaRMSNorm", lambda *_, **kw: nn.Identity()) + m3_config.use_v2_model_runner = use_v2 + if method is None: + m3_config.speculative_config = None + target = model.MiniMaxM3Model(vllm_config=m3_config) + assert isinstance(target.embed_tokens, nn.Embedding) is expected + + +@pytest.mark.parametrize("initial_embed", [False, True]) +def test_m3_weight_transfer_updates_embedding_ownership(make_m3_draft, initial_embed): + """Layer transfers retain ownership; newly loaded embeddings become owned.""" + draft = make_m3_draft() + weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))] + if initial_embed: + weights.append(("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN))) + draft.load_weights(weights) + with disable_mtp_completeness_check(): + draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))]) + assert draft.has_own_embed_tokens is initial_embed + draft.load_weights( + [("model.embed_tokens.weight", torch.full((VOCAB, HIDDEN), 3.0))] + ) + assert draft.has_own_embed_tokens + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + assert draft.model.embed_tokens is original + torch.testing.assert_close(original.weight, torch.full((VOCAB, HIDDEN), 3.0)) diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index e762ad1c47..c6498319cd 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -844,17 +844,24 @@ class PPMissingLayer(torch.nn.Identity): return args[0] if args else next(iter(kwargs.values())) -def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool: - """Whether the last PP rank needs the target input embedding.""" +def spec_decode_needs_target_embed( + vllm_config: VllmConfig, *, include_mtp: bool = False +) -> bool: + """Whether the last PP rank needs the target input embedding. + + MTP targets opt in when their drafter supports sharing this embedding. + """ from vllm.distributed.parallel_state import get_pp_group speculative_config = vllm_config.speculative_config - if speculative_config is None or speculative_config.method not in ( + if speculative_config is None: + return False + if speculative_config.method not in ( "eagle", "eagle3", "dflash", "dspark", - ): + ) and not (include_mtp and speculative_config.method == "mtp"): return False pp = get_pp_group() return pp.world_size > 1 and pp.is_last_rank diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 40daa4a6ee..9105f97af8 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import ( make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, + spec_decode_needs_target_embed, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model from vllm.models.minimax_m3.amd.indexer_aiter import ( @@ -1370,7 +1371,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): self.vocab_size = config.vocab_size - if get_pp_group().is_first_rank: + if get_pp_group().is_first_rank or spec_decode_needs_target_embed( + vllm_config, include_mtp=vllm_config.use_v2_model_runner + ): self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, diff --git a/vllm/models/minimax_m3/amd/mtp.py b/vllm/models/minimax_m3/amd/mtp.py index cfb26d7948..2f7dd8a831 100644 --- a/vllm/models/minimax_m3/amd/mtp.py +++ b/vllm/models/minimax_m3/amd/mtp.py @@ -24,6 +24,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig +from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -164,6 +165,10 @@ class MiniMaxM3MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1: + # Preserve embeddings from loaders that bypass load_weights. + self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy" + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.quant_config = vllm_config.quant_config @@ -333,4 +338,10 @@ class MiniMaxM3MTP(nn.Module): f"Failed to load MTP layer {layer_idx} weights from checkpoint." ) + if hasattr(self, "has_own_embed_tokens"): + if "model.embed_tokens.weight" in loaded_params: + self.has_own_embed_tokens = True + elif is_mtp_completeness_check_enabled(): + # A complete checkpoint without embeddings borrows the target's. + self.has_own_embed_tokens = False return loaded_params diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index c16918bd37..d1c0a01c40 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -68,6 +68,7 @@ from vllm.model_executor.models.utils import ( make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, + spec_decode_needs_target_embed, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model from vllm.models.minimax_m3.common.encoder_cudagraph import ( @@ -803,7 +804,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): self.vocab_size = config.vocab_size - if get_pp_group().is_first_rank: + if get_pp_group().is_first_rank or spec_decode_needs_target_embed( + vllm_config, include_mtp=vllm_config.use_v2_model_runner + ): self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, diff --git a/vllm/models/minimax_m3/nvidia/mtp.py b/vllm/models/minimax_m3/nvidia/mtp.py index 832c872a10..9c10a52b7c 100644 --- a/vllm/models/minimax_m3/nvidia/mtp.py +++ b/vllm/models/minimax_m3/nvidia/mtp.py @@ -8,6 +8,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig +from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -146,6 +147,10 @@ class MiniMaxM3MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1: + # Preserve embeddings from loaders that bypass load_weights. + self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy" + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.quant_config = vllm_config.quant_config @@ -315,4 +320,10 @@ class MiniMaxM3MTP(nn.Module): f"Failed to load MTP layer {layer_idx} weights from checkpoint." ) + if hasattr(self, "has_own_embed_tokens"): + if "model.embed_tokens.weight" in loaded_params: + self.has_own_embed_tokens = True + elif is_mtp_completeness_check_enabled(): + # A complete checkpoint without embeddings borrows the target's. + self.has_own_embed_tokens = False return loaded_params