[Bugfix][MiniMax M3] Share target embeddings with MTP under PP (#58648)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-09-29 20:04:10 -07:00
committed by GitHub
co-authored by Codex Woosuk Kwon Jee Jee Li
parent 2798f66860
commit a8e069f81b
6 changed files with 229 additions and 6 deletions
@@ -1,15 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
from vllm.model_executor.models.utils import (
PPMissingLayer,
spec_decode_needs_target_embed,
)
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils
VOCAB, HIDDEN = 32, 8
@@ -108,3 +113,186 @@ def test_target_embedding_provisioning(
speculative_config = None if method is None else SimpleNamespace(method=method)
config = SimpleNamespace(speculative_config=speculative_config)
assert spec_decode_needs_target_embed(config) is expected
@pytest.mark.parametrize(
"pp_size,is_last_rank,expected",
[(2, True, True), (2, False, False), (1, True, False)],
)
def test_mtp_target_embedding_requires_model_opt_in(
monkeypatch, pp_size, is_last_rank, expected
):
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
_fake_pp(pp_size, is_last_rank),
)
config = SimpleNamespace(speculative_config=SimpleNamespace(method="mtp"))
assert spec_decode_needs_target_embed(config, include_mtp=True) is expected
assert not spec_decode_needs_target_embed(config)
@pytest.fixture
def m3_modules():
if not current_platform.is_cuda_alike():
pytest.skip("M3 model imports require CUDA or ROCm dependencies")
backend = "amd" if current_platform.is_rocm() else "nvidia"
root = f"vllm.models.minimax_m3.{backend}"
return importlib.import_module(f"{root}.mtp"), importlib.import_module(
f"{root}.model"
)
@pytest.fixture
def m3_config():
hf_config = SimpleNamespace(
vocab_size=VOCAB,
hidden_size=HIDDEN,
num_local_experts=0,
num_hidden_layers=0,
rms_norm_eps=1e-5,
)
return SimpleNamespace(
use_v2_model_runner=True,
load_config=SimpleNamespace(load_format="auto"),
model_config=SimpleNamespace(hf_text_config=hf_config),
speculative_config=SimpleNamespace(
method="mtp", draft_model_config=SimpleNamespace(hf_config=hf_config)
),
quant_config=None,
cache_config=None,
)
@pytest.fixture
def make_m3_draft(monkeypatch, m3_modules, m3_config):
mtp, _ = m3_modules
inner = _inner(_embed())
inner.layers = nn.ModuleDict({"0": nn.Linear(1, 1, bias=False)})
inner.num_mtp_layers = 1
monkeypatch.setattr(mtp, "MiniMaxM3MultiTokenPredictor", lambda **_: inner)
monkeypatch.setattr(mtp, "ParallelLMHead", lambda *_, **kw: _embed())
monkeypatch.setattr(mtp, "LogitsProcessor", lambda *_: None)
monkeypatch.setattr(
mtp, "fused_moe_make_expert_params_mapping", lambda *_, **kw: []
)
monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(2))
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
return lambda: mtp.MiniMaxM3MTP(vllm_config=m3_config)
@pytest.mark.parametrize(
"key",
[None, "model.embed_tokens.weight", "language_model.model.embed_tokens.weight"],
)
def test_m3_checkpoint_embedding_ownership(make_m3_draft, key):
"""Actual M3 loading determines sharing and survives an unrelated reload."""
draft = make_m3_draft()
assert draft.has_own_embed_tokens
weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))]
if key is not None:
weights.append((key, torch.ones(VOCAB, HIDDEN)))
loaded = draft.load_weights(weights)
assert ("model.embed_tokens.weight" in loaded) is (key is not None)
with disable_mtp_completeness_check():
draft.load_weights([("lm_head.weight", torch.zeros(VOCAB, HIDDEN))])
assert draft.has_own_embed_tokens is (key is not None)
target = _inner(_embed(fill=2.0))
original = draft.model.embed_tokens
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
if key is None:
assert draft.model.embed_tokens is target.embed_tokens
else:
assert draft.model.embed_tokens is original
torch.testing.assert_close(original.weight, torch.ones(VOCAB, HIDDEN))
@pytest.mark.parametrize(
"load_format", ["dummy", "sharded_state", "runai_streamer_sharded", "tensorizer"]
)
def test_m3_direct_loader_embedding_ownership(make_m3_draft, m3_config, load_format):
"""Directly loaded embeddings survive transfers and share only when equal."""
m3_config.load_config.load_format = load_format
draft = make_m3_draft()
draft.load_state_dict(
{key: torch.ones_like(value) for key, value in draft.state_dict().items()}
)
with disable_mtp_completeness_check():
draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))])
target = _inner(_embed(fill=2.0))
original = draft.model.embed_tokens
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
assert draft.model.embed_tokens is (
target.embed_tokens if load_format == "dummy" else original
)
if load_format != "dummy":
equal_target = _inner(_embed(fill=1.0))
eagle_utils.maybe_share_target_embed(draft, draft.model, equal_target)
assert draft.model.embed_tokens is equal_target.embed_tokens
@pytest.mark.parametrize("use_v2,pp_size", [(True, 1), (False, 1), (False, 2)])
def test_m3_preserves_legacy_embedding_sharing(
monkeypatch, m3_modules, m3_config, make_m3_draft, use_v2, pp_size
):
mtp, _ = m3_modules
m3_config.use_v2_model_runner = use_v2
monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(pp_size))
draft = make_m3_draft()
draft.load_weights(
[
("model.mtp.layers.0.weight", torch.ones(1, 1)),
("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)),
]
)
assert not hasattr(draft, "has_own_embed_tokens")
@pytest.mark.parametrize(
"rank,use_v2,method,expected",
[
(0, True, "mtp", True),
(1, True, "mtp", False),
(2, True, "mtp", True),
(2, False, "mtp", False),
(2, True, None, False),
],
)
def test_m3_target_allocates_embedding_on_required_stage(
monkeypatch, m3_modules, m3_config, rank, use_v2, method, expected
):
_, model = m3_modules
pp = lambda: SimpleNamespace(
world_size=3, is_first_rank=rank == 0, is_last_rank=rank == 2
)
monkeypatch.setattr(model, "get_pp_group", pp)
monkeypatch.setattr("vllm.distributed.parallel_state.get_pp_group", pp)
monkeypatch.setattr(model, "VocabParallelEmbedding", lambda *_, **kw: _embed())
monkeypatch.setattr(model, "make_layers", lambda *_, **kw: (0, 0, nn.ModuleList()))
monkeypatch.setattr(model, "MiniMAXGemmaRMSNorm", lambda *_, **kw: nn.Identity())
m3_config.use_v2_model_runner = use_v2
if method is None:
m3_config.speculative_config = None
target = model.MiniMaxM3Model(vllm_config=m3_config)
assert isinstance(target.embed_tokens, nn.Embedding) is expected
@pytest.mark.parametrize("initial_embed", [False, True])
def test_m3_weight_transfer_updates_embedding_ownership(make_m3_draft, initial_embed):
"""Layer transfers retain ownership; newly loaded embeddings become owned."""
draft = make_m3_draft()
weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))]
if initial_embed:
weights.append(("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)))
draft.load_weights(weights)
with disable_mtp_completeness_check():
draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))])
assert draft.has_own_embed_tokens is initial_embed
draft.load_weights(
[("model.embed_tokens.weight", torch.full((VOCAB, HIDDEN), 3.0))]
)
assert draft.has_own_embed_tokens
target = _inner(_embed(fill=2.0))
original = draft.model.embed_tokens
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
assert draft.model.embed_tokens is original
torch.testing.assert_close(original.weight, torch.full((VOCAB, HIDDEN), 3.0))
+11 -4
View File
@@ -844,17 +844,24 @@ class PPMissingLayer(torch.nn.Identity):
return args[0] if args else next(iter(kwargs.values()))
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
"""Whether the last PP rank needs the target input embedding."""
def spec_decode_needs_target_embed(
vllm_config: VllmConfig, *, include_mtp: bool = False
) -> bool:
"""Whether the last PP rank needs the target input embedding.
MTP targets opt in when their drafter supports sharing this embedding.
"""
from vllm.distributed.parallel_state import get_pp_group
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.method not in (
if speculative_config is None:
return False
if speculative_config.method not in (
"eagle",
"eagle3",
"dflash",
"dspark",
):
) and not (include_mtp and speculative_config.method == "mtp"):
return False
pp = get_pp_group()
return pp.world_size > 1 and pp.is_last_rank
+4 -1
View File
@@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
from vllm.models.minimax_m3.amd.indexer_aiter import (
@@ -1370,7 +1371,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(
vllm_config, include_mtp=vllm_config.use_v2_model_runner
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
+11
View File
@@ -24,6 +24,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -164,6 +165,10 @@ class MiniMaxM3MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1:
# Preserve embeddings from loaders that bypass load_weights.
self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy"
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
@@ -333,4 +338,10 @@ class MiniMaxM3MTP(nn.Module):
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
)
if hasattr(self, "has_own_embed_tokens"):
if "model.embed_tokens.weight" in loaded_params:
self.has_own_embed_tokens = True
elif is_mtp_completeness_check_enabled():
# A complete checkpoint without embeddings borrows the target's.
self.has_own_embed_tokens = False
return loaded_params
+4 -1
View File
@@ -68,6 +68,7 @@ from vllm.model_executor.models.utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
from vllm.models.minimax_m3.common.encoder_cudagraph import (
@@ -803,7 +804,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(
vllm_config, include_mtp=vllm_config.use_v2_model_runner
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
+11
View File
@@ -8,6 +8,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -146,6 +147,10 @@ class MiniMaxM3MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1:
# Preserve embeddings from loaders that bypass load_weights.
self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy"
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
@@ -315,4 +320,10 @@ class MiniMaxM3MTP(nn.Module):
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
)
if hasattr(self, "has_own_embed_tokens"):
if "model.embed_tokens.weight" in loaded_params:
self.has_own_embed_tokens = True
elif is_mtp_completeness_check_enabled():
# A complete checkpoint without embeddings borrows the target's.
self.has_own_embed_tokens = False
return loaded_params