[Bugfix][MiniMax M3] Share target embeddings with MTP under PP (#58648)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Codex
Woosuk Kwon
Jee Jee Li
parent
2798f66860
commit
a8e069f81b
@@ -1,15 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils
|
||||
|
||||
VOCAB, HIDDEN = 32, 8
|
||||
@@ -108,3 +113,186 @@ def test_target_embedding_provisioning(
|
||||
speculative_config = None if method is None else SimpleNamespace(method=method)
|
||||
config = SimpleNamespace(speculative_config=speculative_config)
|
||||
assert spec_decode_needs_target_embed(config) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pp_size,is_last_rank,expected",
|
||||
[(2, True, True), (2, False, False), (1, True, False)],
|
||||
)
|
||||
def test_mtp_target_embedding_requires_model_opt_in(
|
||||
monkeypatch, pp_size, is_last_rank, expected
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_pp_group",
|
||||
_fake_pp(pp_size, is_last_rank),
|
||||
)
|
||||
config = SimpleNamespace(speculative_config=SimpleNamespace(method="mtp"))
|
||||
assert spec_decode_needs_target_embed(config, include_mtp=True) is expected
|
||||
assert not spec_decode_needs_target_embed(config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def m3_modules():
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("M3 model imports require CUDA or ROCm dependencies")
|
||||
backend = "amd" if current_platform.is_rocm() else "nvidia"
|
||||
root = f"vllm.models.minimax_m3.{backend}"
|
||||
return importlib.import_module(f"{root}.mtp"), importlib.import_module(
|
||||
f"{root}.model"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def m3_config():
|
||||
hf_config = SimpleNamespace(
|
||||
vocab_size=VOCAB,
|
||||
hidden_size=HIDDEN,
|
||||
num_local_experts=0,
|
||||
num_hidden_layers=0,
|
||||
rms_norm_eps=1e-5,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
use_v2_model_runner=True,
|
||||
load_config=SimpleNamespace(load_format="auto"),
|
||||
model_config=SimpleNamespace(hf_text_config=hf_config),
|
||||
speculative_config=SimpleNamespace(
|
||||
method="mtp", draft_model_config=SimpleNamespace(hf_config=hf_config)
|
||||
),
|
||||
quant_config=None,
|
||||
cache_config=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_m3_draft(monkeypatch, m3_modules, m3_config):
|
||||
mtp, _ = m3_modules
|
||||
inner = _inner(_embed())
|
||||
inner.layers = nn.ModuleDict({"0": nn.Linear(1, 1, bias=False)})
|
||||
inner.num_mtp_layers = 1
|
||||
monkeypatch.setattr(mtp, "MiniMaxM3MultiTokenPredictor", lambda **_: inner)
|
||||
monkeypatch.setattr(mtp, "ParallelLMHead", lambda *_, **kw: _embed())
|
||||
monkeypatch.setattr(mtp, "LogitsProcessor", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
mtp, "fused_moe_make_expert_params_mapping", lambda *_, **kw: []
|
||||
)
|
||||
monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(2))
|
||||
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
|
||||
return lambda: mtp.MiniMaxM3MTP(vllm_config=m3_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[None, "model.embed_tokens.weight", "language_model.model.embed_tokens.weight"],
|
||||
)
|
||||
def test_m3_checkpoint_embedding_ownership(make_m3_draft, key):
|
||||
"""Actual M3 loading determines sharing and survives an unrelated reload."""
|
||||
draft = make_m3_draft()
|
||||
assert draft.has_own_embed_tokens
|
||||
weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))]
|
||||
if key is not None:
|
||||
weights.append((key, torch.ones(VOCAB, HIDDEN)))
|
||||
loaded = draft.load_weights(weights)
|
||||
assert ("model.embed_tokens.weight" in loaded) is (key is not None)
|
||||
with disable_mtp_completeness_check():
|
||||
draft.load_weights([("lm_head.weight", torch.zeros(VOCAB, HIDDEN))])
|
||||
assert draft.has_own_embed_tokens is (key is not None)
|
||||
target = _inner(_embed(fill=2.0))
|
||||
original = draft.model.embed_tokens
|
||||
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
|
||||
if key is None:
|
||||
assert draft.model.embed_tokens is target.embed_tokens
|
||||
else:
|
||||
assert draft.model.embed_tokens is original
|
||||
torch.testing.assert_close(original.weight, torch.ones(VOCAB, HIDDEN))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_format", ["dummy", "sharded_state", "runai_streamer_sharded", "tensorizer"]
|
||||
)
|
||||
def test_m3_direct_loader_embedding_ownership(make_m3_draft, m3_config, load_format):
|
||||
"""Directly loaded embeddings survive transfers and share only when equal."""
|
||||
m3_config.load_config.load_format = load_format
|
||||
draft = make_m3_draft()
|
||||
draft.load_state_dict(
|
||||
{key: torch.ones_like(value) for key, value in draft.state_dict().items()}
|
||||
)
|
||||
with disable_mtp_completeness_check():
|
||||
draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))])
|
||||
target = _inner(_embed(fill=2.0))
|
||||
original = draft.model.embed_tokens
|
||||
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
|
||||
assert draft.model.embed_tokens is (
|
||||
target.embed_tokens if load_format == "dummy" else original
|
||||
)
|
||||
if load_format != "dummy":
|
||||
equal_target = _inner(_embed(fill=1.0))
|
||||
eagle_utils.maybe_share_target_embed(draft, draft.model, equal_target)
|
||||
assert draft.model.embed_tokens is equal_target.embed_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_v2,pp_size", [(True, 1), (False, 1), (False, 2)])
|
||||
def test_m3_preserves_legacy_embedding_sharing(
|
||||
monkeypatch, m3_modules, m3_config, make_m3_draft, use_v2, pp_size
|
||||
):
|
||||
mtp, _ = m3_modules
|
||||
m3_config.use_v2_model_runner = use_v2
|
||||
monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(pp_size))
|
||||
draft = make_m3_draft()
|
||||
draft.load_weights(
|
||||
[
|
||||
("model.mtp.layers.0.weight", torch.ones(1, 1)),
|
||||
("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)),
|
||||
]
|
||||
)
|
||||
assert not hasattr(draft, "has_own_embed_tokens")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rank,use_v2,method,expected",
|
||||
[
|
||||
(0, True, "mtp", True),
|
||||
(1, True, "mtp", False),
|
||||
(2, True, "mtp", True),
|
||||
(2, False, "mtp", False),
|
||||
(2, True, None, False),
|
||||
],
|
||||
)
|
||||
def test_m3_target_allocates_embedding_on_required_stage(
|
||||
monkeypatch, m3_modules, m3_config, rank, use_v2, method, expected
|
||||
):
|
||||
_, model = m3_modules
|
||||
pp = lambda: SimpleNamespace(
|
||||
world_size=3, is_first_rank=rank == 0, is_last_rank=rank == 2
|
||||
)
|
||||
monkeypatch.setattr(model, "get_pp_group", pp)
|
||||
monkeypatch.setattr("vllm.distributed.parallel_state.get_pp_group", pp)
|
||||
monkeypatch.setattr(model, "VocabParallelEmbedding", lambda *_, **kw: _embed())
|
||||
monkeypatch.setattr(model, "make_layers", lambda *_, **kw: (0, 0, nn.ModuleList()))
|
||||
monkeypatch.setattr(model, "MiniMAXGemmaRMSNorm", lambda *_, **kw: nn.Identity())
|
||||
m3_config.use_v2_model_runner = use_v2
|
||||
if method is None:
|
||||
m3_config.speculative_config = None
|
||||
target = model.MiniMaxM3Model(vllm_config=m3_config)
|
||||
assert isinstance(target.embed_tokens, nn.Embedding) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initial_embed", [False, True])
|
||||
def test_m3_weight_transfer_updates_embedding_ownership(make_m3_draft, initial_embed):
|
||||
"""Layer transfers retain ownership; newly loaded embeddings become owned."""
|
||||
draft = make_m3_draft()
|
||||
weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))]
|
||||
if initial_embed:
|
||||
weights.append(("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)))
|
||||
draft.load_weights(weights)
|
||||
with disable_mtp_completeness_check():
|
||||
draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))])
|
||||
assert draft.has_own_embed_tokens is initial_embed
|
||||
draft.load_weights(
|
||||
[("model.embed_tokens.weight", torch.full((VOCAB, HIDDEN), 3.0))]
|
||||
)
|
||||
assert draft.has_own_embed_tokens
|
||||
target = _inner(_embed(fill=2.0))
|
||||
original = draft.model.embed_tokens
|
||||
eagle_utils.maybe_share_target_embed(draft, draft.model, target)
|
||||
assert draft.model.embed_tokens is original
|
||||
torch.testing.assert_close(original.weight, torch.full((VOCAB, HIDDEN), 3.0))
|
||||
|
||||
@@ -844,17 +844,24 @@ class PPMissingLayer(torch.nn.Identity):
|
||||
return args[0] if args else next(iter(kwargs.values()))
|
||||
|
||||
|
||||
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the last PP rank needs the target input embedding."""
|
||||
def spec_decode_needs_target_embed(
|
||||
vllm_config: VllmConfig, *, include_mtp: bool = False
|
||||
) -> bool:
|
||||
"""Whether the last PP rank needs the target input embedding.
|
||||
|
||||
MTP targets opt in when their drafter supports sharing this embedding.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.method not in (
|
||||
if speculative_config is None:
|
||||
return False
|
||||
if speculative_config.method not in (
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
):
|
||||
) and not (include_mtp and speculative_config.method == "mtp"):
|
||||
return False
|
||||
pp = get_pp_group()
|
||||
return pp.world_size > 1 and pp.is_last_rank
|
||||
|
||||
@@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import (
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
|
||||
from vllm.models.minimax_m3.amd.indexer_aiter import (
|
||||
@@ -1370,7 +1371,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(
|
||||
vllm_config, include_mtp=vllm_config.use_v2_model_runner
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -164,6 +165,10 @@ class MiniMaxM3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1:
|
||||
# Preserve embeddings from loaders that bypass load_weights.
|
||||
self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy"
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
@@ -333,4 +338,10 @@ class MiniMaxM3MTP(nn.Module):
|
||||
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
|
||||
)
|
||||
|
||||
if hasattr(self, "has_own_embed_tokens"):
|
||||
if "model.embed_tokens.weight" in loaded_params:
|
||||
self.has_own_embed_tokens = True
|
||||
elif is_mtp_completeness_check_enabled():
|
||||
# A complete checkpoint without embeddings borrows the target's.
|
||||
self.has_own_embed_tokens = False
|
||||
return loaded_params
|
||||
|
||||
@@ -68,6 +68,7 @@ from vllm.model_executor.models.utils import (
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
|
||||
from vllm.models.minimax_m3.common.encoder_cudagraph import (
|
||||
@@ -803,7 +804,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(
|
||||
vllm_config, include_mtp=vllm_config.use_v2_model_runner
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -146,6 +147,10 @@ class MiniMaxM3MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1:
|
||||
# Preserve embeddings from loaders that bypass load_weights.
|
||||
self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy"
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
@@ -315,4 +320,10 @@ class MiniMaxM3MTP(nn.Module):
|
||||
f"Failed to load MTP layer {layer_idx} weights from checkpoint."
|
||||
)
|
||||
|
||||
if hasattr(self, "has_own_embed_tokens"):
|
||||
if "model.embed_tokens.weight" in loaded_params:
|
||||
self.has_own_embed_tokens = True
|
||||
elif is_mtp_completeness_check_enabled():
|
||||
# A complete checkpoint without embeddings borrows the target's.
|
||||
self.has_own_embed_tokens = False
|
||||
return loaded_params
|
||||
|
||||
Reference in New Issue
Block a user