diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index 8bc791e963..e3803509e3 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -5,6 +5,7 @@ from unittest import mock import pytest import torch +import torch.nn as nn from tests.v1.attention.utils import ( BatchSpec, @@ -31,6 +32,31 @@ mimo_7b_dir = "XiaomiMiMo/MiMo-7B-Base" DEVICE_TYPE = current_platform.device_type +def test_glm_mtp_defers_lm_head(default_vllm_config): + from vllm.model_executor.models import deepseek_mtp + from vllm.models.glm5next.common import mtp + + config = mock.MagicMock( + hidden_size=16, + rms_norm_eps=1e-5, + index_topk=8, + index_kpool=4, + ) + vllm_config = mock.MagicMock() + vllm_config.speculative_config.draft_model_config.hf_config = config + vllm_config.scheduler_config.max_num_batched_tokens = 4 + + with ( + mock.patch.object(mtp, "Glm5NextDecoderLayer", return_value=nn.Identity()), + mock.patch.object(deepseek_mtp, "ParallelLMHead") as parallel_lm_head, + mock.patch.object(mtp.current_platform, "device_type", "cpu"), + ): + layer = mtp.Glm5NextMultiTokenPredictorLayer(vllm_config, "model.layers.1") + + parallel_lm_head.assert_not_called() + assert layer.shared_head.head is None + + def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer: """Create an MTP proposer with unified model configuration.""" model_config = ModelConfig( diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 36f0a240bc..581329a4ac 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -54,15 +54,20 @@ class SharedHead(nn.Module): config: PretrainedConfig, prefix: str, quant_config: QuantizationConfig | None = None, + *, + defer_lm_head: bool = False, ) -> None: super().__init__() self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "head"), - ) + if defer_lm_head: + self.head = None + else: + self.head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "head"), + ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.norm(hidden_states) diff --git a/vllm/models/glm5next/common/mtp.py b/vllm/models/glm5next/common/mtp.py index 33a5402b8f..8361a053b8 100644 --- a/vllm/models/glm5next/common/mtp.py +++ b/vllm/models/glm5next/common/mtp.py @@ -42,7 +42,6 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): assert vllm_config.speculative_config is not None config = vllm_config.speculative_config.draft_model_config.hf_config self.config = config - quant_config = vllm_config.quant_config self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -66,7 +65,9 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): device=current_platform.device_type, ) self.shared_head = SharedHead( - config=config, prefix=prefix, quant_config=quant_config + config=config, + prefix=prefix, + defer_lm_head=True, ) # MTP layers sit past the base model's hidden layers; parse the index # from the prefix (e.g. "...layers.32") so the decoder builds an MLA