[Bugfix][Model][Spec Decode] Defer disposable GLM MTP head (#55442)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
This commit is contained in:
lucamotz
2026-09-22 20:08:27 +00:00
committed by GitHub
parent 71a2392845
commit 276db94fb2
3 changed files with 40 additions and 8 deletions
+26
View File
@@ -5,6 +5,7 @@ from unittest import mock
import pytest import pytest
import torch import torch
import torch.nn as nn
from tests.v1.attention.utils import ( from tests.v1.attention.utils import (
BatchSpec, BatchSpec,
@@ -31,6 +32,31 @@ mimo_7b_dir = "XiaomiMiMo/MiMo-7B-Base"
DEVICE_TYPE = current_platform.device_type DEVICE_TYPE = current_platform.device_type
def test_glm_mtp_defers_lm_head(default_vllm_config):
from vllm.model_executor.models import deepseek_mtp
from vllm.models.glm5next.common import mtp
config = mock.MagicMock(
hidden_size=16,
rms_norm_eps=1e-5,
index_topk=8,
index_kpool=4,
)
vllm_config = mock.MagicMock()
vllm_config.speculative_config.draft_model_config.hf_config = config
vllm_config.scheduler_config.max_num_batched_tokens = 4
with (
mock.patch.object(mtp, "Glm5NextDecoderLayer", return_value=nn.Identity()),
mock.patch.object(deepseek_mtp, "ParallelLMHead") as parallel_lm_head,
mock.patch.object(mtp.current_platform, "device_type", "cpu"),
):
layer = mtp.Glm5NextMultiTokenPredictorLayer(vllm_config, "model.layers.1")
parallel_lm_head.assert_not_called()
assert layer.shared_head.head is None
def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer: def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer:
"""Create an MTP proposer with unified model configuration.""" """Create an MTP proposer with unified model configuration."""
model_config = ModelConfig( model_config = ModelConfig(
+11 -6
View File
@@ -54,15 +54,20 @@ class SharedHead(nn.Module):
config: PretrainedConfig, config: PretrainedConfig,
prefix: str, prefix: str,
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
*,
defer_lm_head: bool = False,
) -> None: ) -> None:
super().__init__() super().__init__()
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.head = ParallelLMHead( if defer_lm_head:
config.vocab_size, self.head = None
config.hidden_size, else:
quant_config=quant_config, self.head = ParallelLMHead(
prefix=maybe_prefix(prefix, "head"), config.vocab_size,
) config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "head"),
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.norm(hidden_states) return self.norm(hidden_states)
+3 -2
View File
@@ -42,7 +42,6 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
assert vllm_config.speculative_config is not None assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config self.config = config
quant_config = vllm_config.quant_config
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -66,7 +65,9 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
device=current_platform.device_type, device=current_platform.device_type,
) )
self.shared_head = SharedHead( self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config config=config,
prefix=prefix,
defer_lm_head=True,
) )
# MTP layers sit past the base model's hidden layers; parse the index # MTP layers sit past the base model's hidden layers; parse the index
# from the prefix (e.g. "...layers.32") so the decoder builds an MLA # from the prefix (e.g. "...layers.32") so the decoder builds an MLA