[Bugfix][Model][Spec Decode] Defer disposable GLM MTP head (#55442)
Signed-off-by: Luca Motz <luca.motz@icloud.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from unittest import mock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
|
||||||
from tests.v1.attention.utils import (
|
from tests.v1.attention.utils import (
|
||||||
BatchSpec,
|
BatchSpec,
|
||||||
@@ -31,6 +32,31 @@ mimo_7b_dir = "XiaomiMiMo/MiMo-7B-Base"
|
|||||||
DEVICE_TYPE = current_platform.device_type
|
DEVICE_TYPE = current_platform.device_type
|
||||||
|
|
||||||
|
|
||||||
|
def test_glm_mtp_defers_lm_head(default_vllm_config):
|
||||||
|
from vllm.model_executor.models import deepseek_mtp
|
||||||
|
from vllm.models.glm5next.common import mtp
|
||||||
|
|
||||||
|
config = mock.MagicMock(
|
||||||
|
hidden_size=16,
|
||||||
|
rms_norm_eps=1e-5,
|
||||||
|
index_topk=8,
|
||||||
|
index_kpool=4,
|
||||||
|
)
|
||||||
|
vllm_config = mock.MagicMock()
|
||||||
|
vllm_config.speculative_config.draft_model_config.hf_config = config
|
||||||
|
vllm_config.scheduler_config.max_num_batched_tokens = 4
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch.object(mtp, "Glm5NextDecoderLayer", return_value=nn.Identity()),
|
||||||
|
mock.patch.object(deepseek_mtp, "ParallelLMHead") as parallel_lm_head,
|
||||||
|
mock.patch.object(mtp.current_platform, "device_type", "cpu"),
|
||||||
|
):
|
||||||
|
layer = mtp.Glm5NextMultiTokenPredictorLayer(vllm_config, "model.layers.1")
|
||||||
|
|
||||||
|
parallel_lm_head.assert_not_called()
|
||||||
|
assert layer.shared_head.head is None
|
||||||
|
|
||||||
|
|
||||||
def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer:
|
def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer:
|
||||||
"""Create an MTP proposer with unified model configuration."""
|
"""Create an MTP proposer with unified model configuration."""
|
||||||
model_config = ModelConfig(
|
model_config = ModelConfig(
|
||||||
|
|||||||
@@ -54,15 +54,20 @@ class SharedHead(nn.Module):
|
|||||||
config: PretrainedConfig,
|
config: PretrainedConfig,
|
||||||
prefix: str,
|
prefix: str,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
|
*,
|
||||||
|
defer_lm_head: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.head = ParallelLMHead(
|
if defer_lm_head:
|
||||||
config.vocab_size,
|
self.head = None
|
||||||
config.hidden_size,
|
else:
|
||||||
quant_config=quant_config,
|
self.head = ParallelLMHead(
|
||||||
prefix=maybe_prefix(prefix, "head"),
|
config.vocab_size,
|
||||||
)
|
config.hidden_size,
|
||||||
|
quant_config=quant_config,
|
||||||
|
prefix=maybe_prefix(prefix, "head"),
|
||||||
|
)
|
||||||
|
|
||||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||||
return self.norm(hidden_states)
|
return self.norm(hidden_states)
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
|||||||
assert vllm_config.speculative_config is not None
|
assert vllm_config.speculative_config is not None
|
||||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||||
self.config = config
|
self.config = config
|
||||||
quant_config = vllm_config.quant_config
|
|
||||||
|
|
||||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
@@ -66,7 +65,9 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
|||||||
device=current_platform.device_type,
|
device=current_platform.device_type,
|
||||||
)
|
)
|
||||||
self.shared_head = SharedHead(
|
self.shared_head = SharedHead(
|
||||||
config=config, prefix=prefix, quant_config=quant_config
|
config=config,
|
||||||
|
prefix=prefix,
|
||||||
|
defer_lm_head=True,
|
||||||
)
|
)
|
||||||
# MTP layers sit past the base model's hidden layers; parse the index
|
# MTP layers sit past the base model's hidden layers; parse the index
|
||||||
# from the prefix (e.g. "...layers.32") so the decoder builds an MLA
|
# from the prefix (e.g. "...layers.32") so the decoder builds an MLA
|
||||||
|
|||||||
Reference in New Issue
Block a user