[Bugfix][Model] Fix Qwen3.5 MTP for text-only checkpoints (#50734)
Signed-off-by: efschu <51944948+efschu@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for Qwen3.5 text-only MTP speculative decoding."""
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
|
||||
|
||||
def _mtp_config(model_type: str) -> PretrainedConfig:
|
||||
return PretrainedConfig(
|
||||
model_type=model_type,
|
||||
architectures=["SomeArch"],
|
||||
mtp_num_hidden_layers=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type,expected_arch",
|
||||
[
|
||||
("qwen3_5", "Qwen3_5MTP"),
|
||||
("qwen3_5_moe", "Qwen3_5MoeMTP"),
|
||||
# Text-only config variants must map to the same MTP architectures.
|
||||
("qwen3_5_text", "Qwen3_5MTP"),
|
||||
("qwen3_5_moe_text", "Qwen3_5MoeMTP"),
|
||||
],
|
||||
)
|
||||
def test_mtp_override_recognizes_text_only_types(model_type, expected_arch):
|
||||
cfg = SpeculativeConfig.hf_config_override(_mtp_config(model_type))
|
||||
assert cfg.model_type == "qwen3_5_mtp"
|
||||
assert cfg.architectures == [expected_arch]
|
||||
assert cfg.n_predict == 1
|
||||
@@ -528,8 +528,16 @@ class SpeculativeConfig:
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["Exaone4_5_MTP"]}
|
||||
)
|
||||
if hf_config.model_type in ("qwen3_5", "qwen3_5_moe"):
|
||||
is_moe = hf_config.model_type == "qwen3_5_moe"
|
||||
if hf_config.model_type in (
|
||||
"qwen3_5",
|
||||
"qwen3_5_moe",
|
||||
"qwen3_5_text",
|
||||
"qwen3_5_moe_text",
|
||||
):
|
||||
# Checkpoints that ship only the text config resolve to the
|
||||
# `qwen3_5_text` / `qwen3_5_moe_text` model types and carry the
|
||||
# same `mtp_num_hidden_layers` field as the multimodal ones.
|
||||
is_moe = hf_config.model_type in ("qwen3_5_moe", "qwen3_5_moe_text")
|
||||
hf_config.model_type = "qwen3_5_mtp"
|
||||
n_predict = getattr(hf_config, "mtp_num_hidden_layers", None)
|
||||
hf_config.update(
|
||||
|
||||
@@ -114,11 +114,18 @@ class MultiModalRegistry:
|
||||
try:
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
except ValueError:
|
||||
logger.warning_once(
|
||||
"Model %s is treated as multimodal but has no registered "
|
||||
"multimodal processor; running in text-only mode.",
|
||||
model_config.model,
|
||||
)
|
||||
# Speculative drafters for multimodal targets (e.g. Qwen3_5MTP,
|
||||
# Exaone4_5_MTP, MiMoV2OmniMTP) declare `SupportsMultiModal` so
|
||||
# that they can consume the embeddings merged by the target model,
|
||||
# but they never run a multi-modal processor of their own. Running
|
||||
# in text-only mode is the expected outcome for them, not a
|
||||
# misconfiguration worth warning about.
|
||||
if model_config.runner_type != "draft":
|
||||
logger.warning_once(
|
||||
"Model %s is treated as multimodal but has no registered "
|
||||
"multimodal processor; running in text-only mode.",
|
||||
model_config.model,
|
||||
)
|
||||
return False
|
||||
|
||||
# Check if all supported modalities have limit == 0
|
||||
|
||||
Reference in New Issue
Block a user