[Qwen3.5] Build MTP head unquantized when checkpoint ships it dense (PR #58813)

Fixes #58807. A compressed-tensors pack-quantized checkpoint that ships
its MTP head dense (plain mtp.*.weight names in a separate bf16 shard)
crashes at weight load: Qwen3_5MultiTokenPredictor builds its Linears
from the target quant config, so the loader expects packed names
(weight_packed) the checkpoint never had, and the weight-map walk dies
with "no module or parameter named fc.weight".

Detect the dense MTP head from the checkpoint itself: read the
safetensors index via the same repo-file channel used for recipe.yaml
(local cache first, Hub download as fallback). If any mtp.* tensor
carries a quantization artifact suffix the head is properly quantized
and nothing changes; if every mtp.* entry is a plain .weight the whole
predictor builds unquantized (fc and layers), mirroring the existing
modelopt_fp4 and GPTQ dynamic carve-outs. Single-file checkpoints have
no index and keep current behavior; NVFP4 checkpoints are unaffected
(their mtp.layers.* carry artifacts).

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-27 09:54:24 +07:00
co-authored by Hinata
parent 835c44af15
commit 5af48b498c
2 changed files with 160 additions and 6 deletions
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Dense-MTP-head detection for pack-quantized checkpoints (#58807).
The whole fix hinges on reading the checkpoint's actual tensor names, so the
tests pin the naming contract: dense means plain `mtp.*.weight` keys with no
quantization artifact suffix anywhere under `mtp.`.
"""
import json
from types import SimpleNamespace
import pytest
from vllm.model_executor.models.qwen3_5_mtp import (
_checkpoint_ships_dense_mtp,
_mtp_weight_names,
)
pytestmark = pytest.mark.cpu_test
def _model_config_with_index(monkeypatch, names):
monkeypatch.setattr(
"vllm.model_executor.models.qwen3_5_mtp._mtp_weight_names",
lambda _config: names,
)
return SimpleNamespace()
@pytest.mark.parametrize(
"names,expected",
[
# The reporter's checkpoint: every mtp.* tensor dense.
(["mtp.fc.weight", "mtp.layers.0.mlp.down_proj.weight"], True),
# Biases are dense too and must not count as artifacts.
(["mtp.fc.weight", "mtp.fc.bias"], True),
# A properly pack-quantized MTP head keeps the target quantization.
(["mtp.fc.weight_packed", "mtp.fc.weight_scale"], False),
# Channel-wise quantized MTP head: dense name plus scale companion.
(["mtp.fc.weight", "mtp.fc.weight_scale"], False),
# No MTP head in the checkpoint at all.
(["model.layers.0.self_attn.q_proj.weight"], False),
# Layout unknown (no index): keep current behavior.
(None, False),
([], False),
],
)
def test_dense_mtp_detection(monkeypatch, names, expected):
config = _model_config_with_index(monkeypatch, names)
assert _checkpoint_ships_dense_mtp(config) is expected
def test_weight_names_read_from_local_index(tmp_path, monkeypatch):
index = {
"weight_map": {
"model.layers.0.self_attn.q_proj.weight_packed": "model-00001.safetensors",
"mtp.fc.weight": "model-mtp-bf16.safetensors",
"mtp.layers.0.mlp.down_proj.weight": "model-mtp-bf16.safetensors",
}
}
(tmp_path / "model.safetensors.index.json").write_text(json.dumps(index))
config = SimpleNamespace(model=str(tmp_path), revision=None)
names = _mtp_weight_names(config)
assert names == ["mtp.fc.weight", "mtp.layers.0.mlp.down_proj.weight"]
def test_weight_names_none_without_index(tmp_path, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.models.qwen3_5_mtp.huggingface_hub.hf_hub_download",
lambda *args, **kwargs: (_ for _ in ()).throw(FileNotFoundError()),
)
config = SimpleNamespace(model=str(tmp_path), revision=None)
assert _mtp_weight_names(config) is None