From c8db00b16cc188b46b7b9517a5836a0da4aa8c3e Mon Sep 17 00:00:00 2001 From: vllmellm Date: Thu, 23 Jul 2026 21:59:48 +0800 Subject: [PATCH] Fix GPTQ quantized Qwen3.5 MTP weight loading with spec decode (#48816) Signed-off-by: vllmellm Co-authored-by: TJian Co-authored-by: noobHappylife <64898326+noobHappylife@users.noreply.github.com> --- vllm/model_executor/models/qwen3_5_mtp.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 0620509f2d..6421c8b60d 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -103,6 +103,16 @@ class Qwen3_5MultiTokenPredictor(nn.Module): prefix=f"{prefix}.fc", ) + # GPTQ: quantized checkpoints may exclude MTP from quantization via + # quantization_config.dynamic with "-:pattern" entries. When detected, + # disable quantization for MTP layers so they use unquantized params. + original_quant = vllm_config.quant_config + if quant_config and quant_config.get_name() not in ("modelopt_fp4",): + hf_qc = getattr(model_config.hf_config, "quantization_config", None) + if isinstance(hf_qc, dict): + dynamic = hf_qc.get("dynamic", {}) + if any(k.startswith("-:") and "mtp" in k for k in dynamic): + vllm_config.quant_config = None self.layers = torch.nn.ModuleList( Qwen3_5DecoderLayer( vllm_config, @@ -111,11 +121,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module): ) for idx in range(self.num_mtp_layers) ) - + vllm_config.quant_config = original_quant self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size ) - self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.pre_fc_norm_hidden = Qwen3_5RMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -170,6 +179,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module): positions.shape[-1], self.config.hidden_size, ) + hidden_states, _ = self.norm(hidden_states, residual) return hidden_states