Fix GPTQ quantized Qwen3.5 MTP weight loading with spec decode (#48816)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: noobHappylife <64898326+noobHappylife@users.noreply.github.com>
This commit is contained in:
vllmellm
2026-07-23 06:59:48 -07:00
committed by GitHub
co-authored by TJian noobHappylife
parent 80c7683923
commit c8db00b16c
+12 -2
View File
@@ -103,6 +103,16 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
prefix=f"{prefix}.fc",
)
# GPTQ: quantized checkpoints may exclude MTP from quantization via
# quantization_config.dynamic with "-:pattern" entries. When detected,
# disable quantization for MTP layers so they use unquantized params.
original_quant = vllm_config.quant_config
if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
hf_qc = getattr(model_config.hf_config, "quantization_config", None)
if isinstance(hf_qc, dict):
dynamic = hf_qc.get("dynamic", {})
if any(k.startswith("-:") and "mtp" in k for k in dynamic):
vllm_config.quant_config = None
self.layers = torch.nn.ModuleList(
Qwen3_5DecoderLayer(
vllm_config,
@@ -111,11 +121,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
)
for idx in range(self.num_mtp_layers)
)
vllm_config.quant_config = original_quant
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -170,6 +179,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
positions.shape[-1],
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states