Fix GPTQ quantized Qwen3.5 MTP weight loading with spec decode (#48816)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: noobHappylife <64898326+noobHappylife@users.noreply.github.com>
This commit is contained in:
vllmellm
2026-07-23 06:59:48 -07:00
committed by GitHub
co-authored by TJian noobHappylife
parent 80c7683923
commit c8db00b16c
+12 -2
View File
@@ -103,6 +103,16 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
prefix=f"{prefix}.fc", prefix=f"{prefix}.fc",
) )
# GPTQ: quantized checkpoints may exclude MTP from quantization via
# quantization_config.dynamic with "-:pattern" entries. When detected,
# disable quantization for MTP layers so they use unquantized params.
original_quant = vllm_config.quant_config
if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
hf_qc = getattr(model_config.hf_config, "quantization_config", None)
if isinstance(hf_qc, dict):
dynamic = hf_qc.get("dynamic", {})
if any(k.startswith("-:") and "mtp" in k for k in dynamic):
vllm_config.quant_config = None
self.layers = torch.nn.ModuleList( self.layers = torch.nn.ModuleList(
Qwen3_5DecoderLayer( Qwen3_5DecoderLayer(
vllm_config, vllm_config,
@@ -111,11 +121,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
) )
for idx in range(self.num_mtp_layers) for idx in range(self.num_mtp_layers)
) )
vllm_config.quant_config = original_quant
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size ["hidden_states", "residual"], config.hidden_size
) )
self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.pre_fc_norm_hidden = Qwen3_5RMSNorm( self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
config.hidden_size, eps=config.rms_norm_eps config.hidden_size, eps=config.rms_norm_eps
@@ -170,6 +179,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
positions.shape[-1], positions.shape[-1],
self.config.hidden_size, self.config.hidden_size,
) )
hidden_states, _ = self.norm(hidden_states, residual) hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states return hidden_states