Fix GPTQ quantized Qwen3.5 MTP weight loading with spec decode (#48816)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: noobHappylife <64898326+noobHappylife@users.noreply.github.com>
This commit is contained in:
co-authored by
TJian
noobHappylife
parent
80c7683923
commit
c8db00b16c
@@ -103,6 +103,16 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
prefix=f"{prefix}.fc",
|
prefix=f"{prefix}.fc",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# GPTQ: quantized checkpoints may exclude MTP from quantization via
|
||||||
|
# quantization_config.dynamic with "-:pattern" entries. When detected,
|
||||||
|
# disable quantization for MTP layers so they use unquantized params.
|
||||||
|
original_quant = vllm_config.quant_config
|
||||||
|
if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
|
||||||
|
hf_qc = getattr(model_config.hf_config, "quantization_config", None)
|
||||||
|
if isinstance(hf_qc, dict):
|
||||||
|
dynamic = hf_qc.get("dynamic", {})
|
||||||
|
if any(k.startswith("-:") and "mtp" in k for k in dynamic):
|
||||||
|
vllm_config.quant_config = None
|
||||||
self.layers = torch.nn.ModuleList(
|
self.layers = torch.nn.ModuleList(
|
||||||
Qwen3_5DecoderLayer(
|
Qwen3_5DecoderLayer(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
@@ -111,11 +121,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
)
|
)
|
||||||
for idx in range(self.num_mtp_layers)
|
for idx in range(self.num_mtp_layers)
|
||||||
)
|
)
|
||||||
|
vllm_config.quant_config = original_quant
|
||||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||||
["hidden_states", "residual"], config.hidden_size
|
["hidden_states", "residual"], config.hidden_size
|
||||||
)
|
)
|
||||||
|
|
||||||
self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
|
self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
|
||||||
config.hidden_size, eps=config.rms_norm_eps
|
config.hidden_size, eps=config.rms_norm_eps
|
||||||
@@ -170,6 +179,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
positions.shape[-1],
|
positions.shape[-1],
|
||||||
self.config.hidden_size,
|
self.config.hidden_size,
|
||||||
)
|
)
|
||||||
|
|
||||||
hidden_states, _ = self.norm(hidden_states, residual)
|
hidden_states, _ = self.norm(hidden_states, residual)
|
||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user