From f1d8d99717b6aebf19eac459e0c1fd04bdbe356c Mon Sep 17 00:00:00 2001 From: "Kai K." <59895482+KaletoAI@users.noreply.github.com> Date: Thu, 11 Jun 2026 17:14:21 +0200 Subject: [PATCH] [Bugfix] CohereModel.load_weights: skip modelopt _quantizer.* keys (#43495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kai Köhler --- vllm/model_executor/models/commandr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index 317269ec3b..66adb9a3ca 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, @@ -397,6 +398,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): } # LoRA specific attributes embedding_modules = {"embed_tokens": "input_embeddings"} + # ModelOpt NVFP4 checkpoints carry raw quantizer-module state + # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925. + hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None}) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -453,4 +457,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): loader = AutoWeightsLoader( self, skip_prefixes=["lm_head", "rotary_emb.inv_freq"] ) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)