diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index 317269ec3b..66adb9a3ca 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, @@ -397,6 +398,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): } # LoRA specific attributes embedding_modules = {"embed_tokens": "input_embeddings"} + # ModelOpt NVFP4 checkpoints carry raw quantizer-module state + # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925. + hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None}) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -453,4 +457,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): loader = AutoWeightsLoader( self, skip_prefixes=["lm_head", "rotary_emb.inv_freq"] ) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)