[Bugfix] CohereModel.load_weights: skip modelopt _quantizer.* keys (#43495)
Signed-off-by: Kai Köhler <kai.koehler@web.de>
This commit is contained in:
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
|
|||||||
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
||||||
from .utils import (
|
from .utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
|
WeightsMapper,
|
||||||
extract_layer_index,
|
extract_layer_index,
|
||||||
is_pp_missing_parameter,
|
is_pp_missing_parameter,
|
||||||
make_empty_intermediate_tensors_factory,
|
make_empty_intermediate_tensors_factory,
|
||||||
@@ -397,6 +398,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
|||||||
}
|
}
|
||||||
# LoRA specific attributes
|
# LoRA specific attributes
|
||||||
embedding_modules = {"embed_tokens": "input_embeddings"}
|
embedding_modules = {"embed_tokens": "input_embeddings"}
|
||||||
|
# ModelOpt NVFP4 checkpoints carry raw quantizer-module state
|
||||||
|
# (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925.
|
||||||
|
hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None})
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -453,4 +457,4 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
|||||||
loader = AutoWeightsLoader(
|
loader = AutoWeightsLoader(
|
||||||
self, skip_prefixes=["lm_head", "rotary_emb.inv_freq"]
|
self, skip_prefixes=["lm_head", "rotary_emb.inv_freq"]
|
||||||
)
|
)
|
||||||
return loader.load_weights(weights)
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||||
|
|||||||
Reference in New Issue
Block a user