[Model] Add TP and BNB quantization support to LlavaMultiModalProjector (#10834)

Signed-off-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Isotr0py
2024-12-02 23:06:09 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 9b14d978aa
commit 4c05edb33a
2 changed files with 34 additions and 15 deletions
+11 -3
View File
@@ -1120,7 +1120,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
model_config.revision,
pre_quant, load_8bit))
model.load_weights(qweight_iterator)
weights_to_load = {name for name, _ in model.named_parameters()}
loaded_weights = model.load_weights(qweight_iterator)
# Some models may have weights loading tracker unimplemented.
if loaded_weights is not None:
weights_not_loaded = weights_to_load - loaded_weights
if weights_not_loaded:
raise ValueError("Following weights were not initialized from "
f"checkpoint: {weights_not_loaded}")
torch.cuda.empty_cache()
@@ -1152,9 +1159,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
shard_name, weight_name)
break
# Models like Clip/Siglip may skip some layers in initialization,
# causing unused quant_param_name in state_dict.
if quant_param_name not in param_dict:
raise ValueError(
f"Parameter {quant_param_name} not found in the model.")
continue
if quant_param_name not in stacked_quant_state_dict:
stacked_quant_state_dict[quant_param_name] = {}