[Migration] Migrate GGUF quantization support to plugin (#39612)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-12 12:02:21 -07:00
committed by GitHub
parent d6fd7ce8da
commit 6635279d8a
57 changed files with 72 additions and 9048 deletions
-26
View File
@@ -952,38 +952,12 @@ class SiglipVisionModel(nn.Module):
break
else:
param = params_dict[name]
param = maybe_swap_ffn_param(
name, param, loaded_weight, params_dict, self.quant_config
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def maybe_swap_ffn_param(
name: str,
param: torch.Tensor,
loaded_weight: torch.Tensor,
params_dict: dict[str, torch.Tensor],
quant_config: QuantizationConfig,
) -> torch.Tensor:
if not (quant_config and quant_config.get_name() == "gguf") or ".fc" not in name:
return param
# Some GGUF models have fc1 and fc2 weights swapped
tp_size = get_tensor_model_parallel_world_size()
output_dim = getattr(param, "output_dim", 0)
output_size = param.size(output_dim) * tp_size
weight_out_size = loaded_weight.size(output_dim)
if ".fc1." in name and output_size != weight_out_size:
new_name = name.replace(".fc1.", ".fc2.")
param = params_dict[new_name]
elif ".fc2." in name and output_size != weight_out_size:
new_name = name.replace(".fc2.", ".fc1.")
param = params_dict[new_name]
return param
# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
class SiglipTextEmbeddings(nn.Module):
def __init__(self, config: SiglipTextConfig):