Fix GPTQ model loading in Transformers backend (#25770)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Harry Mellor
2025-09-27 12:18:20 +00:00
committed by GitHub
co-authored by Isotr0py
parent 7977e5027c
commit ec152c8748
3 changed files with 29 additions and 10 deletions
+17 -5
View File
@@ -447,7 +447,8 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
self.device_config: DeviceConfig = vllm_config.device_config
self.model_config: ModelConfig = vllm_config.model_config
self.parallel_config: ParallelConfig = vllm_config.parallel_config
self.quant_config: QuantizationConfig = vllm_config.quant_config
self.quant_config: Optional[
QuantizationConfig] = vllm_config.quant_config
self.pp_group = get_pp_group()
self.pp_size = self.pp_group.world_size
@@ -456,7 +457,18 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
# Weights to skip in `self.load_weights`
self.skip_prefixes: list[str] = []
"""Skip loading weights whose qualname starts with these prefixes."""
self.skip_substrs: list[str] = []
"""Skip loading weights whose qualname contains these substrings."""
self.ignore_unexpected_prefixes: list[str] = []
"""Ignore unexpected weights whose qualname starts with these prefixes.
"""
self.ignore_unexpected_suffixes: list[str] = []
"""Ignore unexpected weights whose qualname ends with these suffixes."""
# Skip loading extra bias for GPTQ models.
if self.quant_config and "gptq" in self.quant_config.get_name():
self.ignore_unexpected_suffixes.append(".bias")
# Set correct attn and init on "meta" to delay allocating GPU tensors
# TODO: @raushan, use the public `model.set_attn_implementation()`
@@ -563,9 +575,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
raise ValueError(
f"{type(self.model)} does not support tensor parallel. {tip}")
def _tensor_parallel(module: nn.Module,
prefix: str = "",
tp_plan=None):
def _tensor_parallel(module: nn.Module, prefix: str, tp_plan=None):
tp_plan = tp_plan or {}
# If the current module is a PreTrainedModel, set the tp_plan for
@@ -597,7 +607,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
prefix=qual_name,
tp_plan=tp_plan)
_tensor_parallel(self.model)
_tensor_parallel(self.model, prefix="model")
def create_attention_instances(
self,
@@ -696,6 +706,8 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
self,
skip_prefixes=self.skip_prefixes,
skip_substrs=self.skip_substrs,
ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)