[Quant] [Bugfix] Fix quantization config matching with hf_to_vllm_mapper (#20046)

This commit is contained in:
Kyle Sayers
2025-07-01 19:20:34 +09:00
committed by GitHub
parent c05596f1a3
commit 9025a9a705
17 changed files with 107 additions and 29 deletions
+20 -3
View File
@@ -18,6 +18,7 @@ from .interfaces_base import is_pooling_model
if TYPE_CHECKING:
from vllm.attention import AttentionMetadata
from vllm.model_executor.models.utils import WeightsMapper
from vllm.sequence import IntermediateTensors
logger = init_logger(__name__)
@@ -566,20 +567,36 @@ def has_step_pooler(model: Union[type[object], object]) -> bool:
class SupportsQuant:
"""The interface required for all models that support quantization."""
packed_modules_mapping: ClassVar[dict[str, list[str]]] = {}
hf_to_vllm_mapper: ClassVar[Optional["WeightsMapper"]] = None
packed_modules_mapping: ClassVar[Optional[dict[str, list[str]]]] = None
quant_config: Optional[QuantizationConfig] = None
def __new__(cls, *args, **kwargs) -> Self:
instance = super().__new__(cls)
# find config passed in arguments
quant_config = cls._find_quant_config(*args, **kwargs)
if quant_config is not None:
# attach config to model for general use
instance.quant_config = quant_config
instance.quant_config.packed_modules_mapping.update(
cls.packed_modules_mapping)
# apply model mappings to config for proper config-model matching
# NOTE: `TransformersForCausalLM` is not supported due to how this
# class defines `hf_to_vllm_mapper` as a post-init `@property`.
# After this is fixed, get `instance.hf_to_vllm_mapper` directly
if getattr(instance, "hf_to_vllm_mapper", None) is not None:
instance.quant_config.apply_vllm_mapper(
instance.hf_to_vllm_mapper)
if getattr(instance, "packed_modules_mapping", None) is not None:
instance.quant_config.packed_modules_mapping.update(
instance.packed_modules_mapping)
return instance
@staticmethod
def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:
"""Find quant config passed through model constructor args"""
from vllm.config import VllmConfig # avoid circular import
args_values = list(args) + list(kwargs.values())