[Quant] [Bugfix] Fix quantization config matching with hf_to_vllm_mapper (#20046)

This commit is contained in:
Kyle Sayers
2025-07-01 19:20:34 +09:00
committed by GitHub
parent c05596f1a3
commit 9025a9a705
17 changed files with 107 additions and 29 deletions
+20 -3
View File
@@ -18,6 +18,7 @@ from .interfaces_base import is_pooling_model
if TYPE_CHECKING:
from vllm.attention import AttentionMetadata
from vllm.model_executor.models.utils import WeightsMapper
from vllm.sequence import IntermediateTensors
logger = init_logger(__name__)
@@ -566,20 +567,36 @@ def has_step_pooler(model: Union[type[object], object]) -> bool:
class SupportsQuant:
"""The interface required for all models that support quantization."""
packed_modules_mapping: ClassVar[dict[str, list[str]]] = {}
hf_to_vllm_mapper: ClassVar[Optional["WeightsMapper"]] = None
packed_modules_mapping: ClassVar[Optional[dict[str, list[str]]]] = None
quant_config: Optional[QuantizationConfig] = None
def __new__(cls, *args, **kwargs) -> Self:
instance = super().__new__(cls)
# find config passed in arguments
quant_config = cls._find_quant_config(*args, **kwargs)
if quant_config is not None:
# attach config to model for general use
instance.quant_config = quant_config
instance.quant_config.packed_modules_mapping.update(
cls.packed_modules_mapping)
# apply model mappings to config for proper config-model matching
# NOTE: `TransformersForCausalLM` is not supported due to how this
# class defines `hf_to_vllm_mapper` as a post-init `@property`.
# After this is fixed, get `instance.hf_to_vllm_mapper` directly
if getattr(instance, "hf_to_vllm_mapper", None) is not None:
instance.quant_config.apply_vllm_mapper(
instance.hf_to_vllm_mapper)
if getattr(instance, "packed_modules_mapping", None) is not None:
instance.quant_config.packed_modules_mapping.update(
instance.packed_modules_mapping)
return instance
@staticmethod
def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:
"""Find quant config passed through model constructor args"""
from vllm.config import VllmConfig # avoid circular import
args_values = list(args) + list(kwargs.values())
+7 -7
View File
@@ -61,7 +61,7 @@ from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import uses_mrope
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
SupportsMultiModal, SupportsPP)
SupportsMultiModal, SupportsPP, SupportsQuant)
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
apply_rotary_pos_emb_vision)
@@ -821,7 +821,8 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
info=Qwen2_5_VLProcessingInfo,
dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
SupportsLoRA, SupportsPP):
SupportsLoRA, SupportsPP,
SupportsQuant):
# To ensure correct weight loading and mapping.
hf_to_vllm_mapper = WeightsMapper(
@@ -837,7 +838,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
@@ -846,7 +846,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
self.visual = Qwen2_5_VisionTransformer(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=self._maybe_ignore_quant_config(quant_config),
quant_config=self._maybe_ignore_quant_config(self.quant_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -859,12 +859,12 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors)
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
def _maybe_ignore_quant_config(self, config: Optional[QuantizationConfig]):
# GPTQ configs do not have a list of ignored modules, however AutoGPTQ
# seems to avoid vision encoder sections for some models.
if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
if isinstance(config, (GPTQConfig, GPTQMarlinConfig)):
return None
return quant_config
return config
def _validate_and_reshape_mm_tensor(self, mm_input: object,
name: str) -> torch.Tensor:
@@ -467,6 +467,7 @@ class TransformersForCausalLM(nn.Module, SupportsQuant, SupportsLoRA,
# FIXME(Isotr0py): Don't use any weights mapper for Transformers backend,
# this makes thing complicated. We need to remove this mapper after refactor
# `TransformersModel` in the future.
# NOTE: `SupportsQuant` can be updated after property decorator is removed
@property
def hf_to_vllm_mapper(self):
prefix_mapper = {
+14 -1
View File
@@ -4,7 +4,7 @@
import itertools
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Callable, Literal, Optional, Protocol, Union, overload
from typing import Any, Callable, Literal, Optional, Protocol, Union, overload
import torch
import torch.nn as nn
@@ -64,6 +64,19 @@ class WeightsMapper:
return ((out_name, data) for name, data in weights
if (out_name := self._map_name(name)) is not None)
def apply_list(self, values: list[str]) -> list[str]:
return [
out_name for name in values
if (out_name := self._map_name(name)) is not None
]
def apply_dict(self, values: dict[str, Any]) -> dict[str, Any]:
return {
out_name: value
for name, value in values.items()
if (out_name := self._map_name(name)) is not None
}
class AutoWeightsLoader:
"""