[Quant] [Bugfix] Fix quantization config matching with hf_to_vllm_mapper (#20046)
This commit is contained in:
@@ -18,6 +18,7 @@ from .interfaces_base import is_pooling_model
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -566,20 +567,36 @@ def has_step_pooler(model: Union[type[object], object]) -> bool:
|
||||
class SupportsQuant:
|
||||
"""The interface required for all models that support quantization."""
|
||||
|
||||
packed_modules_mapping: ClassVar[dict[str, list[str]]] = {}
|
||||
hf_to_vllm_mapper: ClassVar[Optional["WeightsMapper"]] = None
|
||||
packed_modules_mapping: ClassVar[Optional[dict[str, list[str]]]] = None
|
||||
quant_config: Optional[QuantizationConfig] = None
|
||||
|
||||
def __new__(cls, *args, **kwargs) -> Self:
|
||||
instance = super().__new__(cls)
|
||||
|
||||
# find config passed in arguments
|
||||
quant_config = cls._find_quant_config(*args, **kwargs)
|
||||
if quant_config is not None:
|
||||
|
||||
# attach config to model for general use
|
||||
instance.quant_config = quant_config
|
||||
instance.quant_config.packed_modules_mapping.update(
|
||||
cls.packed_modules_mapping)
|
||||
|
||||
# apply model mappings to config for proper config-model matching
|
||||
# NOTE: `TransformersForCausalLM` is not supported due to how this
|
||||
# class defines `hf_to_vllm_mapper` as a post-init `@property`.
|
||||
# After this is fixed, get `instance.hf_to_vllm_mapper` directly
|
||||
if getattr(instance, "hf_to_vllm_mapper", None) is not None:
|
||||
instance.quant_config.apply_vllm_mapper(
|
||||
instance.hf_to_vllm_mapper)
|
||||
if getattr(instance, "packed_modules_mapping", None) is not None:
|
||||
instance.quant_config.packed_modules_mapping.update(
|
||||
instance.packed_modules_mapping)
|
||||
|
||||
return instance
|
||||
|
||||
@staticmethod
|
||||
def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:
|
||||
"""Find quant config passed through model constructor args"""
|
||||
from vllm.config import VllmConfig # avoid circular import
|
||||
|
||||
args_values = list(args) + list(kwargs.values())
|
||||
|
||||
@@ -61,7 +61,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import uses_mrope
|
||||
|
||||
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||||
SupportsMultiModal, SupportsPP)
|
||||
SupportsMultiModal, SupportsPP, SupportsQuant)
|
||||
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
|
||||
from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
|
||||
apply_rotary_pos_emb_vision)
|
||||
@@ -821,7 +821,8 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
info=Qwen2_5_VLProcessingInfo,
|
||||
dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
|
||||
class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
SupportsLoRA, SupportsPP):
|
||||
SupportsLoRA, SupportsPP,
|
||||
SupportsQuant):
|
||||
|
||||
# To ensure correct weight loading and mapping.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
@@ -837,7 +838,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
self.config = config
|
||||
@@ -846,7 +846,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
self.visual = Qwen2_5_VisionTransformer(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
quant_config=self._maybe_ignore_quant_config(self.quant_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
@@ -859,12 +859,12 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors)
|
||||
|
||||
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
|
||||
def _maybe_ignore_quant_config(self, config: Optional[QuantizationConfig]):
|
||||
# GPTQ configs do not have a list of ignored modules, however AutoGPTQ
|
||||
# seems to avoid vision encoder sections for some models.
|
||||
if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
|
||||
if isinstance(config, (GPTQConfig, GPTQMarlinConfig)):
|
||||
return None
|
||||
return quant_config
|
||||
return config
|
||||
|
||||
def _validate_and_reshape_mm_tensor(self, mm_input: object,
|
||||
name: str) -> torch.Tensor:
|
||||
|
||||
@@ -467,6 +467,7 @@ class TransformersForCausalLM(nn.Module, SupportsQuant, SupportsLoRA,
|
||||
# FIXME(Isotr0py): Don't use any weights mapper for Transformers backend,
|
||||
# this makes thing complicated. We need to remove this mapper after refactor
|
||||
# `TransformersModel` in the future.
|
||||
# NOTE: `SupportsQuant` can be updated after property decorator is removed
|
||||
@property
|
||||
def hf_to_vllm_mapper(self):
|
||||
prefix_mapper = {
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import itertools
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Literal, Optional, Protocol, Union, overload
|
||||
from typing import Any, Callable, Literal, Optional, Protocol, Union, overload
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -64,6 +64,19 @@ class WeightsMapper:
|
||||
return ((out_name, data) for name, data in weights
|
||||
if (out_name := self._map_name(name)) is not None)
|
||||
|
||||
def apply_list(self, values: list[str]) -> list[str]:
|
||||
return [
|
||||
out_name for name in values
|
||||
if (out_name := self._map_name(name)) is not None
|
||||
]
|
||||
|
||||
def apply_dict(self, values: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
out_name: value
|
||||
for name, value in values.items()
|
||||
if (out_name := self._map_name(name)) is not None
|
||||
}
|
||||
|
||||
|
||||
class AutoWeightsLoader:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user