[Quantization] Automatically infer AWQ modules_to_not_convert field (#26909)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2025-10-21 01:49:28 +00:00
committed by GitHub
parent bfe0b4bd2a
commit 352c0c8a28
6 changed files with 96 additions and 45 deletions
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Any, Optional
from typing import TYPE_CHECKING, Any, Optional
import torch
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from torch.nn import Parameter
import vllm.model_executor.layers.fused_moe # noqa
@@ -27,8 +28,7 @@ from vllm.model_executor.layers.linear import (
UnquantizedLinearMethod,
set_weight_attrs,
)
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.awq import AWQConfig, is_layer_skipped_awq
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -49,10 +49,16 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
verify_marlin_supported,
verify_marlin_supports_shape,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.transformers_utils.config import get_safetensors_params_metadata
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
@@ -106,7 +112,7 @@ class AWQMarlinConfig(QuantizationConfig):
)
@classmethod
def get_name(cls) -> QuantizationMethods:
def get_name(cls) -> "QuantizationMethods":
return "awq_marlin"
@classmethod
@@ -142,7 +148,7 @@ class AWQMarlinConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
) -> QuantizationMethods | None:
) -> Optional["QuantizationMethods"]:
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
is_valid_user_quant = (
user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"
@@ -171,7 +177,9 @@ class AWQMarlinConfig(QuantizationConfig):
if isinstance(layer, LinearBase) or (
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
):
if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
if is_layer_skipped(
prefix, self.modules_to_not_convert, self.packed_modules_mapping
):
return UnquantizedLinearMethod()
# Check if the layer is supported by AWQMarlin.
if not check_marlin_supports_layer(layer, self.group_size):
@@ -186,9 +194,7 @@ class AWQMarlinConfig(QuantizationConfig):
elif isinstance(layer, FusedMoE):
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if is_layer_skipped_awq(
prefix, getattr(self, "modules_to_not_convert", [])
):
if is_layer_skipped(prefix, getattr(self, "modules_to_not_convert", [])):
return UnquantizedFusedMoEMethod(layer.moe_config)
if not check_moe_marlin_supports_layer(layer, self.group_size):
logger.warning_once(
@@ -226,6 +232,27 @@ class AWQMarlinConfig(QuantizationConfig):
quant_type=cls.TYPE_MAP[num_bits], group_size=group_size, has_zp=zero_point
)
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.modules_to_not_convert:
self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(
self.modules_to_not_convert
)
def maybe_update_config(self, model_name: str, revision: str | None = None):
if self.modules_to_not_convert:
return
unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
metadata = get_safetensors_params_metadata(model_name, revision=revision)
layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}
quant_layers: set[str] = {
param_name.rsplit(".", 1)[0]
for param_name, info in metadata.items()
if (dtype := info.get("dtype", None))
and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
}
self.modules_to_not_convert = list(layers - quant_layers)
class AWQMarlinLinearMethod(LinearMethodBase):
"""Linear method for AWQ Marlin.