[Kernel] Add GPTQv2 format support for low-bit or asymmetric quantization, by adapting gptq_gemm (#26092)
This commit is contained in:
@@ -11,6 +11,7 @@ from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
@@ -36,6 +37,8 @@ if TYPE_CHECKING:
|
||||
else:
|
||||
QuantizationMethods = str
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPTQConfig(QuantizationConfig):
|
||||
"""Config class for GPTQ.
|
||||
@@ -52,6 +55,7 @@ class GPTQConfig(QuantizationConfig):
|
||||
dynamic: dict[str, dict[str, int | bool]],
|
||||
autoround_version: str = "",
|
||||
modules_in_block_to_quantize: list[str] | None = None,
|
||||
checkpoint_format: str = "",
|
||||
) -> None:
|
||||
# GPTQModel use `dynamic` config property to allow per module
|
||||
# quantization config so each module can be individually optimized.
|
||||
@@ -89,12 +93,24 @@ class GPTQConfig(QuantizationConfig):
|
||||
"Currently, only 2/3/4/8-bit weight quantization is "
|
||||
f"supported for GPTQ, but got {self.weight_bits} bits."
|
||||
)
|
||||
# Somehow gptq_gemm 4-bit is buggy, maybe fix it in the future.
|
||||
# For now, show a warning, since gptq_marlin will be used by default.
|
||||
if self.weight_bits == 4:
|
||||
logger.warning_once(
|
||||
"Currently, the 4-bit gptq_gemm kernel for GPTQ is buggy. "
|
||||
"Please switch to gptq_marlin or gptq_bitblas."
|
||||
)
|
||||
|
||||
self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
|
||||
|
||||
# used to identify GPTQ model quantized by autoround
|
||||
self.autoround_version = autoround_version
|
||||
|
||||
# GPTQ v1 and v2 format deals with zero points differently.
|
||||
# Currently GPTQModel stores v1 format checkpoints by default,
|
||||
# but provides the option to set `format="gptq_v2"` in `QuantizeConfig`.
|
||||
self.checkpoint_format = checkpoint_format
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"GPTQConfig(weight_bits={self.weight_bits}, "
|
||||
@@ -102,7 +118,8 @@ class GPTQConfig(QuantizationConfig):
|
||||
f"desc_act={self.desc_act}), "
|
||||
f"lm_head_quantized={self.lm_head_quantized}, "
|
||||
f"dynamic={self.dynamic}, "
|
||||
f"modules_in_block_to_quantize={self.modules_in_block_to_quantize})"
|
||||
f"modules_in_block_to_quantize={self.modules_in_block_to_quantize}), "
|
||||
f"checkpoint_format={self.checkpoint_format})"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -137,6 +154,9 @@ class GPTQConfig(QuantizationConfig):
|
||||
modules_in_block_to_quantize = cls.get_from_keys_or(
|
||||
config, ["modules_in_block_to_quantize"], default=None
|
||||
)
|
||||
checkpoint_format = cls.get_from_keys_or(
|
||||
config, ["checkpoint_format"], default=""
|
||||
)
|
||||
return cls(
|
||||
weight_bits,
|
||||
group_size,
|
||||
@@ -145,6 +165,7 @@ class GPTQConfig(QuantizationConfig):
|
||||
dynamic,
|
||||
autoround_version,
|
||||
modules_in_block_to_quantize,
|
||||
checkpoint_format,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
@@ -154,6 +175,7 @@ class GPTQConfig(QuantizationConfig):
|
||||
# GPTQ MoE support: fall back to MoeWNA16 for broad compatibility
|
||||
from .moe_wna16 import MoeWNA16Config
|
||||
|
||||
# TODO: maybe update this for GPTQv2 format checkpoints
|
||||
config = {
|
||||
"quant_method": "gptq",
|
||||
"bits": self.weight_bits,
|
||||
@@ -210,6 +232,9 @@ class GPTQLinearMethod(LinearMethodBase):
|
||||
def __init__(self, quant_config: GPTQConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
# GPTQ v1 and v2 format deals with zero points differently
|
||||
self.use_v2_format = quant_config.checkpoint_format == "gptq_v2"
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -351,6 +376,8 @@ class GPTQLinearMethod(LinearMethodBase):
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[-1],)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
|
||||
# GPTQ v1 and v2 format checkpoints deals with zero points differently,
|
||||
# and require different gemm kernels.
|
||||
output = ops.gptq_gemm(
|
||||
reshaped_x,
|
||||
layer.qweight,
|
||||
@@ -358,6 +385,7 @@ class GPTQLinearMethod(LinearMethodBase):
|
||||
layer.scales,
|
||||
layer.g_idx,
|
||||
layer.exllama_state == ExllamaState.READY,
|
||||
self.use_v2_format,
|
||||
self.quant_config.weight_bits,
|
||||
)
|
||||
if bias is not None:
|
||||
|
||||
@@ -145,10 +145,15 @@ class ExllamaLinearKernel(MPLinearKernel):
|
||||
|
||||
w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
|
||||
|
||||
# gptq_gemm supports GPTQv2 format by passing use_v2_format=True.
|
||||
# However, the MPLinearLayerConfig doesn't contain format info.
|
||||
# So hardcode GPTQv1 format here, to keep its behavior unchanged.
|
||||
use_v2_format = False
|
||||
|
||||
assert w_zp is not None, "Zero points are required by Exllama"
|
||||
assert w_g_idx is not None, "Group index is required by Exllama"
|
||||
output = ops.gptq_gemm(
|
||||
x_2d, w_q, w_zp, w_s, w_g_idx, True, c.weight_type.size_bits
|
||||
x_2d, w_q, w_zp, w_s, w_g_idx, True, use_v2_format, c.weight_type.size_bits
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
|
||||
Reference in New Issue
Block a user