[Quant] Consolidate GPTQ: rename gptq_marlin.py to auto_gptq.py (#38288)
Signed-off-by: Chengyi Nie <cnie@roblox.com> Co-authored-by: Chengyi Nie <cnie@roblox.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Chengyi Nie
Cursor
parent
3b6a204789
commit
fa2a33b893
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from fractions import Fraction
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -16,16 +15,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..gptq import GPTQConfig
|
||||
from ..gptq_marlin import GPTQMarlinConfig
|
||||
from ..auto_gptq import AutoGPTQConfig
|
||||
else:
|
||||
GPTQConfig = object
|
||||
GPTQMarlinConfig = object
|
||||
AutoGPTQConfig = object
|
||||
|
||||
|
||||
# Match dynamic rules with module name (prefix) and override quantize
|
||||
# config if module (prefix) matches a rule
|
||||
def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
|
||||
def override_config(config: AutoGPTQConfig, prefix: str):
|
||||
weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits)
|
||||
if isinstance(weight_bits, int):
|
||||
config.weight_bits = weight_bits
|
||||
@@ -36,31 +33,23 @@ def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
|
||||
if isinstance(desc_act, bool):
|
||||
config.desc_act = desc_act
|
||||
|
||||
config.pack_factor = Fraction(32, config.weight_bits) # packed into int32
|
||||
if config.get_name() == "gptq_marlin":
|
||||
assert isinstance(config, GPTQMarlinConfig)
|
||||
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
|
||||
if isinstance(is_sym, bool):
|
||||
config.is_sym = is_sym
|
||||
config.pack_factor = 32 // config.weight_bits # packed into int32
|
||||
assert isinstance(config, AutoGPTQConfig)
|
||||
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
|
||||
if isinstance(is_sym, bool):
|
||||
config.is_sym = is_sym
|
||||
|
||||
if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
|
||||
raise ValueError(
|
||||
"Unsupported quantization config: "
|
||||
f"bits={config.weight_bits}, sym={config.is_sym}"
|
||||
)
|
||||
if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
|
||||
raise ValueError(
|
||||
"Unsupported quantization config: "
|
||||
f"bits={config.weight_bits}, sym={config.is_sym}"
|
||||
)
|
||||
|
||||
config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
|
||||
elif config.get_name() == "gptq":
|
||||
assert isinstance(config, GPTQConfig)
|
||||
if config.weight_bits not in [2, 3, 4, 8]:
|
||||
raise ValueError(
|
||||
"Currently, only 2/3/4/8-bit weight quantization is "
|
||||
f"supported for GPTQ, but got {config.weight_bits} bits."
|
||||
)
|
||||
config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
|
||||
|
||||
|
||||
def get_dynamic_override(
|
||||
config: GPTQConfig | GPTQMarlinConfig,
|
||||
config: AutoGPTQConfig,
|
||||
layer_name: str,
|
||||
key: str | None = None,
|
||||
default_value: int | bool | None = None,
|
||||
@@ -126,7 +115,7 @@ def is_layer_gptq_quantized(
|
||||
|
||||
|
||||
def get_linear_quant_method(
|
||||
config: GPTQConfig | GPTQMarlinConfig,
|
||||
config: AutoGPTQConfig,
|
||||
layer: torch.nn.Module,
|
||||
prefix: str,
|
||||
linear_method_cls: type,
|
||||
|
||||
Reference in New Issue
Block a user