Add AWQ support for all models (#1714)

This commit is contained in:
Woosuk Kwon
2023-11-18 17:56:47 -08:00
committed by GitHub
parent e946260cf3
commit 8d17774f92
13 changed files with 90 additions and 17 deletions
@@ -63,6 +63,9 @@ class AWQConfig(QuantizationConfig):
def get_linear_method(self) -> "AWQLinearMethod":
return AWQLinearMethod(self)
def get_scaled_act_names(self) -> List[str]:
return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
class AWQLinearMethod(LinearMethodBase):
"""Linear method for AWQ.
@@ -54,3 +54,11 @@ class QuantizationConfig(ABC):
def get_linear_method(self) -> LinearMethodBase:
"""Get the linear method to use for the quantized linear layer."""
raise NotImplementedError
@abstractmethod
def get_scaled_act_names(self) -> List[str]:
"""Returns the activation function names that should be post-scaled.
For now, this is only used by AWQ.
"""
raise NotImplementedError
@@ -52,6 +52,9 @@ class SqueezeLLMConfig(QuantizationConfig):
def get_linear_method(self) -> "SqueezeLLMLinearMethod":
return SqueezeLLMLinearMethod(self)
def get_scaled_act_names(self) -> List[str]:
return []
class SqueezeLLMLinearMethod(LinearMethodBase):
"""Linear method for SqueezeLLM.