Add minimum capability requirement for AWQ (#1064)

This commit is contained in:
Woosuk Kwon
2023-09-18 12:02:01 -07:00
committed by GitHub
parent cc796b1358
commit 2b1c116b5a
5 changed files with 47 additions and 2 deletions
@@ -40,6 +40,11 @@ class AWQConfig(QuantizationConfig):
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
return [torch.half]
@classmethod
def get_min_capability(cls) -> int:
# The AWQ kernel only supports Ampere or newer GPUs.
return 80
@classmethod
def get_config_filenames(cls) -> List[str]:
return [
@@ -15,6 +15,16 @@ class QuantizationConfig:
"""List of supported activation dtypes."""
raise NotImplementedError
@classmethod
def get_min_capability(cls) -> int:
"""Minimum GPU capability to support the quantization method.
E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
This requirement is due to the custom CUDA kernels used by the
quantization method.
"""
raise NotImplementedError
@classmethod
def get_config_filenames(cls) -> List[str]:
"""List of filenames to search for in the model directory."""