[Kernel] add bfloat16 support for gptq marlin kernel (#4788)

This commit is contained in:
Jinzhen Lin
2024-05-16 09:55:29 -04:00
committed by GitHub
parent 5c342570d7
commit 99caa49106
4 changed files with 245 additions and 72 deletions
@@ -99,7 +99,7 @@ class GPTQMarlinConfig(QuantizationConfig):
@classmethod
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
return [torch.half]
return [torch.half, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
@@ -186,9 +186,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
group_size = input_size
# Validate dtype
if params_dtype != torch.float16:
raise ValueError(
f"The params dtype must be float16, but got {params_dtype}")
if params_dtype not in [torch.float16, torch.bfloat16]:
raise ValueError(f"The params dtype must be float16 "
f"or bfloat16, but got {params_dtype}")
# Validate output_size_per_partition
output_size_per_partition = sum(output_partition_sizes)