AQLM CUDA support (#3287)

Co-authored-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
James Fleming
2024-04-23 13:59:33 -04:00
committed by GitHub
co-authored by mgoin
parent 62b5166bd4
commit 2b7949c1c2
14 changed files with 1592 additions and 11 deletions
@@ -81,7 +81,7 @@ class AWQLinearMethod(LinearMethodBase):
def create_weights(self, layer: torch.nn.Module,
input_size_per_partition: int,
output_size_per_partition: int, input_size: int,
output_partition_sizes: List[int], input_size: int,
output_size: int, params_dtype: torch.dtype,
**extra_weight_attrs):
if input_size_per_partition % self.quant_config.group_size != 0:
@@ -89,6 +89,8 @@ class AWQLinearMethod(LinearMethodBase):
"The input size is not aligned with the quantized "
"weight shape. This can be caused by too large "
"tensor parallel size.")
output_size_per_partition = sum(output_partition_sizes)
if output_size_per_partition % self.quant_config.pack_factor != 0:
raise ValueError(
"The output size is not aligned with the quantized "