[Bugfix] Fix GPTQ and GPTQ Marlin CPU Offloading (#7225)

This commit is contained in:
Michael Goin
2024-08-06 18:34:26 -07:00
committed by GitHub
parent fd95e026e0
commit f9a5600649
4 changed files with 33 additions and 14 deletions
@@ -204,13 +204,7 @@ class GPTQLinearMethod(LinearMethodBase):
layer.exllama_state = exllama_state
def apply(self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
qweight = layer.qweight
out_shape = x.shape[:-1] + (qweight.shape[-1], )
reshaped_x = x.reshape(-1, x.shape[-1])
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# exllama needs to shuffle the weight after the weight is loaded
# here we do the shuffle on first forward pass
if layer.exllama_state == ExllamaState.UNINITIALIZED:
@@ -222,6 +216,14 @@ class GPTQLinearMethod(LinearMethodBase):
layer.exllama_state = ExllamaState.READY
ops.gptq_shuffle(layer.qweight, layer.g_idx,
self.quant_config.weight_bits)
def apply(self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
out_shape = x.shape[:-1] + (layer.qweight.shape[-1], )
reshaped_x = x.reshape(-1, x.shape[-1])
output = ops.gptq_gemm(reshaped_x, layer.qweight, layer.qzeros,
layer.scales, layer.g_idx,
layer.exllama_state == ExllamaState.READY,
@@ -251,7 +251,6 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
scales_and_zp_size,
output_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int32,
device="meta",
),
requires_grad=False,
)