[bugfix] Fix static asymmetric quantization case (#10334)
Signed-off-by: Daniël de Kok <me@danieldk.eu> Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: Daniël de Kok <me@danieldk.eu>
This commit is contained in:
co-authored by
Daniël de Kok
parent
972112d82f
commit
bf2ddc6610
+7
-1
@@ -510,10 +510,16 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
|
||||
azp_adj: torch.Tensor,
|
||||
azp: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
"""
|
||||
:param azp_adj: In the per-tensor case, this should include the azp.
|
||||
Always per-channel.
|
||||
:param azp: Only set in the per-token case. Per-token if set.
|
||||
"""
|
||||
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
|
||||
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
|
||||
assert bias is None or bias.numel(
|
||||
) == b.shape[1] and bias.dtype == out_dtype
|
||||
assert azp is None or azp.numel() == a.shape[0]
|
||||
|
||||
m = a.shape[0]
|
||||
n = b.shape[1]
|
||||
@@ -735,7 +741,7 @@ def scaled_int8_quant(
|
||||
azp is
|
||||
None), "azp must only be provided for asymmetric quantization."
|
||||
torch.ops._C.static_scaled_int8_quant(output, input, scale, azp)
|
||||
return output, scale, None
|
||||
return output, scale, azp
|
||||
|
||||
# dynamic-per-token quantization.
|
||||
input_scales = torch.empty((input.numel() // input.shape[-1], 1),
|
||||
|
||||
Reference in New Issue
Block a user