[Kernel] AQ AZP 3/4: Asymmetric quantization kernels (#7270)

This commit is contained in:
Luka Govedič
2024-09-16 11:52:40 -07:00
committed by GitHub
parent 781e3b9a42
commit 5d73ae49d6
9 changed files with 339 additions and 57 deletions
@@ -260,7 +260,7 @@ class QQQLinearMethod(LinearMethodBase):
size_k = x_2d.shape[1]
size_n = s_ch.shape[1]
x_int8, s_tok = ops.scaled_int8_quant(x_2d)
x_int8, s_tok, _ = ops.scaled_int8_quant(x_2d)
output_2d = ops.marlin_qqq_gemm(x_int8, qweight, s_tok, s_ch, s_group,
workspace, size_m, size_n, size_k)
@@ -188,7 +188,7 @@ def apply_int8_linear(
# ops.scaled_int8_quant supports both dynamic and static quant.
# * dynamic, layer.input_scale is None and x_scale computed from x.
# * static, layer.input_scale is scalar and x_scale is input_scale.
x_q, x_scale = ops.scaled_int8_quant(input, input_scale)
x_q, x_scale, _ = ops.scaled_int8_quant(input, input_scale)
return ops.cutlass_scaled_mm(x_q,
weight,