[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
+112 -423
View File
@@ -121,7 +121,7 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,
const uint32_t*, const half*,
half*, const int, const int,
const int, const int,
const bool, const int*);
const bool);
template <bool first_block, int m_count>
__global__ void gemm_half_q_half_gptq_4bit_kernel(
@@ -129,7 +129,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
const int size_m, const int size_n, const int size_k, const int groups,
const bool use_v2_format, const int* __restrict__ b_q_perm) {
const bool use_v2_format) {
MatrixView_half a_(a, size_m, size_k);
MatrixView_half_rw c_(c, size_m, size_n);
MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
@@ -157,12 +157,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
half* block_a_ptr = block_a[m];
half a0;
if (b_q_perm)
a0 = a_ptr[b_q_perm[offset_k + t]];
else
a0 = a_ptr[offset_k + t];
block_a_ptr[t] = a0;
block_a_ptr[t] = a_ptr[offset_k + t];
}
}
@@ -263,7 +258,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
const int size_m, const int size_n, const int size_k, const int groups,
const bool use_v2_format, const int* __restrict__ b_q_perm) {
const bool use_v2_format) {
MatrixView_half a_(a, size_m, size_k);
MatrixView_half_rw c_(c, size_m, size_n);
MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
@@ -291,12 +286,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
half* block_a_ptr = block_a[m];
half a0;
if (b_q_perm)
a0 = a_ptr[b_q_perm[offset_k + t]];
else
a0 = a_ptr[offset_k + t];
block_a_ptr[t] = a0;
block_a_ptr[t] = a_ptr[offset_k + t];
}
}
@@ -380,7 +370,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
const int size_m, const int size_n, const int size_k, const int groups,
const bool use_v2_format, const int* __restrict__ b_q_perm) {
const bool use_v2_format) {
MatrixView_half a_(a, size_m, size_k);
MatrixView_half_rw c_(c, size_m, size_n);
MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
@@ -408,12 +398,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
half* block_a_ptr = block_a[m];
half a0;
if (b_q_perm)
a0 = a_ptr[b_q_perm[offset_k + t]];
else
a0 = a_ptr[offset_k + t];
block_a_ptr[t] = a0;
block_a_ptr[t] = a_ptr[offset_k + t];
}
}
@@ -504,7 +489,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
const int size_m, const int size_n, const int size_k, const int groups,
const bool use_v2_format, const int* __restrict__ b_q_perm) {
const bool use_v2_format) {
MatrixView_half a_(a, size_m, size_k);
MatrixView_half_rw c_(c, size_m, size_n);
MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
@@ -532,12 +517,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
half* block_a_ptr = block_a[m];
half a0;
if (b_q_perm)
a0 = a_ptr[b_q_perm[offset_k + t]];
else
a0 = a_ptr[offset_k + t];
block_a_ptr[t] = a0;
block_a_ptr[t] = a_ptr[offset_k + t];
}
}
@@ -656,10 +636,9 @@ fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
const uint32_t* b_gptq_qzeros,
const half* b_gptq_scales, const int* b_q_perm,
half* c, int size_m, int size_n, int size_k,
int m_count, int groups, bool use_v2_format,
int bit) {
const half* b_gptq_scales, half* c, int size_m,
int size_n, int size_k, int m_count, int groups,
bool use_v2_format, int bit) {
dim3 blockDim, gridDim;
blockDim.x = BLOCK_KN_SIZE;
blockDim.y = 1;
@@ -672,13 +651,13 @@ void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
const cudaStream_t stream = get_current_cuda_stream();
kernel<<<gridDim, blockDim, 0, stream>>>(
a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k,
groups, use_v2_format, b_q_perm);
kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,
b_gptq_scales, c, size_m, size_n,
size_k, groups, use_v2_format);
}
__global__ void reconstruct_exllama_8bit_kernel(
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
const uint32_t* __restrict__ b_q_weight,
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
const int groups, const bool use_v2_format, half* __restrict__ b) {
@@ -694,14 +673,8 @@ __global__ void reconstruct_exllama_8bit_kernel(
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
// Preload remapping table
__shared__ int perm[BLOCK_KN_SIZE];
auto t = threadIdx.x;
if (b_q_perm) {
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
}
// Column
int n = offset_n + t * 4;
if (n >= size_n) return;
@@ -722,8 +695,6 @@ __global__ void reconstruct_exllama_8bit_kernel(
b_gptq_qzeros_.item4(zeros, group, n);
b_gptq_scales_.item4_h2(scales, group, n);
__syncthreads();
int k = offset_k;
int lk = 0;
@@ -752,25 +723,13 @@ __global__ void reconstruct_exllama_8bit_kernel(
dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
zeros[3] + zero_offset);
// half* dqh = (half*)dq;
if (b_q_perm) {
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
__high2half(dq[2][j]), __high2half(dq[3][j]));
}
} else {
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
__low2half(dq[1][j]), __low2half(dq[2][j]),
__low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
}
k += 32;
@@ -778,7 +737,7 @@ __global__ void reconstruct_exllama_8bit_kernel(
}
__global__ void reconstruct_exllama_4bit_kernel(
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
const uint32_t* __restrict__ b_q_weight,
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
const int groups, const bool use_v2_format, half* __restrict__ b) {
@@ -794,14 +753,8 @@ __global__ void reconstruct_exllama_4bit_kernel(
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
// Preload remapping table
__shared__ int perm[BLOCK_KN_SIZE];
auto t = threadIdx.x;
if (b_q_perm) {
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
}
// Column
int n = offset_n + t * 4;
if (n >= size_n) return;
@@ -828,8 +781,6 @@ __global__ void reconstruct_exllama_4bit_kernel(
dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);
dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);
__syncthreads();
int k = offset_k;
int lk = 0;
@@ -860,25 +811,13 @@ __global__ void reconstruct_exllama_4bit_kernel(
false);
b_ptr += size_n;
// half* dqh = (half*)dq;
if (b_q_perm) {
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
__high2half(dq[2][j]), __high2half(dq[3][j]));
}
} else {
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
__low2half(dq[1][j]), __low2half(dq[2][j]),
__low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
for (int j = 0; j < 4; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
}
k += 32;
@@ -886,7 +825,7 @@ __global__ void reconstruct_exllama_4bit_kernel(
}
__global__ void reconstruct_exllama_3bit_kernel(
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
const uint32_t* __restrict__ b_q_weight,
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
const int groups, const bool use_v2_format, half* __restrict__ b) {
@@ -902,14 +841,8 @@ __global__ void reconstruct_exllama_3bit_kernel(
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
// Preload remapping table
__shared__ int perm[BLOCK_KN_SIZE];
auto t = threadIdx.x;
if (b_q_perm) {
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
}
// Column
int n = offset_n + t * 4;
if (n >= size_n) return;
@@ -930,8 +863,6 @@ __global__ void reconstruct_exllama_3bit_kernel(
b_gptq_qzeros_.item4(zeros, group, n);
b_gptq_scales_.item4_h2(scales, group, n);
__syncthreads();
int k = offset_k;
int lk = 0;
@@ -962,24 +893,13 @@ __global__ void reconstruct_exllama_3bit_kernel(
dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
size_n, zeros[3] + zero_offset);
if (b_q_perm) {
for (int j = 0; j < 16; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
__high2half(dq[2][j]), __high2half(dq[3][j]));
}
} else {
for (int j = 0; j < 16; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
__low2half(dq[1][j]), __low2half(dq[2][j]),
__low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
for (int j = 0; j < 16; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
}
k += 32;
@@ -987,7 +907,7 @@ __global__ void reconstruct_exllama_3bit_kernel(
}
__global__ void reconstruct_exllama_2bit_kernel(
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
const uint32_t* __restrict__ b_q_weight,
const uint32_t* __restrict__ b_gptq_qzeros,
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
const int groups, const bool use_v2_format, half* __restrict__ b) {
@@ -1003,14 +923,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
// Preload remapping table
__shared__ int perm[BLOCK_KN_SIZE];
auto t = threadIdx.x;
if (b_q_perm) {
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
}
// Column
int n = offset_n + t * 4;
if (n >= size_n) return;
@@ -1031,8 +945,6 @@ __global__ void reconstruct_exllama_2bit_kernel(
b_gptq_qzeros_.item4(zeros, group, n);
b_gptq_scales_.item4_h2(scales, group, n);
__syncthreads();
int k = offset_k;
int lk = 0;
@@ -1055,25 +967,13 @@ __global__ void reconstruct_exllama_2bit_kernel(
dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + zero_offset);
b_ptr += size_n;
// half* dqh = (half*)dq;
if (b_q_perm) {
for (int j = 0; j < 8; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
__high2half(dq[2][j]), __high2half(dq[3][j]));
}
} else {
for (int j = 0; j < 8; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
__low2half(dq[1][j]), __low2half(dq[2][j]),
__low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
for (int j = 0; j < 8; j++) {
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
__low2half(dq[2][j]), __low2half(dq[3][j]));
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
__high2half(dq[1][j]), __high2half(dq[2][j]),
__high2half(dq[3][j]));
}
}
k += 32;
@@ -1082,9 +982,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
void reconstruct_exllama(const uint32_t* b_q_weight,
const uint32_t* b_gptq_qzeros,
const half* b_gptq_scales, const int* b_q_perm,
half* out, int height, int width, int groups,
bool use_v2_format, int bit) {
const half* b_gptq_scales, half* out, int height,
int width, int groups, bool use_v2_format, int bit) {
dim3 blockDim, gridDim;
blockDim.x = BLOCK_KN_SIZE;
blockDim.y = 1;
@@ -1102,15 +1001,15 @@ void reconstruct_exllama(const uint32_t* b_q_weight,
const cudaStream_t stream = get_current_cuda_stream();
reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(
b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,
b_q_weight, b_gptq_qzeros, b_gptq_scales, height, width, groups,
use_v2_format, out);
}
__global__ void gemm_half_q_half_alt_4bit_kernel(
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
half* __restrict__ mul, const half* __restrict__ scales,
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
int batch, int height, int width, bool use_v2_format) {
const uint32_t* __restrict__ zeros, int batch, int height, int width,
int groups, bool use_v2_format) {
int zero_width = width / 8;
int vec_height = height * 4;
const int blockwidth2 = BLOCK_KN_SIZE / 2;
@@ -1144,6 +1043,7 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
int i = width * h + w;
int g_h = h * 8;
int group_size = vec_height * 2 / groups;
int k = 0;
int z_w = w / 8;
int z_mod = (w % 8) * 4;
@@ -1156,8 +1056,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
half2 scales_tmp[4];
half2 zeros_tmp[4];
for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
int g = g_idx[g_h + (k + tmp_k) * 2];
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
int g = (g_h + (k + tmp_k) * 2) / group_size;
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
half scale_f = scales[g * width + w];
half scale_f2 = scales[g2 * width + w];
half2 scale = __halves2half2(scale_f, scale_f2);
@@ -1209,8 +1109,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
__global__ void gemm_half_q_half_alt_8bit_kernel(
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
half* __restrict__ mul, const half* __restrict__ scales,
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
int batch, int height, int width, bool use_v2_format) {
const uint32_t* __restrict__ zeros, int batch, int height, int width,
int groups, bool use_v2_format) {
int zero_width = width / 4;
int vec_height = height * 2;
const int blockwidth2 = BLOCK_KN_SIZE / 2;
@@ -1236,6 +1136,7 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
int i = width * h + w;
int g_h = h * 4;
int group_size = vec_height * 2 / groups;
int k = 0;
int z_w = w / 4;
int z_mod = (w % 4) * 8;
@@ -1248,8 +1149,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
half2 scales_tmp[2];
half2 zeros_tmp[2];
for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
int g = g_idx[g_h + (k + tmp_k) * 2];
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
int g = (g_h + (k + tmp_k) * 2) / group_size;
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
half scale_f = scales[g * width + w];
half scale_f2 = scales[g2 * width + w];
half2 scale = __halves2half2(scale_f, scale_f2);
@@ -1296,8 +1197,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
const uint32_t* b_gptq_qzeros,
const half* b_gptq_scales, const int* b_g_idx,
half* c, int size_m, int size_n, int size_k,
const half* b_gptq_scales, half* c, int size_m,
int size_n, int size_k, int groups,
bool use_v2_format, int bit) {
dim3 blockDim, gridDim;
blockDim.x = BLOCK_KN_SIZE;
@@ -1314,16 +1215,15 @@ void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
const cudaStream_t stream = get_current_cuda_stream();
kernel<<<gridDim, blockDim, 0, stream>>>(
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,
size_m, size_k / 32 * bit, size_n, use_v2_format);
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, size_m,
size_k / 32 * bit, size_n, groups, use_v2_format);
}
template <class T, int bit>
__global__ void reconstruct_gptq_kernel(
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
const int height, const int width, const int group,
const bool use_v2_format, half* __restrict__ out) {
const uint32_t* __restrict__ w_zeros, const int height, const int width,
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
// Start of block
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
@@ -1333,8 +1233,8 @@ __global__ void reconstruct_gptq_kernel(
// Views
MatrixView_half_rw out_(out, height, width);
MatrixView_half w_scales_(w_scales, group, width);
T w_zeros_(w_zeros, group, width);
MatrixView_half w_scales_(w_scales, num_groups, width);
T w_zeros_(w_zeros, num_groups, width);
// GPTQv2 and GPTQv1 handles zero points differently
int zero_offset = use_v2_format ? 0 : 1;
@@ -1344,7 +1244,7 @@ __global__ void reconstruct_gptq_kernel(
#pragma unroll
for (int s = 0; s < 32; s += bit) {
int group = g_idx[row + s / bit];
int group = (row + s / bit) / (height / num_groups);
half w_scale = w_scales_.item(group, column);
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
half w_item =
@@ -1357,9 +1257,8 @@ __global__ void reconstruct_gptq_kernel(
__global__ void reconstruct_gptq_3bit_kernel(
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
const int height, const int width, const int group,
const bool use_v2_format, half* __restrict__ out) {
const uint32_t* __restrict__ w_zeros, const int height, const int width,
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
// Start of block
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
auto row = blockIdx.y * 32;
@@ -1368,8 +1267,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
// Views
MatrixView_half_rw out_(out, height, width);
MatrixView_half w_scales_(w_scales, group, width);
MatrixView_q3_row w_zeros_(w_zeros, group, width);
MatrixView_half w_scales_(w_scales, num_groups, width);
MatrixView_q3_row w_zeros_(w_zeros, num_groups, width);
// GPTQv2 and GPTQv1 handles zero points differently
int zero_offset = use_v2_format ? 0 : 1;
@@ -1381,7 +1280,7 @@ __global__ void reconstruct_gptq_3bit_kernel(
#pragma unroll
for (int i = 0; i < 32; i += 1) {
int group = g_idx[row + i];
int group = (row + i) / (height / num_groups);
half w_scale = w_scales_.item(group, column);
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
int w_item;
@@ -1402,9 +1301,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
}
void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
const half* b_gptq_scales, const int* b_g_idx, half* out,
int height, int width, int groups, bool use_v2_format,
int bit) {
const half* b_gptq_scales, half* out, int height,
int width, int groups, bool use_v2_format, int bit) {
dim3 blockDim, gridDim;
blockDim.x = BLOCK_KN_SIZE;
blockDim.y = 1;
@@ -1423,17 +1321,16 @@ void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
const cudaStream_t stream = get_current_cuda_stream();
kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,
b_gptq_qzeros, b_g_idx, height,
width, groups, use_v2_format, out);
b_gptq_qzeros, height, width, groups,
use_v2_format, out);
}
void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
const uint32_t* b_q_weight,
const uint32_t* b_gptq_qzeros,
const half* b_gptq_scales, const int* b_g_idx,
half* c, half* temp_dq, int size_m, int size_n,
int size_k, int groups, bool use_exllama,
bool use_v2_format, int bit) {
const half* b_gptq_scales, half* c, half* temp_dq,
int size_m, int size_n, int size_k, int groups,
bool use_exllama, bool use_v2_format, int bit) {
bool use_reconstruct;
if (use_exllama) {
use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||
@@ -1446,11 +1343,11 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
if (use_reconstruct) {
// Reconstruct FP16 matrix, then cuBLAS
if (use_exllama) {
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
temp_dq, size_k, size_n, groups, use_v2_format, bit);
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
size_k, size_n, groups, use_v2_format, bit);
} else {
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
temp_dq, size_k, size_n, groups, use_v2_format, bit);
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
size_k, size_n, groups, use_v2_format, bit);
}
const half alpha = __float2half(1.0f);
@@ -1464,20 +1361,20 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
int last_chunk_size = size_m - last_chunk;
if (max_chunks) {
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
b_g_idx, c, last_chunk, size_n, size_k,
BLOCK_M_SIZE_MAX, groups, use_v2_format, bit);
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c,
last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
groups, use_v2_format, bit);
}
if (last_chunk_size) {
gemm_half_q_half_cuda_part(
a + last_chunk * size_k, b_q_weight, b_gptq_qzeros, b_gptq_scales,
b_g_idx, c + last_chunk * size_n, last_chunk_size, size_n, size_k,
c + last_chunk * size_n, last_chunk_size, size_n, size_k,
last_chunk_size, groups, use_v2_format, bit);
}
} else {
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
c, size_m, size_n, size_k, use_v2_format, bit);
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m,
size_n, size_k, groups, use_v2_format, bit);
}
}
@@ -1533,214 +1430,8 @@ __global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight,
}
}
__global__ void make_sequential_4bit_kernel(const uint32_t* __restrict__ w,
uint32_t* __restrict__ w_new,
const int* __restrict__ q_perm,
const int w_width) {
const uint64_t* w2 = (uint64_t*)w;
uint64_t* w_new2 = (uint64_t*)w_new;
int w2_stride = w_width >> 1;
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
if (w2_column >= w2_stride) return;
auto w_new2_row = blockIdx.y;
int q_perm_idx = w_new2_row << 3;
uint64_t dst = 0;
#pragma unroll
for (int i = 0; i < 8; i++) {
int source_row = q_perm[q_perm_idx++];
int w2_row = source_row >> 3;
int w2_subrow = source_row & 0x07;
int w2_row_shift = w2_subrow << 2;
int wnew2_row_shift = i << 2;
uint64_t src = w2[w2_row * w2_stride + w2_column];
src >>= w2_row_shift;
src &= 0x0000000f0000000f;
src <<= wnew2_row_shift;
dst |= src;
}
w_new2[w_new2_row * w2_stride + w2_column] = dst;
}
__global__ void make_sequential_2bit_kernel(const uint32_t* __restrict__ w,
uint32_t* __restrict__ w_new,
const int* __restrict__ q_perm,
const int w_width) {
const uint64_t* w2 = (uint64_t*)w;
uint64_t* w_new2 = (uint64_t*)w_new;
int w2_stride = w_width >> 1;
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
if (w2_column >= w2_stride) return;
auto w_new2_row = blockIdx.y;
int q_perm_idx = w_new2_row << 4;
uint64_t dst = 0;
#pragma unroll
for (int i = 0; i < 16; i++) {
int source_row = q_perm[q_perm_idx++];
int w2_row = source_row >> 4;
int w2_subrow = source_row & 0x0f;
int w2_row_shift = w2_subrow << 1;
int wnew2_row_shift = i << 1;
uint64_t src = w2[w2_row * w2_stride + w2_column];
src >>= w2_row_shift;
src &= 0x0000000300000003;
src <<= wnew2_row_shift;
dst |= src;
}
w_new2[w_new2_row * w2_stride + w2_column] = dst;
}
__global__ void make_sequential_3bit_kernel(const uint32_t* __restrict__ w,
uint32_t* __restrict__ w_new,
const int* __restrict__ q_perm,
const int w_width) {
auto w_column = THREADS_X * blockIdx.x + threadIdx.x;
if (w_column >= w_width) return;
auto w_new_row = blockIdx.y * 3;
auto q_perm_idx = blockIdx.y << 5;
uint32_t dst[3] = {0, 0, 0};
#pragma unroll
for (int i = 0; i < 32; i++) {
int source_row = q_perm[q_perm_idx++];
int z_w = (source_row / 32) * 3;
int z_mod = source_row % 32;
int z_bit;
if (z_mod != 10) {
if (z_mod != 21) {
z_bit = z_mod;
if (z_bit > 21) {
z_bit *= 3;
z_bit -= 64;
z_w += 2;
} else if (z_bit > 10) {
z_bit *= 3;
z_bit -= 32;
z_w += 1;
} else {
z_bit *= 3;
}
} else {
z_w += 1;
}
}
uint64_t src;
if (z_mod == 10) {
src = (w[z_w * w_width + w_column] >> 30) |
((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
} else if (z_mod == 21) {
src = (w[z_w * w_width + w_column] >> 31) |
((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
} else {
src = w[z_w * w_width + w_column];
src >>= z_bit;
src &= 0x07;
}
z_w = 0;
if (i != 10) {
if (i != 21) {
z_bit = i;
if (z_bit > 21) {
z_bit *= 3;
z_bit -= 64;
z_w += 2;
} else if (z_bit > 10) {
z_bit *= 3;
z_bit -= 32;
z_w += 1;
} else {
z_bit *= 3;
}
} else {
z_w += 1;
}
}
if (i == 10) {
dst[z_w] |= (src & 0x03) << 30;
dst[z_w + 1] |= ((src & 0x4) >> 2);
} else if (i == 21) {
dst[z_w] |= (src & 0x01) << 31;
dst[z_w + 1] |= ((src & 0x6) >> 1);
} else {
dst[z_w] |= (src << z_bit);
}
}
w_new[w_new_row * w_width + w_column] = dst[0];
w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
}
__global__ void make_sequential_8bit_kernel(const uint32_t* __restrict__ w,
uint32_t* __restrict__ w_new,
const int* __restrict__ q_perm,
const int w_width) {
const uint64_t* w2 = (uint64_t*)w;
uint64_t* w_new2 = (uint64_t*)w_new;
int w2_stride = w_width >> 1;
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
if (w2_column >= w2_stride) return;
auto w_new2_row = blockIdx.y;
int q_perm_idx = w_new2_row << 2;
uint64_t dst = 0;
#pragma unroll
for (int i = 0; i < 4; i++) {
int source_row = q_perm[q_perm_idx++];
int w2_row = source_row >> 2;
int w2_subrow = source_row & 0x03;
int w2_row_shift = w2_subrow << 3;
int wnew2_row_shift = i << 3;
uint64_t src = w2[w2_row * w2_stride + w2_column];
src >>= w2_row_shift;
src &= 0x000000ff000000ff;
src <<= wnew2_row_shift;
dst |= src;
}
w_new2[w_new2_row * w2_stride + w2_column] = dst;
}
void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
int width, int bit) {
if (q_perm) {
uint32_t* new_qweight = NULL;
cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
dim3 blockDim, gridDim;
blockDim.x = THREADS_X;
blockDim.y = 1;
gridDim.x = DIVIDE(width, THREADS_X);
gridDim.y = height / 32 * bit;
auto kernel = make_sequential_4bit_kernel;
if (bit == 2) {
kernel = make_sequential_2bit_kernel;
} else if (bit == 3) {
kernel = make_sequential_3bit_kernel;
gridDim.y = height / 32;
} else if (bit == 8) {
kernel = make_sequential_8bit_kernel;
}
const cudaStream_t stream = get_current_cuda_stream();
kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, new_qweight, q_perm,
width);
// Replace qweights
cudaMemcpyAsync(q_weight, new_qweight,
height / 32 * bit * width * sizeof(uint32_t),
cudaMemcpyDeviceToDevice);
// Cleanup
cudaDeviceSynchronize();
cudaFree(new_qweight);
}
void shuffle_exllama_weight(uint32_t* q_weight, int height, int width,
int bit) {
dim3 blockDim, gridDim;
blockDim.x = THREADS_X;
blockDim.y = 1;
@@ -1765,10 +1456,17 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
torch::stable::Tensor b_q_weight,
torch::stable::Tensor b_gptq_qzeros,
torch::stable::Tensor b_gptq_scales,
torch::stable::Tensor b_g_idx, bool use_exllama,
bool use_v2_format, int64_t bit) {
bool use_exllama, bool use_v2_format,
int64_t bit) {
const torch::stable::accelerator::DeviceGuard device_guard(
a.get_device_index());
const int64_t num_groups = b_gptq_qzeros.size(0);
STD_TORCH_CHECK(num_groups > 0, "GPTQ requires at least one scale group");
STD_TORCH_CHECK(b_gptq_scales.size(0) == num_groups,
"GPTQ scales and zero-points must have the same number of "
"groups");
STD_TORCH_CHECK(a.size(1) % num_groups == 0, "Input size K = ", a.size(1),
" is not divisible by the number of groups = ", num_groups);
auto c = torch::stable::new_zeros(a, {a.size(0), b_q_weight.size(1)});
auto temp_dq =
torch::stable::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)},
@@ -1778,28 +1476,19 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
get_current_cuda_blas_handle(), (const half*)a.data_ptr(),
(const uint32_t*)b_q_weight.data_ptr(),
(const uint32_t*)b_gptq_qzeros.data_ptr(),
(const half*)b_gptq_scales.data_ptr(),
b_g_idx.device().type() == torch::stable::DeviceType::Meta
? NULL
: (const int*)b_g_idx.data_ptr(),
(half*)c.data_ptr(), (half*)temp_dq.data_ptr(),
c.size(0), // m
c.size(1), // n
a.size(1), // k
b_gptq_qzeros.size(0), // group number
use_exllama, use_v2_format, bit);
(const half*)b_gptq_scales.data_ptr(), (half*)c.data_ptr(),
(half*)temp_dq.data_ptr(),
c.size(0), // m
c.size(1), // n
a.size(1), // k
num_groups, use_exllama, use_v2_format, bit);
return c;
}
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
int64_t bit) {
void gptq_shuffle(torch::stable::Tensor q_weight, int64_t bit) {
const torch::stable::accelerator::DeviceGuard device_guard(
q_weight.get_device_index());
vllm::gptq::shuffle_exllama_weight(
(uint32_t*)q_weight.data_ptr(),
q_perm.device().type() == torch::stable::DeviceType::Meta ||
q_perm.numel() == 0
? NULL
: (int*)q_perm.data_ptr(),
q_weight.size(0) * 32 / bit, q_weight.size(1), bit);
vllm::gptq::shuffle_exllama_weight((uint32_t*)q_weight.data_ptr(),
q_weight.size(0) * 32 / bit,
q_weight.size(1), bit);
}