[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -121,7 +121,7 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,
|
||||
const uint32_t*, const half*,
|
||||
half*, const int, const int,
|
||||
const int, const int,
|
||||
const bool, const int*);
|
||||
const bool);
|
||||
|
||||
template <bool first_block, int m_count>
|
||||
__global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
@@ -129,7 +129,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -157,12 +157,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,7 +258,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -291,12 +286,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,7 +370,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -408,12 +398,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -504,7 +489,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -532,12 +517,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -656,10 +636,9 @@ fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
|
||||
|
||||
void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_q_perm,
|
||||
half* c, int size_m, int size_n, int size_k,
|
||||
int m_count, int groups, bool use_v2_format,
|
||||
int bit) {
|
||||
const half* b_gptq_scales, half* c, int size_m,
|
||||
int size_n, int size_k, int m_count, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -672,13 +651,13 @@ void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
|
||||
pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k,
|
||||
groups, use_v2_format, b_q_perm);
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,
|
||||
b_gptq_scales, c, size_m, size_n,
|
||||
size_k, groups, use_v2_format);
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_8bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -694,14 +673,8 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -722,8 +695,6 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -752,25 +723,13 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
|
||||
zeros[3] + zero_offset);
|
||||
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -778,7 +737,7 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_4bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -794,14 +753,8 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -828,8 +781,6 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);
|
||||
dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -860,25 +811,13 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
false);
|
||||
|
||||
b_ptr += size_n;
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -886,7 +825,7 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_3bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -902,14 +841,8 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -930,8 +863,6 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -962,24 +893,13 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
|
||||
size_n, zeros[3] + zero_offset);
|
||||
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -987,7 +907,7 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_2bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -1003,14 +923,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -1031,8 +945,6 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -1055,25 +967,13 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + zero_offset);
|
||||
|
||||
b_ptr += size_n;
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -1082,9 +982,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
|
||||
void reconstruct_exllama(const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_q_perm,
|
||||
half* out, int height, int width, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
const half* b_gptq_scales, half* out, int height,
|
||||
int width, int groups, bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -1102,15 +1001,15 @@ void reconstruct_exllama(const uint32_t* b_q_weight,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,
|
||||
b_q_weight, b_gptq_qzeros, b_gptq_scales, height, width, groups,
|
||||
use_v2_format, out);
|
||||
}
|
||||
|
||||
__global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
|
||||
half* __restrict__ mul, const half* __restrict__ scales,
|
||||
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
|
||||
int batch, int height, int width, bool use_v2_format) {
|
||||
const uint32_t* __restrict__ zeros, int batch, int height, int width,
|
||||
int groups, bool use_v2_format) {
|
||||
int zero_width = width / 8;
|
||||
int vec_height = height * 4;
|
||||
const int blockwidth2 = BLOCK_KN_SIZE / 2;
|
||||
@@ -1144,6 +1043,7 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
|
||||
int i = width * h + w;
|
||||
int g_h = h * 8;
|
||||
int group_size = vec_height * 2 / groups;
|
||||
int k = 0;
|
||||
int z_w = w / 8;
|
||||
int z_mod = (w % 8) * 4;
|
||||
@@ -1156,8 +1056,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
half2 scales_tmp[4];
|
||||
half2 zeros_tmp[4];
|
||||
for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
|
||||
int g = g_idx[g_h + (k + tmp_k) * 2];
|
||||
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
|
||||
int g = (g_h + (k + tmp_k) * 2) / group_size;
|
||||
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
|
||||
half scale_f = scales[g * width + w];
|
||||
half scale_f2 = scales[g2 * width + w];
|
||||
half2 scale = __halves2half2(scale_f, scale_f2);
|
||||
@@ -1209,8 +1109,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
__global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
|
||||
half* __restrict__ mul, const half* __restrict__ scales,
|
||||
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
|
||||
int batch, int height, int width, bool use_v2_format) {
|
||||
const uint32_t* __restrict__ zeros, int batch, int height, int width,
|
||||
int groups, bool use_v2_format) {
|
||||
int zero_width = width / 4;
|
||||
int vec_height = height * 2;
|
||||
const int blockwidth2 = BLOCK_KN_SIZE / 2;
|
||||
@@ -1236,6 +1136,7 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
|
||||
int i = width * h + w;
|
||||
int g_h = h * 4;
|
||||
int group_size = vec_height * 2 / groups;
|
||||
int k = 0;
|
||||
int z_w = w / 4;
|
||||
int z_mod = (w % 4) * 8;
|
||||
@@ -1248,8 +1149,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
half2 scales_tmp[2];
|
||||
half2 zeros_tmp[2];
|
||||
for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
|
||||
int g = g_idx[g_h + (k + tmp_k) * 2];
|
||||
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
|
||||
int g = (g_h + (k + tmp_k) * 2) / group_size;
|
||||
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
|
||||
half scale_f = scales[g * width + w];
|
||||
half scale_f2 = scales[g2 * width + w];
|
||||
half2 scale = __halves2half2(scale_f, scale_f2);
|
||||
@@ -1296,8 +1197,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
|
||||
void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx,
|
||||
half* c, int size_m, int size_n, int size_k,
|
||||
const half* b_gptq_scales, half* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
@@ -1314,16 +1215,15 @@ void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,
|
||||
size_m, size_k / 32 * bit, size_n, use_v2_format);
|
||||
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, size_m,
|
||||
size_k / 32 * bit, size_n, groups, use_v2_format);
|
||||
}
|
||||
|
||||
template <class T, int bit>
|
||||
__global__ void reconstruct_gptq_kernel(
|
||||
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
|
||||
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
|
||||
const int height, const int width, const int group,
|
||||
const bool use_v2_format, half* __restrict__ out) {
|
||||
const uint32_t* __restrict__ w_zeros, const int height, const int width,
|
||||
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
|
||||
// Start of block
|
||||
|
||||
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
|
||||
@@ -1333,8 +1233,8 @@ __global__ void reconstruct_gptq_kernel(
|
||||
// Views
|
||||
|
||||
MatrixView_half_rw out_(out, height, width);
|
||||
MatrixView_half w_scales_(w_scales, group, width);
|
||||
T w_zeros_(w_zeros, group, width);
|
||||
MatrixView_half w_scales_(w_scales, num_groups, width);
|
||||
T w_zeros_(w_zeros, num_groups, width);
|
||||
|
||||
// GPTQv2 and GPTQv1 handles zero points differently
|
||||
int zero_offset = use_v2_format ? 0 : 1;
|
||||
@@ -1344,7 +1244,7 @@ __global__ void reconstruct_gptq_kernel(
|
||||
|
||||
#pragma unroll
|
||||
for (int s = 0; s < 32; s += bit) {
|
||||
int group = g_idx[row + s / bit];
|
||||
int group = (row + s / bit) / (height / num_groups);
|
||||
half w_scale = w_scales_.item(group, column);
|
||||
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
|
||||
half w_item =
|
||||
@@ -1357,9 +1257,8 @@ __global__ void reconstruct_gptq_kernel(
|
||||
|
||||
__global__ void reconstruct_gptq_3bit_kernel(
|
||||
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
|
||||
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
|
||||
const int height, const int width, const int group,
|
||||
const bool use_v2_format, half* __restrict__ out) {
|
||||
const uint32_t* __restrict__ w_zeros, const int height, const int width,
|
||||
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
|
||||
// Start of block
|
||||
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
|
||||
auto row = blockIdx.y * 32;
|
||||
@@ -1368,8 +1267,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
// Views
|
||||
|
||||
MatrixView_half_rw out_(out, height, width);
|
||||
MatrixView_half w_scales_(w_scales, group, width);
|
||||
MatrixView_q3_row w_zeros_(w_zeros, group, width);
|
||||
MatrixView_half w_scales_(w_scales, num_groups, width);
|
||||
MatrixView_q3_row w_zeros_(w_zeros, num_groups, width);
|
||||
|
||||
// GPTQv2 and GPTQv1 handles zero points differently
|
||||
int zero_offset = use_v2_format ? 0 : 1;
|
||||
@@ -1381,7 +1280,7 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; i += 1) {
|
||||
int group = g_idx[row + i];
|
||||
int group = (row + i) / (height / num_groups);
|
||||
half w_scale = w_scales_.item(group, column);
|
||||
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
|
||||
int w_item;
|
||||
@@ -1402,9 +1301,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
}
|
||||
|
||||
void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx, half* out,
|
||||
int height, int width, int groups, bool use_v2_format,
|
||||
int bit) {
|
||||
const half* b_gptq_scales, half* out, int height,
|
||||
int width, int groups, bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -1423,17 +1321,16 @@ void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,
|
||||
b_gptq_qzeros, b_g_idx, height,
|
||||
width, groups, use_v2_format, out);
|
||||
b_gptq_qzeros, height, width, groups,
|
||||
use_v2_format, out);
|
||||
}
|
||||
|
||||
void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx,
|
||||
half* c, half* temp_dq, int size_m, int size_n,
|
||||
int size_k, int groups, bool use_exllama,
|
||||
bool use_v2_format, int bit) {
|
||||
const half* b_gptq_scales, half* c, half* temp_dq,
|
||||
int size_m, int size_n, int size_k, int groups,
|
||||
bool use_exllama, bool use_v2_format, int bit) {
|
||||
bool use_reconstruct;
|
||||
if (use_exllama) {
|
||||
use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||
|
||||
@@ -1446,11 +1343,11 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
if (use_reconstruct) {
|
||||
// Reconstruct FP16 matrix, then cuBLAS
|
||||
if (use_exllama) {
|
||||
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
temp_dq, size_k, size_n, groups, use_v2_format, bit);
|
||||
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
|
||||
size_k, size_n, groups, use_v2_format, bit);
|
||||
} else {
|
||||
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
temp_dq, size_k, size_n, groups, use_v2_format, bit);
|
||||
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
|
||||
size_k, size_n, groups, use_v2_format, bit);
|
||||
}
|
||||
|
||||
const half alpha = __float2half(1.0f);
|
||||
@@ -1464,20 +1361,20 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
int last_chunk_size = size_m - last_chunk;
|
||||
|
||||
if (max_chunks) {
|
||||
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
|
||||
b_g_idx, c, last_chunk, size_n, size_k,
|
||||
BLOCK_M_SIZE_MAX, groups, use_v2_format, bit);
|
||||
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c,
|
||||
last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
|
||||
groups, use_v2_format, bit);
|
||||
}
|
||||
|
||||
if (last_chunk_size) {
|
||||
gemm_half_q_half_cuda_part(
|
||||
a + last_chunk * size_k, b_q_weight, b_gptq_qzeros, b_gptq_scales,
|
||||
b_g_idx, c + last_chunk * size_n, last_chunk_size, size_n, size_k,
|
||||
c + last_chunk * size_n, last_chunk_size, size_n, size_k,
|
||||
last_chunk_size, groups, use_v2_format, bit);
|
||||
}
|
||||
} else {
|
||||
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
c, size_m, size_n, size_k, use_v2_format, bit);
|
||||
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m,
|
||||
size_n, size_k, groups, use_v2_format, bit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1533,214 +1430,8 @@ __global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight,
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void make_sequential_4bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 3;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 3;
|
||||
int w2_subrow = source_row & 0x07;
|
||||
int w2_row_shift = w2_subrow << 2;
|
||||
int wnew2_row_shift = i << 2;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x0000000f0000000f;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
__global__ void make_sequential_2bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 4;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 4;
|
||||
int w2_subrow = source_row & 0x0f;
|
||||
int w2_row_shift = w2_subrow << 1;
|
||||
int wnew2_row_shift = i << 1;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x0000000300000003;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
__global__ void make_sequential_3bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
auto w_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w_column >= w_width) return;
|
||||
auto w_new_row = blockIdx.y * 3;
|
||||
auto q_perm_idx = blockIdx.y << 5;
|
||||
uint32_t dst[3] = {0, 0, 0};
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
int z_w = (source_row / 32) * 3;
|
||||
int z_mod = source_row % 32;
|
||||
int z_bit;
|
||||
|
||||
if (z_mod != 10) {
|
||||
if (z_mod != 21) {
|
||||
z_bit = z_mod;
|
||||
if (z_bit > 21) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 64;
|
||||
z_w += 2;
|
||||
} else if (z_bit > 10) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 32;
|
||||
z_w += 1;
|
||||
} else {
|
||||
z_bit *= 3;
|
||||
}
|
||||
} else {
|
||||
z_w += 1;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t src;
|
||||
if (z_mod == 10) {
|
||||
src = (w[z_w * w_width + w_column] >> 30) |
|
||||
((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
|
||||
} else if (z_mod == 21) {
|
||||
src = (w[z_w * w_width + w_column] >> 31) |
|
||||
((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
|
||||
} else {
|
||||
src = w[z_w * w_width + w_column];
|
||||
src >>= z_bit;
|
||||
src &= 0x07;
|
||||
}
|
||||
|
||||
z_w = 0;
|
||||
if (i != 10) {
|
||||
if (i != 21) {
|
||||
z_bit = i;
|
||||
if (z_bit > 21) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 64;
|
||||
z_w += 2;
|
||||
} else if (z_bit > 10) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 32;
|
||||
z_w += 1;
|
||||
} else {
|
||||
z_bit *= 3;
|
||||
}
|
||||
} else {
|
||||
z_w += 1;
|
||||
}
|
||||
}
|
||||
if (i == 10) {
|
||||
dst[z_w] |= (src & 0x03) << 30;
|
||||
dst[z_w + 1] |= ((src & 0x4) >> 2);
|
||||
} else if (i == 21) {
|
||||
dst[z_w] |= (src & 0x01) << 31;
|
||||
dst[z_w + 1] |= ((src & 0x6) >> 1);
|
||||
} else {
|
||||
dst[z_w] |= (src << z_bit);
|
||||
}
|
||||
}
|
||||
w_new[w_new_row * w_width + w_column] = dst[0];
|
||||
w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
|
||||
w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
|
||||
}
|
||||
|
||||
__global__ void make_sequential_8bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 2;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 2;
|
||||
int w2_subrow = source_row & 0x03;
|
||||
int w2_row_shift = w2_subrow << 3;
|
||||
int wnew2_row_shift = i << 3;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x000000ff000000ff;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
|
||||
int width, int bit) {
|
||||
if (q_perm) {
|
||||
uint32_t* new_qweight = NULL;
|
||||
cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
|
||||
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = THREADS_X;
|
||||
blockDim.y = 1;
|
||||
gridDim.x = DIVIDE(width, THREADS_X);
|
||||
gridDim.y = height / 32 * bit;
|
||||
|
||||
auto kernel = make_sequential_4bit_kernel;
|
||||
if (bit == 2) {
|
||||
kernel = make_sequential_2bit_kernel;
|
||||
} else if (bit == 3) {
|
||||
kernel = make_sequential_3bit_kernel;
|
||||
gridDim.y = height / 32;
|
||||
} else if (bit == 8) {
|
||||
kernel = make_sequential_8bit_kernel;
|
||||
}
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, new_qweight, q_perm,
|
||||
width);
|
||||
// Replace qweights
|
||||
cudaMemcpyAsync(q_weight, new_qweight,
|
||||
height / 32 * bit * width * sizeof(uint32_t),
|
||||
cudaMemcpyDeviceToDevice);
|
||||
// Cleanup
|
||||
cudaDeviceSynchronize();
|
||||
cudaFree(new_qweight);
|
||||
}
|
||||
void shuffle_exllama_weight(uint32_t* q_weight, int height, int width,
|
||||
int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = THREADS_X;
|
||||
blockDim.y = 1;
|
||||
@@ -1765,10 +1456,17 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
torch::stable::Tensor b_q_weight,
|
||||
torch::stable::Tensor b_gptq_qzeros,
|
||||
torch::stable::Tensor b_gptq_scales,
|
||||
torch::stable::Tensor b_g_idx, bool use_exllama,
|
||||
bool use_v2_format, int64_t bit) {
|
||||
bool use_exllama, bool use_v2_format,
|
||||
int64_t bit) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a.get_device_index());
|
||||
const int64_t num_groups = b_gptq_qzeros.size(0);
|
||||
STD_TORCH_CHECK(num_groups > 0, "GPTQ requires at least one scale group");
|
||||
STD_TORCH_CHECK(b_gptq_scales.size(0) == num_groups,
|
||||
"GPTQ scales and zero-points must have the same number of "
|
||||
"groups");
|
||||
STD_TORCH_CHECK(a.size(1) % num_groups == 0, "Input size K = ", a.size(1),
|
||||
" is not divisible by the number of groups = ", num_groups);
|
||||
auto c = torch::stable::new_zeros(a, {a.size(0), b_q_weight.size(1)});
|
||||
auto temp_dq =
|
||||
torch::stable::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)},
|
||||
@@ -1778,28 +1476,19 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
get_current_cuda_blas_handle(), (const half*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_gptq_qzeros.data_ptr(),
|
||||
(const half*)b_gptq_scales.data_ptr(),
|
||||
b_g_idx.device().type() == torch::stable::DeviceType::Meta
|
||||
? NULL
|
||||
: (const int*)b_g_idx.data_ptr(),
|
||||
(half*)c.data_ptr(), (half*)temp_dq.data_ptr(),
|
||||
c.size(0), // m
|
||||
c.size(1), // n
|
||||
a.size(1), // k
|
||||
b_gptq_qzeros.size(0), // group number
|
||||
use_exllama, use_v2_format, bit);
|
||||
(const half*)b_gptq_scales.data_ptr(), (half*)c.data_ptr(),
|
||||
(half*)temp_dq.data_ptr(),
|
||||
c.size(0), // m
|
||||
c.size(1), // n
|
||||
a.size(1), // k
|
||||
num_groups, use_exllama, use_v2_format, bit);
|
||||
return c;
|
||||
}
|
||||
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
|
||||
int64_t bit) {
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, int64_t bit) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
q_weight.get_device_index());
|
||||
vllm::gptq::shuffle_exllama_weight(
|
||||
(uint32_t*)q_weight.data_ptr(),
|
||||
q_perm.device().type() == torch::stable::DeviceType::Meta ||
|
||||
q_perm.numel() == 0
|
||||
? NULL
|
||||
: (int*)q_perm.data_ptr(),
|
||||
q_weight.size(0) * 32 / bit, q_weight.size(1), bit);
|
||||
vllm::gptq::shuffle_exllama_weight((uint32_t*)q_weight.data_ptr(),
|
||||
q_weight.size(0) * 32 / bit,
|
||||
q_weight.size(1), bit);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user