[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
+2
-3
@@ -26,12 +26,11 @@ void wvSplitKQ(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros, torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
bool use_v2_format);
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
torch::Tensor b_scales, bool use_v2_format);
|
||||
|
||||
void moe_gptq_gemm_rdna3(torch::Tensor a, torch::Tensor c,
|
||||
torch::Tensor b_q_weight, torch::Tensor b_scales,
|
||||
|
||||
+40
-63
@@ -253,11 +253,13 @@ __forceinline__ __device__ void load4_scales(const T* scales_row, int n,
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <typename T, int M_COUNT>
|
||||
__global__ void gemm_q4_kernel_rdna3(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
__global__ void gemm_q4_kernel_rdna3(const T* __restrict__ a,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_qzeros,
|
||||
const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m,
|
||||
const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset) {
|
||||
const int t = threadIdx.x;
|
||||
const int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
|
||||
const int offset_m = blockIdx.y * M_COUNT;
|
||||
@@ -274,8 +276,8 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
constexpr int LDS_PAD = 8;
|
||||
__shared__ T block_a[M_COUNT][BLOCK_KN_SIZE + LDS_PAD];
|
||||
|
||||
// Stage A: each thread loads 1 K element per M row into LDS (with optional
|
||||
// act-order permutation). THREADS_X == BLOCK_KN_SIZE so this is a 1:1 map.
|
||||
// Stage A: each thread loads 1 K element per M row into LDS.
|
||||
// THREADS_X == BLOCK_KN_SIZE so this is a 1:1 map.
|
||||
// For M_COUNT > 1 with size_m not a multiple of M_COUNT, slots past size_m
|
||||
// are zero-padded so the dot product contribution is 0 (we then skip the
|
||||
// atomic write for those rows below).
|
||||
@@ -298,10 +300,7 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
T av;
|
||||
if (offset_m + m < size_m) {
|
||||
const T* a_row = a + (offset_m + m) * size_k;
|
||||
if (b_q_perm)
|
||||
av = a_row[b_q_perm[offset_k + t]];
|
||||
else
|
||||
av = a_row[offset_k + t];
|
||||
av = a_row[offset_k + t];
|
||||
} else {
|
||||
av = tzero<T>(); // zero-pad invalid M rows
|
||||
}
|
||||
@@ -314,14 +313,6 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
// the LDS load above — but here all THREADS_X (=256) threads always do,
|
||||
// regardless of whether their `n` is in bounds.
|
||||
__syncthreads();
|
||||
} else if (b_q_perm) {
|
||||
// bf16 M=1 fast path skips LDS, but its global read below is sequential
|
||||
// and cannot apply act-order. When a permutation is present, stage the
|
||||
// single A row through LDS (as fp16 / M>1 do) so the read picks it up.
|
||||
// b_q_perm is block-uniform, so the __syncthreads is non-divergent.
|
||||
if (offset_k + t < end_k)
|
||||
block_a[0][t] = a[offset_m * size_k + b_q_perm[offset_k + t]];
|
||||
__syncthreads();
|
||||
}
|
||||
if (n >= size_n) return;
|
||||
|
||||
@@ -452,14 +443,11 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
// fp32-aliased union. Storing as uint32 keeps the IR-level type
|
||||
// opaque so the inner v_dot2 cannot be folded to fp32 widening.
|
||||
//
|
||||
// A is read direct from global (no LDS staging — see USE_LDS_A above),
|
||||
// except under act-order, where it comes from the permuted LDS copy.
|
||||
// A is read direct from global (no LDS staging — see USE_LDS_A above).
|
||||
pack4 a_pack;
|
||||
{
|
||||
const uint32_t* a_words =
|
||||
b_q_perm
|
||||
? reinterpret_cast<const uint32_t*>(&block_a[0][a_off])
|
||||
: reinterpret_cast<const uint32_t*>(a + offset_k + a_off);
|
||||
reinterpret_cast<const uint32_t*>(a + offset_k + a_off);
|
||||
a_pack.u[0] = a_words[0];
|
||||
a_pack.u[1] = a_words[1];
|
||||
a_pack.u[2] = a_words[2];
|
||||
@@ -623,8 +611,7 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
template <typename T, int M_COUNT>
|
||||
__global__ void gemm_q4_kernel_rdna3(const T*, const uint32_t*, const uint32_t*,
|
||||
const T*, T*, const int, const int,
|
||||
const int, const int, const int,
|
||||
const int*) {}
|
||||
const int, const int, const int) {}
|
||||
|
||||
#endif // __HIP__RDNA3__ || !__HIP_DEVICE_COMPILE__
|
||||
|
||||
@@ -635,17 +622,17 @@ __global__ void gemm_q4_kernel_rdna3(const T*, const uint32_t*, const uint32_t*,
|
||||
template <typename T, int M_COUNT>
|
||||
void launch_gemm_q4_for_mcount(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
dim3 block(THREADS_X);
|
||||
dim3 grid((size_n + BLOCK_KN_SIZE * 4 - 1) / (BLOCK_KN_SIZE * 4),
|
||||
(size_m + M_COUNT - 1) / M_COUNT,
|
||||
(size_k + BLOCK_KN_SIZE - 1) / BLOCK_KN_SIZE);
|
||||
|
||||
gemm_q4_kernel_rdna3<T, M_COUNT><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_kernel_rdna3<T, M_COUNT>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
// Dispatch to the largest M_COUNT template that doesn't waste more than
|
||||
@@ -661,31 +648,30 @@ void launch_gemm_q4_for_mcount(const T* a, const uint32_t* b_q_weight,
|
||||
// only burn instructions on the last block, never affect correctness.
|
||||
template <typename T>
|
||||
void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m, int size_n,
|
||||
int size_k, int groups, bool use_v2_format,
|
||||
cudaStream_t stream) {
|
||||
const uint32_t* b_qzeros, const T* b_scales, T* c,
|
||||
int size_m, int size_n, int size_k, int groups,
|
||||
bool use_v2_format, cudaStream_t stream) {
|
||||
const int zero_offset = use_v2_format ? 0 : 1;
|
||||
|
||||
if (size_m == 1) {
|
||||
launch_gemm_q4_for_mcount<T, 1>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 1>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else if (size_m <= 3) {
|
||||
launch_gemm_q4_for_mcount<T, 2>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 2>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else if (size_m <= 7) {
|
||||
launch_gemm_q4_for_mcount<T, 4>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 4>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else {
|
||||
// M_COUNT=8 covers M up to 15 here; M >= 16 should ideally take the
|
||||
// WMMA path, but if it falls through we still produce correct output —
|
||||
// just leaving 3-5× of throughput on the table for prefill workloads.
|
||||
launch_gemm_q4_for_mcount<T, 8>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 8>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -701,7 +687,6 @@ void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
// b_q_weight[K/8, N] uint32 (already shuffled via gptq_shuffle)
|
||||
// b_qzeros [groups, N/8] uint32 (packed 4-bit zeros)
|
||||
// b_scales [groups, N] half or bfloat16
|
||||
// b_g_idx [K] or empty int32 (act-order permutation; empty=identity)
|
||||
// use_v2_format bool (true = GPTQv2, no +1 zero offset)
|
||||
//
|
||||
// Output:
|
||||
@@ -709,17 +694,16 @@ void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
torch::Tensor b_scales, bool use_v2_format);
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros, torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format) {
|
||||
bool use_v2_format) {
|
||||
if (a.dim() == 2 && b_q_weight.dim() == 2 && a.size(1) % 16 == 0 &&
|
||||
b_q_weight.size(1) % 16 == 0 &&
|
||||
((a.scalar_type() == torch::kBFloat16 && a.size(0) >= 16) ||
|
||||
(a.scalar_type() == torch::kHalf && a.size(0) >= 64))) {
|
||||
return gptq_gemm_rdna3_wmma(a, b_q_weight, b_qzeros, b_scales, b_g_idx,
|
||||
return gptq_gemm_rdna3_wmma(a, b_q_weight, b_qzeros, b_scales,
|
||||
use_v2_format);
|
||||
}
|
||||
|
||||
@@ -753,25 +737,18 @@ torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
auto opts = torch::TensorOptions().dtype(a.dtype()).device(a.device());
|
||||
at::Tensor c = torch::zeros({size_m, size_n}, opts);
|
||||
|
||||
const int* g_idx_ptr = nullptr;
|
||||
if (!b_g_idx.device().is_meta() && b_g_idx.numel() > 0) {
|
||||
TORCH_CHECK(b_g_idx.scalar_type() == torch::kInt32,
|
||||
"b_g_idx must be int32");
|
||||
g_idx_ptr = (const int*)b_g_idx.data_ptr();
|
||||
}
|
||||
|
||||
if (a.scalar_type() == torch::kHalf) {
|
||||
vllm::gptq_rdna3::launch_gemm_q4<half>(
|
||||
(const half*)a.data_ptr(), (const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(), (const half*)b_scales.data_ptr(),
|
||||
g_idx_ptr, (half*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
use_v2_format, stream);
|
||||
(half*)c.data_ptr(), size_m, size_n, size_k, groups, use_v2_format,
|
||||
stream);
|
||||
} else {
|
||||
vllm::gptq_rdna3::launch_gemm_q4<vllm::gptq_rdna3::bf16_t>(
|
||||
(const vllm::gptq_rdna3::bf16_t*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(),
|
||||
(const vllm::gptq_rdna3::bf16_t*)b_scales.data_ptr(), g_idx_ptr,
|
||||
(const vllm::gptq_rdna3::bf16_t*)b_scales.data_ptr(),
|
||||
(vllm::gptq_rdna3::bf16_t*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
use_v2_format, stream);
|
||||
}
|
||||
|
||||
+83
-142
@@ -373,7 +373,7 @@ __global__ void gemm_q4_wmma_kernel_16x16_1w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -471,25 +471,16 @@ __global__ void gemm_q4_wmma_kernel_16x16_1w(
|
||||
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
// Permuted (act-order): scattered global reads, no vectorization.
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
// Sequential A reads: replace 16 single-element global_load_b16 with
|
||||
// a bulk 32-byte copy. The AMDGPU backend lowers a memcpy of this
|
||||
// size + alignment to two `global_load_b128` instructions. size_k is
|
||||
// a multiple of 16 (TORCH_CHECK above) and k_tile increments by 16,
|
||||
// so k_tile + 16 is always within bounds — no tail handling needed.
|
||||
// Note: we memcpy into the whole vector (`&a_frag`) rather than
|
||||
// `&a_frag[0]`; ext_vector_type element addresses aren't reliably
|
||||
// valid C pointers across compiler versions.
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes (16 × 2)");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
// Sequential A reads: replace 16 single-element global_load_b16 with
|
||||
// a bulk 32-byte copy. The AMDGPU backend lowers a memcpy of this
|
||||
// size + alignment to two `global_load_b128` instructions. size_k is
|
||||
// a multiple of 16 (TORCH_CHECK above) and k_tile increments by 16,
|
||||
// so k_tile + 16 is always within bounds — no tail handling needed.
|
||||
// Note: we memcpy into the whole vector (`&a_frag`) rather than
|
||||
// `&a_frag[0]`; ext_vector_type element addresses aren't reliably
|
||||
// valid C pointers across compiler versions.
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes (16 × 2)");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -597,25 +588,24 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_16x16_1w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_16x16_1w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// 1 wave per block (32 lanes), 16x16 C tile per block. gridDim.z splits
|
||||
// K so that more blocks (and therefore more waves) are in flight; with
|
||||
// K_SPLIT > 1 the kernel switches to atomic write-back at the epilogue.
|
||||
const int k_split = compute_wmma_k_split(size_k);
|
||||
dim3 block(32);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 15) / 16, k_split);
|
||||
gemm_q4_wmma_kernel_16x16_1w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_16x16_1w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -657,7 +647,7 @@ __global__ void gemm_q4_wmma_kernel_32x16_2w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -757,16 +747,8 @@ __global__ void gemm_q4_wmma_kernel_32x16_2w(
|
||||
V16 a_frag, b_frag;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -844,16 +826,15 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_32x16_2w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fallback to v1 for size_m < 32. With M-tile=32 the v2 block has 2 waves
|
||||
// working on rows [0..15] and [16..31]; at M < 32 the second wave processes
|
||||
// out-of-range M rows (zero-padded a_frag → wmma produces nothing useful)
|
||||
@@ -862,9 +843,9 @@ void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
// max-num-seqs=32 lands at M≈32 steady-state; the M=16 sliver is edge),
|
||||
// but the fallback costs nothing and is the right shape.
|
||||
if (size_m < 32) {
|
||||
launch_gemm_q4_wmma_16x16_1w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_16x16_1w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -875,9 +856,9 @@ void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split(size_k);
|
||||
dim3 block(64);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 31) / 32, k_split);
|
||||
gemm_q4_wmma_kernel_32x16_2w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_32x16_2w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -917,7 +898,7 @@ __global__ void gemm_q4_wmma_kernel_64x16_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1019,16 +1000,8 @@ __global__ void gemm_q4_wmma_kernel_64x16_4w(
|
||||
V16 a_frag, b_frag;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1100,21 +1073,20 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x16_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x16_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v2 for M < 64 (would waste 1+ waves on out-of-range rows).
|
||||
if (size_m < 64) {
|
||||
launch_gemm_q4_wmma_32x16_2w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_32x16_2w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1122,9 +1094,9 @@ void launch_gemm_q4_wmma_64x16_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 16);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x16_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x16_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -1158,7 +1130,7 @@ __global__ void gemm_q4_wmma_kernel_64x32_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1264,16 +1236,8 @@ __global__ void gemm_q4_wmma_kernel_64x32_4w(
|
||||
V16 a_frag, b_frag0, b_frag1;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1352,23 +1316,22 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x32_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x32_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v3 when M < 64 (small-M decode/prefill stays on the
|
||||
// narrower 64M × 16N path) or when N < 32 (tile would waste a wave on
|
||||
// out-of-range cols).
|
||||
if (size_m < 64 || size_n < 32) {
|
||||
launch_gemm_q4_wmma_64x16_4w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_64x16_4w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1376,9 +1339,9 @@ void launch_gemm_q4_wmma_64x32_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 32);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 31) / 32, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x32_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x32_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -1412,7 +1375,7 @@ __global__ void gemm_q4_wmma_kernel_64x64_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1510,16 +1473,8 @@ __global__ void gemm_q4_wmma_kernel_64x64_4w(
|
||||
V16 a_frag, b_frag0, b_frag1, b_frag2, b_frag3;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1611,7 +1566,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k16(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1712,8 +1667,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k16(
|
||||
dequant_into(next_buf, k_next);
|
||||
}
|
||||
|
||||
// A-load: vectorized 256-bit. b_q_perm branch removed from V7 hot path
|
||||
// to eliminate ~450 ISA instructions of dead code (6× icache bloat).
|
||||
// A-load: vectorized 256-bit.
|
||||
V16 a_frag, b_frag0, b_frag1, b_frag2, b_frag3;
|
||||
if (a_row_ptr) {
|
||||
__builtin_memcpy(&a_frag, a_row_ptr + k_tile, sizeof(a_frag));
|
||||
@@ -1810,7 +1764,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k32(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -2006,40 +1960,36 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x64_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_128x64_k16(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int,
|
||||
const int*) {}
|
||||
const int, const int) {}
|
||||
template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_128x64_k32(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int,
|
||||
const int*) {}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v4 when N < 64 (would waste 1+ waves on out-of-range cols).
|
||||
if (size_m < 64 || size_n < 64) {
|
||||
launch_gemm_q4_wmma_64x32_4w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_64x32_4w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
// V8 (128M × 64N, K=32/iter, 8-wave dequant) when K%32==0 and gs≥32.
|
||||
// Falls back to V7 otherwise. V7/V8 read A sequentially, so act-order
|
||||
// (b_q_perm != null) must skip them and use v5, which honors the perm.
|
||||
if (size_m >= 128 && b_q_perm == nullptr) {
|
||||
// Falls back to V7 otherwise.
|
||||
if (size_m >= 128) {
|
||||
const int k_split =
|
||||
compute_wmma_k_split_mn(size_m, size_n, size_k, 128, 64);
|
||||
const int groupsize = size_k / groups;
|
||||
@@ -2048,11 +1998,11 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
if (size_k % 32 == 0 && groupsize >= 32 && (size_k / k_split) % 32 == 0) {
|
||||
gemm_q4_wmma_kernel_128x64_k32<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
zero_offset);
|
||||
} else {
|
||||
gemm_q4_wmma_kernel_128x64_k16<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
zero_offset);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -2061,9 +2011,9 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 64);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 63) / 64, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x64_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x64_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
} // namespace gptq_rdna3_wmma
|
||||
@@ -2078,7 +2028,6 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
// b_q_weight[K/8, N] uint32 (already shuffled via gptq_shuffle)
|
||||
// b_qzeros [groups, N/8] uint32 (packed 4-bit zeros)
|
||||
// b_scales [groups, N] half or bfloat16
|
||||
// b_g_idx [K] or empty int32 (act-order permutation; empty=identity)
|
||||
// use_v2_format bool (true = GPTQv2, no +1 zero offset)
|
||||
//
|
||||
// Output:
|
||||
@@ -2091,8 +2040,7 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format) {
|
||||
torch::Tensor b_scales, bool use_v2_format) {
|
||||
TORCH_CHECK(a.is_cuda(), "a must be a CUDA/HIP tensor");
|
||||
TORCH_CHECK(b_q_weight.is_cuda(), "b_q_weight must be a CUDA/HIP tensor");
|
||||
TORCH_CHECK(b_qzeros.is_cuda(), "b_qzeros must be a CUDA/HIP tensor");
|
||||
@@ -2128,13 +2076,6 @@ torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
// negligible (< 1.5% of prefill time on gfx1100).
|
||||
at::Tensor c = torch::zeros({size_m, size_n}, opts);
|
||||
|
||||
const int* g_idx_ptr = nullptr;
|
||||
if (!b_g_idx.device().is_meta() && b_g_idx.numel() > 0) {
|
||||
TORCH_CHECK(b_g_idx.scalar_type() == torch::kInt32,
|
||||
"b_g_idx must be int32");
|
||||
g_idx_ptr = (const int*)b_g_idx.data_ptr();
|
||||
}
|
||||
|
||||
const int zero_offset = use_v2_format ? 0 : 1;
|
||||
|
||||
// launch_gemm_q4_wmma_64x64_4w dispatches:
|
||||
@@ -2148,15 +2089,15 @@ torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
vllm::gptq_rdna3_wmma::launch_gemm_q4_wmma_64x64_4w<half>(
|
||||
(const half*)a.data_ptr(), (const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(), (const half*)b_scales.data_ptr(),
|
||||
g_idx_ptr, (half*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
(half*)c.data_ptr(), size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else {
|
||||
vllm::gptq_rdna3_wmma::launch_gemm_q4_wmma_64x64_4w<
|
||||
vllm::gptq_rdna3_wmma::bf16_t>(
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(),
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)b_scales.data_ptr(), g_idx_ptr,
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)b_scales.data_ptr(),
|
||||
(vllm::gptq_rdna3_wmma::bf16_t*)c.data_ptr(), size_m, size_n, size_k,
|
||||
groups, zero_offset, stream);
|
||||
}
|
||||
|
||||
@@ -56,12 +56,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
|
||||
// W4A16 GPTQ kernels for AMD RDNA3 (gfx1100).
|
||||
rocm_ops.def(
|
||||
"gptq_gemm_rdna3(Tensor a, Tensor b_q_weight, Tensor b_qzeros, "
|
||||
"Tensor b_scales, Tensor b_g_idx, bool use_v2_format) -> Tensor");
|
||||
"Tensor b_scales, bool use_v2_format) -> Tensor");
|
||||
rocm_ops.impl("gptq_gemm_rdna3", torch::kCUDA, &gptq_gemm_rdna3);
|
||||
|
||||
rocm_ops.def(
|
||||
"gptq_gemm_rdna3_wmma(Tensor a, Tensor b_q_weight, Tensor b_qzeros, "
|
||||
"Tensor b_scales, Tensor b_g_idx, bool use_v2_format) -> Tensor");
|
||||
"Tensor b_scales, bool use_v2_format) -> Tensor");
|
||||
rocm_ops.impl("gptq_gemm_rdna3_wmma", torch::kCUDA, &gptq_gemm_rdna3_wmma);
|
||||
|
||||
rocm_ops.def(
|
||||
|
||||
Reference in New Issue
Block a user