[ROCm][Perf] W4A16: pad gfx11 weight and activation strides (#56301)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
|
||||
#include <stdexcept>
|
||||
#include <algorithm>
|
||||
#include <utility> // std::in_range
|
||||
|
||||
#include "../cuda_compat.h"
|
||||
#include "dispatch_utils.h"
|
||||
@@ -121,6 +122,7 @@ __device__ __forceinline__ uint32_t zp_nibble(const uint32_t* zp_packed,
|
||||
// GROUP_SIZE: 0 = per-channel scale [M], >0 = per-group scale [M,
|
||||
// K/GROUP_SIZE].
|
||||
// Requires GROUP_SIZE % A_CHUNK == 0 when GROUP_SIZE > 0.
|
||||
// B_row_stride_bytes is the per-row byte stride of the packed weights.
|
||||
#if defined(__HIP__GFX1X__)
|
||||
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
|
||||
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
|
||||
@@ -130,9 +132,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
const scalar_t* __restrict__ A, const scalar_t* scale,
|
||||
const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS, scalar_t* C,
|
||||
const int _WvPrGrp, const int CuCount) {
|
||||
const int _WvPrGrp, const int CuCount,
|
||||
const int B_row_stride_bytes) {
|
||||
constexpr int max_lds_len = LDS_SIZE / 2;
|
||||
const int K_packed = K / 2;
|
||||
|
||||
union bigTypeA {
|
||||
scalar_t h[A_CHUNK];
|
||||
@@ -181,9 +183,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
uint32_t k_ = k + threadIdx.x * A_CHUNK;
|
||||
if (k_ >= K) break;
|
||||
|
||||
const uint8_t* B_ = &B_packed[(m + 0) * K_packed + k_ / 2];
|
||||
const uint8_t* B_ = &B_packed[(m + 0) * B_row_stride_bytes + k_ / 2];
|
||||
for (int y = 0; y < YTILE; y++) {
|
||||
const float* src = (const float*)(&B_[y * K_packed]);
|
||||
const float* src = (const float*)(&B_[y * B_row_stride_bytes]);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < A_CHUNK / 8; i++)
|
||||
bigB[y][k2].f[i] = loadnt((float*)&src[i]);
|
||||
@@ -344,14 +346,12 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
#else // !defined(__HIP__GFX1X__)
|
||||
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
|
||||
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
|
||||
__global__ void wvSplitK_int4_hf_sml_(const int K, const int M, const int Bx,
|
||||
const int By, const uint8_t* B_packed,
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* scale,
|
||||
const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS,
|
||||
scalar_t* C, const int _WvPrGrp,
|
||||
const int CuCount) {
|
||||
__global__ void wvSplitK_int4_hf_sml_(
|
||||
const int K, const int M, const int Bx, const int By,
|
||||
const uint8_t* B_packed, const scalar_t* __restrict__ A,
|
||||
const scalar_t* scale, const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS, scalar_t* C, const int _WvPrGrp,
|
||||
const int CuCount, const int B_row_stride_bytes) {
|
||||
UNREACHABLE_CODE
|
||||
}
|
||||
#endif // defined(__HIP__GFX1X__)
|
||||
@@ -367,9 +367,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
const uint8_t* B_packed, const scalar_t* __restrict__ A,
|
||||
const scalar_t* scale, const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS, scalar_t* C,
|
||||
const int _WvPrGrp, const int CuCount) {
|
||||
const int _WvPrGrp, const int CuCount,
|
||||
const int B_row_stride_bytes) {
|
||||
constexpr int max_lds_len = LDS_SIZE / 2;
|
||||
const int K_packed = K / 2;
|
||||
|
||||
union bigTypeA {
|
||||
scalar_t h[A_CHUNK];
|
||||
@@ -430,9 +430,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
uint32_t k_ = k + threadIdx.x * A_CHUNK;
|
||||
if (k_ >= K) break;
|
||||
|
||||
const uint8_t* B_ = &B_packed[(m + 0) * K_packed + k_ / 2];
|
||||
const uint8_t* B_ = &B_packed[(m + 0) * B_row_stride_bytes + k_ / 2];
|
||||
for (int y = 0; y < YTILE; y++) {
|
||||
const float* src = (const float*)(&B_[y * K_packed]);
|
||||
const float* src = (const float*)(&B_[y * B_row_stride_bytes]);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < A_CHUNK / 8; i++)
|
||||
bigB[y][k2].f[i] = loadnt((float*)&src[i]);
|
||||
@@ -606,14 +606,12 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
|
||||
#else // !defined(__HIP__GFX1X__)
|
||||
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
|
||||
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
|
||||
__global__ void wvSplitK_int4_hf_(const int K, const int M, const int Bx,
|
||||
const int By, const uint8_t* B_packed,
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* scale,
|
||||
const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS,
|
||||
scalar_t* C, const int _WvPrGrp,
|
||||
const int CuCount) {
|
||||
__global__ void wvSplitK_int4_hf_(
|
||||
const int K, const int M, const int Bx, const int By,
|
||||
const uint8_t* B_packed, const scalar_t* __restrict__ A,
|
||||
const scalar_t* scale, const uint32_t* zero_points,
|
||||
const scalar_t* __restrict__ BIAS, scalar_t* C, const int _WvPrGrp,
|
||||
const int CuCount, const int B_row_stride_bytes) {
|
||||
UNREACHABLE_CODE
|
||||
}
|
||||
#endif // defined(__HIP__GFX1X__)
|
||||
@@ -658,10 +656,17 @@ torch::Tensor wvSplitK_int4_g(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
? in_bias->size(0)
|
||||
: 1;
|
||||
|
||||
int64_t expected_weight_bytes = M_in * K_in / 2;
|
||||
int64_t actual_weight_bytes = in_a.numel() * in_a.element_size();
|
||||
const int64_t expected_weight_bytes = M_in * K_in / 2;
|
||||
const int64_t actual_weight_bytes = in_a.numel() * in_a.element_size();
|
||||
TORCH_CHECK(actual_weight_bytes == expected_weight_bytes,
|
||||
"Weight tensor must contain M*K/2 bytes for int4 packing");
|
||||
const int64_t b_row_stride_bytes = in_a.stride(0) * in_a.element_size();
|
||||
TORCH_CHECK(b_row_stride_bytes >= K_in / 2, "Weight row stride (",
|
||||
b_row_stride_bytes, " bytes) must hold at least K/2=", K_in / 2,
|
||||
" bytes per row");
|
||||
TORCH_CHECK(std::in_range<int>(b_row_stride_bytes), "Weight row stride (",
|
||||
b_row_stride_bytes, " bytes) exceeds int range");
|
||||
const int b_row_stride_bytes_i32 = static_cast<int>(b_row_stride_bytes);
|
||||
TORCH_CHECK(
|
||||
in_b.dtype() == torch::kFloat16 || in_b.dtype() == torch::kBFloat16,
|
||||
"Activation must be float16 or bfloat16");
|
||||
@@ -722,12 +727,12 @@ torch::Tensor wvSplitK_int4_g(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
wvSplitK_int4_hf_sml_<fptype, _THRDS, _YTILE, 16, 16, _UNRL, _N, _GS, \
|
||||
_HAS_ZP><<<grid, block, 0, stream>>>( \
|
||||
K_in, M_in, Bx_in, By_in, wptr, aptr, sptr, zpptr, biasptr, cptr, \
|
||||
__wvPrGrp, CuCount); \
|
||||
__wvPrGrp, CuCount, b_row_stride_bytes_i32); \
|
||||
else \
|
||||
wvSplitK_int4_hf_<fptype, _THRDS, _YTILE, 16, 16, _UNRL, _N, _GS, \
|
||||
_HAS_ZP><<<grid, block, 0, stream>>>( \
|
||||
K_in, M_in, Bx_in, By_in, wptr, aptr, sptr, zpptr, biasptr, cptr, \
|
||||
__wvPrGrp, CuCount); \
|
||||
__wvPrGrp, CuCount, b_row_stride_bytes_i32); \
|
||||
}
|
||||
|
||||
#define WVSPLITK_INT4G(_YTILE, _UNRL, _N, _GS, _HAS_ZP) \
|
||||
|
||||
Reference in New Issue
Block a user