[ROCm][Perf] W4A16: pad gfx11 weight and activation strides (#56301)

Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
Matthias Gehre
2026-10-06 08:22:33 -05:00
committed by GitHub
parent 049507aa76
commit b267fe4881
4 changed files with 316 additions and 46 deletions
+33 -28
View File
@@ -8,6 +8,7 @@
#include <stdexcept>
#include <algorithm>
#include <utility> // std::in_range
#include "../cuda_compat.h"
#include "dispatch_utils.h"
@@ -121,6 +122,7 @@ __device__ __forceinline__ uint32_t zp_nibble(const uint32_t* zp_packed,
// GROUP_SIZE: 0 = per-channel scale [M], >0 = per-group scale [M,
// K/GROUP_SIZE].
// Requires GROUP_SIZE % A_CHUNK == 0 when GROUP_SIZE > 0.
// B_row_stride_bytes is the per-row byte stride of the packed weights.
#if defined(__HIP__GFX1X__)
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
@@ -130,9 +132,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
const scalar_t* __restrict__ A, const scalar_t* scale,
const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS, scalar_t* C,
const int _WvPrGrp, const int CuCount) {
const int _WvPrGrp, const int CuCount,
const int B_row_stride_bytes) {
constexpr int max_lds_len = LDS_SIZE / 2;
const int K_packed = K / 2;
union bigTypeA {
scalar_t h[A_CHUNK];
@@ -181,9 +183,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
uint32_t k_ = k + threadIdx.x * A_CHUNK;
if (k_ >= K) break;
const uint8_t* B_ = &B_packed[(m + 0) * K_packed + k_ / 2];
const uint8_t* B_ = &B_packed[(m + 0) * B_row_stride_bytes + k_ / 2];
for (int y = 0; y < YTILE; y++) {
const float* src = (const float*)(&B_[y * K_packed]);
const float* src = (const float*)(&B_[y * B_row_stride_bytes]);
#pragma unroll
for (int i = 0; i < A_CHUNK / 8; i++)
bigB[y][k2].f[i] = loadnt((float*)&src[i]);
@@ -344,14 +346,12 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
#else // !defined(__HIP__GFX1X__)
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
__global__ void wvSplitK_int4_hf_sml_(const int K, const int M, const int Bx,
const int By, const uint8_t* B_packed,
const scalar_t* __restrict__ A,
const scalar_t* scale,
const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS,
scalar_t* C, const int _WvPrGrp,
const int CuCount) {
__global__ void wvSplitK_int4_hf_sml_(
const int K, const int M, const int Bx, const int By,
const uint8_t* B_packed, const scalar_t* __restrict__ A,
const scalar_t* scale, const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS, scalar_t* C, const int _WvPrGrp,
const int CuCount, const int B_row_stride_bytes) {
UNREACHABLE_CODE
}
#endif // defined(__HIP__GFX1X__)
@@ -367,9 +367,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
const uint8_t* B_packed, const scalar_t* __restrict__ A,
const scalar_t* scale, const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS, scalar_t* C,
const int _WvPrGrp, const int CuCount) {
const int _WvPrGrp, const int CuCount,
const int B_row_stride_bytes) {
constexpr int max_lds_len = LDS_SIZE / 2;
const int K_packed = K / 2;
union bigTypeA {
scalar_t h[A_CHUNK];
@@ -430,9 +430,9 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
uint32_t k_ = k + threadIdx.x * A_CHUNK;
if (k_ >= K) break;
const uint8_t* B_ = &B_packed[(m + 0) * K_packed + k_ / 2];
const uint8_t* B_ = &B_packed[(m + 0) * B_row_stride_bytes + k_ / 2];
for (int y = 0; y < YTILE; y++) {
const float* src = (const float*)(&B_[y * K_packed]);
const float* src = (const float*)(&B_[y * B_row_stride_bytes]);
#pragma unroll
for (int i = 0; i < A_CHUNK / 8; i++)
bigB[y][k2].f[i] = loadnt((float*)&src[i]);
@@ -606,14 +606,12 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
#else // !defined(__HIP__GFX1X__)
template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
int UNRL, int N, int GROUP_SIZE = 0, bool HAS_ZERO_POINTS = false>
__global__ void wvSplitK_int4_hf_(const int K, const int M, const int Bx,
const int By, const uint8_t* B_packed,
const scalar_t* __restrict__ A,
const scalar_t* scale,
const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS,
scalar_t* C, const int _WvPrGrp,
const int CuCount) {
__global__ void wvSplitK_int4_hf_(
const int K, const int M, const int Bx, const int By,
const uint8_t* B_packed, const scalar_t* __restrict__ A,
const scalar_t* scale, const uint32_t* zero_points,
const scalar_t* __restrict__ BIAS, scalar_t* C, const int _WvPrGrp,
const int CuCount, const int B_row_stride_bytes) {
UNREACHABLE_CODE
}
#endif // defined(__HIP__GFX1X__)
@@ -658,10 +656,17 @@ torch::Tensor wvSplitK_int4_g(const at::Tensor& in_a, const at::Tensor& in_b,
? in_bias->size(0)
: 1;
int64_t expected_weight_bytes = M_in * K_in / 2;
int64_t actual_weight_bytes = in_a.numel() * in_a.element_size();
const int64_t expected_weight_bytes = M_in * K_in / 2;
const int64_t actual_weight_bytes = in_a.numel() * in_a.element_size();
TORCH_CHECK(actual_weight_bytes == expected_weight_bytes,
"Weight tensor must contain M*K/2 bytes for int4 packing");
const int64_t b_row_stride_bytes = in_a.stride(0) * in_a.element_size();
TORCH_CHECK(b_row_stride_bytes >= K_in / 2, "Weight row stride (",
b_row_stride_bytes, " bytes) must hold at least K/2=", K_in / 2,
" bytes per row");
TORCH_CHECK(std::in_range<int>(b_row_stride_bytes), "Weight row stride (",
b_row_stride_bytes, " bytes) exceeds int range");
const int b_row_stride_bytes_i32 = static_cast<int>(b_row_stride_bytes);
TORCH_CHECK(
in_b.dtype() == torch::kFloat16 || in_b.dtype() == torch::kBFloat16,
"Activation must be float16 or bfloat16");
@@ -722,12 +727,12 @@ torch::Tensor wvSplitK_int4_g(const at::Tensor& in_a, const at::Tensor& in_b,
wvSplitK_int4_hf_sml_<fptype, _THRDS, _YTILE, 16, 16, _UNRL, _N, _GS, \
_HAS_ZP><<<grid, block, 0, stream>>>( \
K_in, M_in, Bx_in, By_in, wptr, aptr, sptr, zpptr, biasptr, cptr, \
__wvPrGrp, CuCount); \
__wvPrGrp, CuCount, b_row_stride_bytes_i32); \
else \
wvSplitK_int4_hf_<fptype, _THRDS, _YTILE, 16, 16, _UNRL, _N, _GS, \
_HAS_ZP><<<grid, block, 0, stream>>>( \
K_in, M_in, Bx_in, By_in, wptr, aptr, sptr, zpptr, biasptr, cptr, \
__wvPrGrp, CuCount); \
__wvPrGrp, CuCount, b_row_stride_bytes_i32); \
}
#define WVSPLITK_INT4G(_YTILE, _UNRL, _N, _GS, _HAS_ZP) \