[CPU][RISC-V] Add RVV path for W4A8 INT4 GEMM (#45269)

Signed-off-by: wcy <233313160abc@gmail.com>
Co-authored-by: lyd1992 <liuyudong@iscas.ac.cn>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
wcy
2026-06-25 08:18:10 +00:00
committed by GitHub
co-authored by lyd1992 OpenAI Codex
parent 72adb20a6a
commit 638b1a99cc
5 changed files with 157 additions and 7 deletions
+124
View File
@@ -285,6 +285,125 @@ inline int32_t load_uint4_vnni(const uint8_t* __restrict__ B, int64_t k, int64_t
return (n_group % 2 == 0) ? (packed & 0x0f) : ((packed >> 4) & 0x0f);
}
#if defined(CPU_CAPABILITY_RVV)
template <int64_t N, int64_t ldb, int group>
inline fixed_i8x8_t load_uint4_as_int8_rvv(const uint8_t* __restrict__ B, int64_t k) {
constexpr int64_t n_group_size = 8;
constexpr int64_t vnni_size = 4;
static_assert(N == 32);
static_assert(ldb == N / 2);
static_assert(group >= 0 && group < N / n_group_size);
// Unpack: gather 8 packed int4 values from the VNNI4 layout.
const int64_t ki = k % vnni_size;
const int64_t k_base = k - ki;
constexpr int64_t packed_group = group / 2;
const uint8_t* packed_ptr = B + k_base * ldb + packed_group * n_group_size * vnni_size + ki;
fixed_u8x8_t packed = RVVI(__riscv_vlse8_v_u8, LMUL_64)(packed_ptr, vnni_size, n_group_size);
if constexpr (group % 2 == 1) {
packed = RVVI(__riscv_vsrl_vx_u8, LMUL_64)(packed, 4, n_group_size);
}
fixed_u8x8_t nibbles = RVVI(__riscv_vand_vx_u8, LMUL_64)(packed, 0x0f, n_group_size);
return RVVI4(__riscv_vreinterpret_v_u8, LMUL_64, _i8, LMUL_64)(nibbles);
}
inline fixed_i32x8_t gemm_accum_uint8_int8_rvv(fixed_i32x8_t acc, uint8_t a, fixed_i8x8_t b) {
constexpr int64_t vl = 8;
fixed_i16x8_t b_i16 = RVVI(__riscv_vsext_vf2_i16, LMUL_128)(b, vl);
return RVVI(__riscv_vwmacc_vx_i32, LMUL_256)(acc, static_cast<int16_t>(a), b_i16, vl);
}
template <int64_t N, int64_t ldb, int group>
inline fixed_i32x8_t gemm_accum_uint4_rvv(
fixed_i32x8_t acc,
const uint8_t* __restrict__ B,
const int8_t* __restrict__ qzeros_b,
uint8_t a,
int64_t k) {
constexpr int64_t n_group_size = 8;
fixed_i8x8_t b = load_uint4_as_int8_rvv<N, ldb, group>(B, k);
fixed_i8x8_t qzeros =
RVVI(__riscv_vle8_v_i8, LMUL_64)(qzeros_b + group * n_group_size, n_group_size);
b = RVVI(__riscv_vsub_vv_i8, LMUL_64)(b, qzeros, n_group_size);
return gemm_accum_uint8_int8_rvv(acc, a, b);
}
template <int group>
inline void _dequant_and_store_rvv(
float* __restrict__ C,
fixed_i32x8_t acc,
const float* __restrict__ scales_a,
const int32_t* __restrict__ qzeros_a,
const float* __restrict__ scales_b,
const int32_t* __restrict__ compensation,
int64_t m,
int64_t ldc) {
constexpr int64_t n_group_size = 8;
constexpr int64_t n = group * n_group_size;
constexpr int64_t vl = n_group_size;
// Dequant compensation: remove activation zero-point contribution.
fixed_i32x8_t comp = RVVI(__riscv_vle32_v_i32, LMUL_256)(compensation + n, vl);
fixed_i32x8_t zp_comp = RVVI(__riscv_vmul_vx_i32, LMUL_256)(comp, qzeros_a[m], vl);
acc = RVVI(__riscv_vsub_vv_i32, LMUL_256)(acc, zp_comp, vl);
// Scale: convert int32 accumulators to fp32 and apply activation/weight scales.
fixed_fp32x8_t acc_f = RVVI(__riscv_vfcvt_f_x_v_f32, LMUL_256)(acc, vl);
acc_f = RVVI(__riscv_vfmul_vf_f32, LMUL_256)(acc_f, scales_a[m], vl);
fixed_fp32x8_t scale_b = RVVI(__riscv_vle32_v_f32, LMUL_256)(scales_b + n, vl);
acc_f = RVVI(__riscv_vfmul_vv_f32, LMUL_256)(acc_f, scale_b, vl);
// Store: accumulate into the float scratch buffer that already holds bias/zero.
float* c_ptr = C + m * ldc + n;
fixed_fp32x8_t c_old = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr, vl);
fixed_fp32x8_t c_new = RVVI(__riscv_vfadd_vv_f32, LMUL_256)(c_old, acc_f, vl);
RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr, c_new, vl);
}
template <int64_t N, int64_t ldb>
void _dequant_gemm_accum_rvv(
float* __restrict__ C,
const uint8_t* __restrict__ A,
const float* __restrict__ scales_a,
const int32_t* __restrict__ qzeros_a,
const uint8_t* __restrict__ B,
const float* __restrict__ scales_b,
const int8_t* __restrict__ qzeros_b,
const int32_t* __restrict__ compensation,
int64_t M,
int64_t K,
int64_t lda,
int64_t ldc) {
static_assert(N == 32);
static_assert(ldb == N / 2);
constexpr int64_t vl = 8;
// Accumulate one C row over the 32-column block.
for (int64_t m = 0; m < M; ++m) {
fixed_i32x8_t acc0 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
fixed_i32x8_t acc1 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
fixed_i32x8_t acc2 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
fixed_i32x8_t acc3 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
// A[m][k] @ B[k][0:32] -> acc[m][0:32]
for (int64_t k = 0; k < K; ++k) {
// GEMM K step: one scalar activation updates four 8-column RVV tiles.
const uint8_t a = A[m * lda + k];
acc0 = gemm_accum_uint4_rvv<N, ldb, 0>(acc0, B, qzeros_b, a, k);
acc1 = gemm_accum_uint4_rvv<N, ldb, 1>(acc1, B, qzeros_b, a, k);
acc2 = gemm_accum_uint4_rvv<N, ldb, 2>(acc2, B, qzeros_b, a, k);
acc3 = gemm_accum_uint4_rvv<N, ldb, 3>(acc3, B, qzeros_b, a, k);
}
// Dequant/scale/store each 8-column group back into C.
_dequant_and_store_rvv<0>(C, acc0, scales_a, qzeros_a, scales_b, compensation, m, ldc);
_dequant_and_store_rvv<1>(C, acc1, scales_a, qzeros_a, scales_b, compensation, m, ldc);
_dequant_and_store_rvv<2>(C, acc2, scales_a, qzeros_a, scales_b, compensation, m, ldc);
_dequant_and_store_rvv<3>(C, acc3, scales_a, qzeros_a, scales_b, compensation, m, ldc);
}
}
#endif
template <int64_t N, int64_t ldb, bool sym_quant_act>
void _dequant_gemm_accum(
float* C,
@@ -336,6 +455,11 @@ void _dequant_gemm_accum(
_dequant_and_store<true, N, sym_quant_act>(
C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/);
} else
#elif defined(CPU_CAPABILITY_RVV)
if constexpr (!sym_quant_act && N == BLOCK_N && ldb == BLOCK_N / 2) {
_dequant_gemm_accum_rvv<N, ldb>(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, M, K, lda, ldc);
return;
} else
#endif
{
for (int64_t m = 0; m < M; ++m) {