[CPU][RISC-V] Add RVV path for W4A8 INT4 GEMM (#45269)
Signed-off-by: wcy <233313160abc@gmail.com> Co-authored-by: lyd1992 <liuyudong@iscas.ac.cn> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
lyd1992
OpenAI Codex
parent
72adb20a6a
commit
638b1a99cc
@@ -166,12 +166,13 @@ elseif (S390_FOUND)
|
||||
"-mtune=native")
|
||||
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
|
||||
message(STATUS "RISC-V detected")
|
||||
if(DEFINED VLLM_RVV_VLEN AND NOT VLLM_RVV_VLEN GREATER 0)
|
||||
if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0)
|
||||
message(FATAL_ERROR
|
||||
"VLLM_RVV_VLEN must be a positive integer; got '${VLLM_RVV_VLEN}'")
|
||||
"VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'")
|
||||
endif()
|
||||
# VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo
|
||||
# by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256.
|
||||
# by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build.
|
||||
# Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV.
|
||||
if(NOT DEFINED VLLM_RVV_VLEN)
|
||||
# Auto-detect: find the largest zvl<N>b in /proc/cpuinfo isa line.
|
||||
if(EXISTS /proc/cpuinfo)
|
||||
|
||||
@@ -3,13 +3,17 @@
|
||||
|
||||
// VLEN-to-LMUL mapping for RISC-V Vector extension.
|
||||
//
|
||||
// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data:
|
||||
// VLEN=128: LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8
|
||||
// VLEN=256: LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4
|
||||
// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data.
|
||||
// LMUL_64 is used by 8-lane int8/uint8 vectors.
|
||||
// VLEN=128:
|
||||
// LMUL_64=mf2, LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8
|
||||
// VLEN=256:
|
||||
// LMUL_64=mf4, LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4
|
||||
|
||||
#include <riscv_vector.h>
|
||||
|
||||
#if __riscv_v_min_vlen == 128
|
||||
#define LMUL_64 mf2
|
||||
#define LMUL_128 m1
|
||||
#define LMUL_256 m2
|
||||
#define LMUL_512 m4
|
||||
@@ -17,6 +21,7 @@
|
||||
#define BOOL_256 b16
|
||||
#define BOOL_512 b8
|
||||
#elif __riscv_v_min_vlen == 256
|
||||
#define LMUL_64 mf4
|
||||
#define LMUL_128 mf2
|
||||
#define LMUL_256 m1
|
||||
#define LMUL_512 m2
|
||||
@@ -41,6 +46,16 @@
|
||||
|
||||
// ---- Semantic fixed-vector typedefs (named by element count) ----
|
||||
|
||||
// uint8 / int8
|
||||
typedef RVVTYPE(vuint8, LMUL_64, _t) fixed_u8x8_t
|
||||
__attribute__((riscv_rvv_vector_bits(64)));
|
||||
typedef RVVTYPE(vint8, LMUL_64, _t) fixed_i8x8_t
|
||||
__attribute__((riscv_rvv_vector_bits(64)));
|
||||
|
||||
// int16
|
||||
typedef RVVTYPE(vint16, LMUL_128, _t) fixed_i16x8_t
|
||||
__attribute__((riscv_rvv_vector_bits(128)));
|
||||
|
||||
// float16
|
||||
typedef RVVTYPE(vfloat16, LMUL_128, _t) fixed_fp16x8_t
|
||||
__attribute__((riscv_rvv_vector_bits(128)));
|
||||
|
||||
@@ -285,6 +285,125 @@ inline int32_t load_uint4_vnni(const uint8_t* __restrict__ B, int64_t k, int64_t
|
||||
return (n_group % 2 == 0) ? (packed & 0x0f) : ((packed >> 4) & 0x0f);
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_RVV)
|
||||
template <int64_t N, int64_t ldb, int group>
|
||||
inline fixed_i8x8_t load_uint4_as_int8_rvv(const uint8_t* __restrict__ B, int64_t k) {
|
||||
constexpr int64_t n_group_size = 8;
|
||||
constexpr int64_t vnni_size = 4;
|
||||
static_assert(N == 32);
|
||||
static_assert(ldb == N / 2);
|
||||
static_assert(group >= 0 && group < N / n_group_size);
|
||||
|
||||
// Unpack: gather 8 packed int4 values from the VNNI4 layout.
|
||||
const int64_t ki = k % vnni_size;
|
||||
const int64_t k_base = k - ki;
|
||||
constexpr int64_t packed_group = group / 2;
|
||||
const uint8_t* packed_ptr = B + k_base * ldb + packed_group * n_group_size * vnni_size + ki;
|
||||
|
||||
fixed_u8x8_t packed = RVVI(__riscv_vlse8_v_u8, LMUL_64)(packed_ptr, vnni_size, n_group_size);
|
||||
if constexpr (group % 2 == 1) {
|
||||
packed = RVVI(__riscv_vsrl_vx_u8, LMUL_64)(packed, 4, n_group_size);
|
||||
}
|
||||
fixed_u8x8_t nibbles = RVVI(__riscv_vand_vx_u8, LMUL_64)(packed, 0x0f, n_group_size);
|
||||
return RVVI4(__riscv_vreinterpret_v_u8, LMUL_64, _i8, LMUL_64)(nibbles);
|
||||
}
|
||||
|
||||
inline fixed_i32x8_t gemm_accum_uint8_int8_rvv(fixed_i32x8_t acc, uint8_t a, fixed_i8x8_t b) {
|
||||
constexpr int64_t vl = 8;
|
||||
fixed_i16x8_t b_i16 = RVVI(__riscv_vsext_vf2_i16, LMUL_128)(b, vl);
|
||||
return RVVI(__riscv_vwmacc_vx_i32, LMUL_256)(acc, static_cast<int16_t>(a), b_i16, vl);
|
||||
}
|
||||
|
||||
template <int64_t N, int64_t ldb, int group>
|
||||
inline fixed_i32x8_t gemm_accum_uint4_rvv(
|
||||
fixed_i32x8_t acc,
|
||||
const uint8_t* __restrict__ B,
|
||||
const int8_t* __restrict__ qzeros_b,
|
||||
uint8_t a,
|
||||
int64_t k) {
|
||||
constexpr int64_t n_group_size = 8;
|
||||
fixed_i8x8_t b = load_uint4_as_int8_rvv<N, ldb, group>(B, k);
|
||||
fixed_i8x8_t qzeros =
|
||||
RVVI(__riscv_vle8_v_i8, LMUL_64)(qzeros_b + group * n_group_size, n_group_size);
|
||||
b = RVVI(__riscv_vsub_vv_i8, LMUL_64)(b, qzeros, n_group_size);
|
||||
return gemm_accum_uint8_int8_rvv(acc, a, b);
|
||||
}
|
||||
|
||||
template <int group>
|
||||
inline void _dequant_and_store_rvv(
|
||||
float* __restrict__ C,
|
||||
fixed_i32x8_t acc,
|
||||
const float* __restrict__ scales_a,
|
||||
const int32_t* __restrict__ qzeros_a,
|
||||
const float* __restrict__ scales_b,
|
||||
const int32_t* __restrict__ compensation,
|
||||
int64_t m,
|
||||
int64_t ldc) {
|
||||
constexpr int64_t n_group_size = 8;
|
||||
constexpr int64_t n = group * n_group_size;
|
||||
constexpr int64_t vl = n_group_size;
|
||||
|
||||
// Dequant compensation: remove activation zero-point contribution.
|
||||
fixed_i32x8_t comp = RVVI(__riscv_vle32_v_i32, LMUL_256)(compensation + n, vl);
|
||||
fixed_i32x8_t zp_comp = RVVI(__riscv_vmul_vx_i32, LMUL_256)(comp, qzeros_a[m], vl);
|
||||
acc = RVVI(__riscv_vsub_vv_i32, LMUL_256)(acc, zp_comp, vl);
|
||||
|
||||
// Scale: convert int32 accumulators to fp32 and apply activation/weight scales.
|
||||
fixed_fp32x8_t acc_f = RVVI(__riscv_vfcvt_f_x_v_f32, LMUL_256)(acc, vl);
|
||||
acc_f = RVVI(__riscv_vfmul_vf_f32, LMUL_256)(acc_f, scales_a[m], vl);
|
||||
fixed_fp32x8_t scale_b = RVVI(__riscv_vle32_v_f32, LMUL_256)(scales_b + n, vl);
|
||||
acc_f = RVVI(__riscv_vfmul_vv_f32, LMUL_256)(acc_f, scale_b, vl);
|
||||
|
||||
// Store: accumulate into the float scratch buffer that already holds bias/zero.
|
||||
float* c_ptr = C + m * ldc + n;
|
||||
fixed_fp32x8_t c_old = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr, vl);
|
||||
fixed_fp32x8_t c_new = RVVI(__riscv_vfadd_vv_f32, LMUL_256)(c_old, acc_f, vl);
|
||||
RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr, c_new, vl);
|
||||
}
|
||||
|
||||
template <int64_t N, int64_t ldb>
|
||||
void _dequant_gemm_accum_rvv(
|
||||
float* __restrict__ C,
|
||||
const uint8_t* __restrict__ A,
|
||||
const float* __restrict__ scales_a,
|
||||
const int32_t* __restrict__ qzeros_a,
|
||||
const uint8_t* __restrict__ B,
|
||||
const float* __restrict__ scales_b,
|
||||
const int8_t* __restrict__ qzeros_b,
|
||||
const int32_t* __restrict__ compensation,
|
||||
int64_t M,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldc) {
|
||||
static_assert(N == 32);
|
||||
static_assert(ldb == N / 2);
|
||||
constexpr int64_t vl = 8;
|
||||
|
||||
// Accumulate one C row over the 32-column block.
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
fixed_i32x8_t acc0 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
|
||||
fixed_i32x8_t acc1 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
|
||||
fixed_i32x8_t acc2 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
|
||||
fixed_i32x8_t acc3 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
|
||||
// A[m][k] @ B[k][0:32] -> acc[m][0:32]
|
||||
for (int64_t k = 0; k < K; ++k) {
|
||||
// GEMM K step: one scalar activation updates four 8-column RVV tiles.
|
||||
const uint8_t a = A[m * lda + k];
|
||||
acc0 = gemm_accum_uint4_rvv<N, ldb, 0>(acc0, B, qzeros_b, a, k);
|
||||
acc1 = gemm_accum_uint4_rvv<N, ldb, 1>(acc1, B, qzeros_b, a, k);
|
||||
acc2 = gemm_accum_uint4_rvv<N, ldb, 2>(acc2, B, qzeros_b, a, k);
|
||||
acc3 = gemm_accum_uint4_rvv<N, ldb, 3>(acc3, B, qzeros_b, a, k);
|
||||
}
|
||||
|
||||
// Dequant/scale/store each 8-column group back into C.
|
||||
_dequant_and_store_rvv<0>(C, acc0, scales_a, qzeros_a, scales_b, compensation, m, ldc);
|
||||
_dequant_and_store_rvv<1>(C, acc1, scales_a, qzeros_a, scales_b, compensation, m, ldc);
|
||||
_dequant_and_store_rvv<2>(C, acc2, scales_a, qzeros_a, scales_b, compensation, m, ldc);
|
||||
_dequant_and_store_rvv<3>(C, acc3, scales_a, qzeros_a, scales_b, compensation, m, ldc);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
template <int64_t N, int64_t ldb, bool sym_quant_act>
|
||||
void _dequant_gemm_accum(
|
||||
float* C,
|
||||
@@ -336,6 +455,11 @@ void _dequant_gemm_accum(
|
||||
_dequant_and_store<true, N, sym_quant_act>(
|
||||
C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/);
|
||||
} else
|
||||
#elif defined(CPU_CAPABILITY_RVV)
|
||||
if constexpr (!sym_quant_act && N == BLOCK_N && ldb == BLOCK_N / 2) {
|
||||
_dequant_gemm_accum_rvv<N, ldb>(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, M, K, lda, ldc);
|
||||
return;
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
|
||||
@@ -9,11 +9,19 @@
|
||||
#define CPU_CAPABILITY_AVX512
|
||||
#endif
|
||||
|
||||
#if defined(__riscv_v_min_vlen) && (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256)
|
||||
#define CPU_CAPABILITY_RVV
|
||||
#endif
|
||||
|
||||
#include <ATen/cpu/vec/functional.h>
|
||||
#include <ATen/cpu/vec/vec.h>
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
#if defined(CPU_CAPABILITY_RVV)
|
||||
#include "../cpu_types_riscv_defs.hpp"
|
||||
#endif
|
||||
namespace {
|
||||
|
||||
using namespace at::vec;
|
||||
|
||||
@@ -168,11 +168,13 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
if zp.output_dim == 0:
|
||||
zp.data = zp.t().contiguous()
|
||||
|
||||
supports_amx = torch.cpu._is_amx_tile_supported()
|
||||
supports_riscv = current_platform.get_cpu_architecture() == CpuArchEnum.RISCV
|
||||
layer.use_w4a8 = (
|
||||
envs.VLLM_CPU_INT4_W4A8
|
||||
and not self.config.has_g_idx
|
||||
and self.config.act_type == torch.bfloat16
|
||||
and torch.cpu._is_amx_tile_supported()
|
||||
and (supports_amx or supports_riscv)
|
||||
)
|
||||
# layer.use_w4a8 = False
|
||||
# AWQ format will be converted to GPTQ format in `AutoAWQMarlinLinearMethod`
|
||||
|
||||
Reference in New Issue
Block a user