From 638b1a99ccd7b47f45f9a773456f77aeb66d4a1b Mon Sep 17 00:00:00 2001 From: wcy <86111164+wcynb1023@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:18:10 +0800 Subject: [PATCH] [CPU][RISC-V] Add RVV path for W4A8 INT4 GEMM (#45269) Signed-off-by: wcy <233313160abc@gmail.com> Co-authored-by: lyd1992 Co-authored-by: OpenAI Codex --- cmake/cpu_extension.cmake | 7 +- csrc/cpu/cpu_types_riscv_defs.hpp | 21 ++- csrc/cpu/sgl-kernels/gemm_int4.cpp | 124 ++++++++++++++++++ csrc/cpu/sgl-kernels/vec.h | 8 ++ .../kernels/linear/mixed_precision/cpu.py | 4 +- 5 files changed, 157 insertions(+), 7 deletions(-) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 386f9e30c7..5c19446601 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -166,12 +166,13 @@ elseif (S390_FOUND) "-mtune=native") elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") message(STATUS "RISC-V detected") - if(DEFINED VLLM_RVV_VLEN AND NOT VLLM_RVV_VLEN GREATER 0) + if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0) message(FATAL_ERROR - "VLLM_RVV_VLEN must be a positive integer; got '${VLLM_RVV_VLEN}'") + "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'") endif() # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo - # by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256. + # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build. + # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV. if(NOT DEFINED VLLM_RVV_VLEN) # Auto-detect: find the largest zvlb in /proc/cpuinfo isa line. if(EXISTS /proc/cpuinfo) diff --git a/csrc/cpu/cpu_types_riscv_defs.hpp b/csrc/cpu/cpu_types_riscv_defs.hpp index 650dc5bcc7..16475505d9 100644 --- a/csrc/cpu/cpu_types_riscv_defs.hpp +++ b/csrc/cpu/cpu_types_riscv_defs.hpp @@ -3,13 +3,17 @@ // VLEN-to-LMUL mapping for RISC-V Vector extension. // -// LMUL_ expands to the LMUL suffix giving N total bits of vector data: -// VLEN=128: LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8 -// VLEN=256: LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4 +// LMUL_ expands to the LMUL suffix giving N total bits of vector data. +// LMUL_64 is used by 8-lane int8/uint8 vectors. +// VLEN=128: +// LMUL_64=mf2, LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8 +// VLEN=256: +// LMUL_64=mf4, LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4 #include #if __riscv_v_min_vlen == 128 + #define LMUL_64 mf2 #define LMUL_128 m1 #define LMUL_256 m2 #define LMUL_512 m4 @@ -17,6 +21,7 @@ #define BOOL_256 b16 #define BOOL_512 b8 #elif __riscv_v_min_vlen == 256 + #define LMUL_64 mf4 #define LMUL_128 mf2 #define LMUL_256 m1 #define LMUL_512 m2 @@ -41,6 +46,16 @@ // ---- Semantic fixed-vector typedefs (named by element count) ---- +// uint8 / int8 +typedef RVVTYPE(vuint8, LMUL_64, _t) fixed_u8x8_t + __attribute__((riscv_rvv_vector_bits(64))); +typedef RVVTYPE(vint8, LMUL_64, _t) fixed_i8x8_t + __attribute__((riscv_rvv_vector_bits(64))); + +// int16 +typedef RVVTYPE(vint16, LMUL_128, _t) fixed_i16x8_t + __attribute__((riscv_rvv_vector_bits(128))); + // float16 typedef RVVTYPE(vfloat16, LMUL_128, _t) fixed_fp16x8_t __attribute__((riscv_rvv_vector_bits(128))); diff --git a/csrc/cpu/sgl-kernels/gemm_int4.cpp b/csrc/cpu/sgl-kernels/gemm_int4.cpp index 1fec14c956..6dbd09080d 100644 --- a/csrc/cpu/sgl-kernels/gemm_int4.cpp +++ b/csrc/cpu/sgl-kernels/gemm_int4.cpp @@ -285,6 +285,125 @@ inline int32_t load_uint4_vnni(const uint8_t* __restrict__ B, int64_t k, int64_t return (n_group % 2 == 0) ? (packed & 0x0f) : ((packed >> 4) & 0x0f); } +#if defined(CPU_CAPABILITY_RVV) +template +inline fixed_i8x8_t load_uint4_as_int8_rvv(const uint8_t* __restrict__ B, int64_t k) { + constexpr int64_t n_group_size = 8; + constexpr int64_t vnni_size = 4; + static_assert(N == 32); + static_assert(ldb == N / 2); + static_assert(group >= 0 && group < N / n_group_size); + + // Unpack: gather 8 packed int4 values from the VNNI4 layout. + const int64_t ki = k % vnni_size; + const int64_t k_base = k - ki; + constexpr int64_t packed_group = group / 2; + const uint8_t* packed_ptr = B + k_base * ldb + packed_group * n_group_size * vnni_size + ki; + + fixed_u8x8_t packed = RVVI(__riscv_vlse8_v_u8, LMUL_64)(packed_ptr, vnni_size, n_group_size); + if constexpr (group % 2 == 1) { + packed = RVVI(__riscv_vsrl_vx_u8, LMUL_64)(packed, 4, n_group_size); + } + fixed_u8x8_t nibbles = RVVI(__riscv_vand_vx_u8, LMUL_64)(packed, 0x0f, n_group_size); + return RVVI4(__riscv_vreinterpret_v_u8, LMUL_64, _i8, LMUL_64)(nibbles); +} + +inline fixed_i32x8_t gemm_accum_uint8_int8_rvv(fixed_i32x8_t acc, uint8_t a, fixed_i8x8_t b) { + constexpr int64_t vl = 8; + fixed_i16x8_t b_i16 = RVVI(__riscv_vsext_vf2_i16, LMUL_128)(b, vl); + return RVVI(__riscv_vwmacc_vx_i32, LMUL_256)(acc, static_cast(a), b_i16, vl); +} + +template +inline fixed_i32x8_t gemm_accum_uint4_rvv( + fixed_i32x8_t acc, + const uint8_t* __restrict__ B, + const int8_t* __restrict__ qzeros_b, + uint8_t a, + int64_t k) { + constexpr int64_t n_group_size = 8; + fixed_i8x8_t b = load_uint4_as_int8_rvv(B, k); + fixed_i8x8_t qzeros = + RVVI(__riscv_vle8_v_i8, LMUL_64)(qzeros_b + group * n_group_size, n_group_size); + b = RVVI(__riscv_vsub_vv_i8, LMUL_64)(b, qzeros, n_group_size); + return gemm_accum_uint8_int8_rvv(acc, a, b); +} + +template +inline void _dequant_and_store_rvv( + float* __restrict__ C, + fixed_i32x8_t acc, + const float* __restrict__ scales_a, + const int32_t* __restrict__ qzeros_a, + const float* __restrict__ scales_b, + const int32_t* __restrict__ compensation, + int64_t m, + int64_t ldc) { + constexpr int64_t n_group_size = 8; + constexpr int64_t n = group * n_group_size; + constexpr int64_t vl = n_group_size; + + // Dequant compensation: remove activation zero-point contribution. + fixed_i32x8_t comp = RVVI(__riscv_vle32_v_i32, LMUL_256)(compensation + n, vl); + fixed_i32x8_t zp_comp = RVVI(__riscv_vmul_vx_i32, LMUL_256)(comp, qzeros_a[m], vl); + acc = RVVI(__riscv_vsub_vv_i32, LMUL_256)(acc, zp_comp, vl); + + // Scale: convert int32 accumulators to fp32 and apply activation/weight scales. + fixed_fp32x8_t acc_f = RVVI(__riscv_vfcvt_f_x_v_f32, LMUL_256)(acc, vl); + acc_f = RVVI(__riscv_vfmul_vf_f32, LMUL_256)(acc_f, scales_a[m], vl); + fixed_fp32x8_t scale_b = RVVI(__riscv_vle32_v_f32, LMUL_256)(scales_b + n, vl); + acc_f = RVVI(__riscv_vfmul_vv_f32, LMUL_256)(acc_f, scale_b, vl); + + // Store: accumulate into the float scratch buffer that already holds bias/zero. + float* c_ptr = C + m * ldc + n; + fixed_fp32x8_t c_old = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr, vl); + fixed_fp32x8_t c_new = RVVI(__riscv_vfadd_vv_f32, LMUL_256)(c_old, acc_f, vl); + RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr, c_new, vl); +} + +template +void _dequant_gemm_accum_rvv( + float* __restrict__ C, + const uint8_t* __restrict__ A, + const float* __restrict__ scales_a, + const int32_t* __restrict__ qzeros_a, + const uint8_t* __restrict__ B, + const float* __restrict__ scales_b, + const int8_t* __restrict__ qzeros_b, + const int32_t* __restrict__ compensation, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc) { + static_assert(N == 32); + static_assert(ldb == N / 2); + constexpr int64_t vl = 8; + + // Accumulate one C row over the 32-column block. + for (int64_t m = 0; m < M; ++m) { + fixed_i32x8_t acc0 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl); + fixed_i32x8_t acc1 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl); + fixed_i32x8_t acc2 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl); + fixed_i32x8_t acc3 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl); + // A[m][k] @ B[k][0:32] -> acc[m][0:32] + for (int64_t k = 0; k < K; ++k) { + // GEMM K step: one scalar activation updates four 8-column RVV tiles. + const uint8_t a = A[m * lda + k]; + acc0 = gemm_accum_uint4_rvv(acc0, B, qzeros_b, a, k); + acc1 = gemm_accum_uint4_rvv(acc1, B, qzeros_b, a, k); + acc2 = gemm_accum_uint4_rvv(acc2, B, qzeros_b, a, k); + acc3 = gemm_accum_uint4_rvv(acc3, B, qzeros_b, a, k); + } + + // Dequant/scale/store each 8-column group back into C. + _dequant_and_store_rvv<0>(C, acc0, scales_a, qzeros_a, scales_b, compensation, m, ldc); + _dequant_and_store_rvv<1>(C, acc1, scales_a, qzeros_a, scales_b, compensation, m, ldc); + _dequant_and_store_rvv<2>(C, acc2, scales_a, qzeros_a, scales_b, compensation, m, ldc); + _dequant_and_store_rvv<3>(C, acc3, scales_a, qzeros_a, scales_b, compensation, m, ldc); + } +} +#endif + template void _dequant_gemm_accum( float* C, @@ -336,6 +455,11 @@ void _dequant_gemm_accum( _dequant_and_store( C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/); } else +#elif defined(CPU_CAPABILITY_RVV) + if constexpr (!sym_quant_act && N == BLOCK_N && ldb == BLOCK_N / 2) { + _dequant_gemm_accum_rvv(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, M, K, lda, ldc); + return; + } else #endif { for (int64_t m = 0; m < M; ++m) { diff --git a/csrc/cpu/sgl-kernels/vec.h b/csrc/cpu/sgl-kernels/vec.h index 72143fedc6..407cfe6043 100644 --- a/csrc/cpu/sgl-kernels/vec.h +++ b/csrc/cpu/sgl-kernels/vec.h @@ -9,11 +9,19 @@ #define CPU_CAPABILITY_AVX512 #endif +#if defined(__riscv_v_min_vlen) && (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256) +#define CPU_CAPABILITY_RVV +#endif + #include #include #if defined(CPU_CAPABILITY_AVX512) #include #endif + +#if defined(CPU_CAPABILITY_RVV) +#include "../cpu_types_riscv_defs.hpp" +#endif namespace { using namespace at::vec; diff --git a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py index 1301201506..c2627668cf 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py @@ -168,11 +168,13 @@ class CPUWNA16LinearKernel(MPLinearKernel): if zp.output_dim == 0: zp.data = zp.t().contiguous() + supports_amx = torch.cpu._is_amx_tile_supported() + supports_riscv = current_platform.get_cpu_architecture() == CpuArchEnum.RISCV layer.use_w4a8 = ( envs.VLLM_CPU_INT4_W4A8 and not self.config.has_g_idx and self.config.act_type == torch.bfloat16 - and torch.cpu._is_amx_tile_supported() + and (supports_amx or supports_riscv) ) # layer.use_w4a8 = False # AWQ format will be converted to GPTQ format in `AutoAWQMarlinLinearMethod`