diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index ffab4015f4..c51384e319 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -396,6 +396,13 @@ set(VLLM_EXT_SRC "csrc/cpu/cpu_attn.cpp" "csrc/cpu/torch_bindings.cpp") +if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND + VLLM_RVV_VLEN GREATER 0 AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) + set(VLLM_EXT_SRC + "csrc/cpu/cpu_wna16.cpp" + ${VLLM_EXT_SRC}) +endif() + if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) set(VLLM_EXT_SRC "csrc/cpu/shm.cpp" diff --git a/csrc/cpu/cpu_types_riscv_impl.hpp b/csrc/cpu/cpu_types_riscv_impl.hpp index d6cae76c45..06a38c780a 100644 --- a/csrc/cpu/cpu_types_riscv_impl.hpp +++ b/csrc/cpu/cpu_types_riscv_impl.hpp @@ -94,6 +94,10 @@ struct FP16Vec16 : public Vec { : reg(RVVI(__riscv_vle16_v_f16, LMUL_256)( static_cast(ptr), VEC_ELEM_NUM)) {}; + explicit FP16Vec16(const c10::Half v) + : reg(RVVI4(__riscv_vreinterpret_v_u16, LMUL_256, _f16, LMUL_256)( + RVVI(__riscv_vmv_v_x_u16, LMUL_256)(v.x, VEC_ELEM_NUM))) {}; + explicit FP16Vec16(const FP32Vec16& vec); void save(void* ptr) const { @@ -165,6 +169,9 @@ struct BF16Vec16 : public Vec { reinterpret_cast(ptr), VEC_ELEM_NUM))) {}; explicit BF16Vec16(fixed_bf16x16_t data) : reg(data) {}; + explicit BF16Vec16(const c10::BFloat16 v) + : reg(RVVI4(__riscv_vreinterpret_v_u16, LMUL_256, _bf16, LMUL_256)( + RVVI(__riscv_vmv_v_x_u16, LMUL_256)(v.x, VEC_ELEM_NUM))) {}; explicit BF16Vec16(const FP32Vec16&); void save(void* ptr) const { @@ -290,6 +297,9 @@ struct BF16Vec16 : public Vec { } reg_fp32 = RVVI(__riscv_vle32_v_f32, LMUL_512)(tmp, 16); } + explicit BF16Vec16(const c10::BFloat16 v) + : reg_fp32(RVVI(__riscv_vfmv_v_f_f32, LMUL_512)(static_cast(v), + VEC_ELEM_NUM)) {} explicit BF16Vec16(const FP32Vec16&); void save(void* ptr) const { float tmp[16]; @@ -629,6 +639,19 @@ struct FP32Vec16 : public Vec { : reg(RVVI4(__riscv_vcreate_v_f32, LMUL_256, _f32, LMUL_512)( data.reg, data.reg)) {}; explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {}; + explicit FP32Vec16(int64_t value, const FP32Vec16& lut) { + const uint64_t q_values = static_cast(value); + auto packed = RVVI(__riscv_vmv_v_x_u64, LMUL_1024)(q_values, VEC_ELEM_NUM); + auto lane_ids = RVVI(__riscv_vid_v_u64, LMUL_1024)(VEC_ELEM_NUM); + auto shifts = + RVVI(__riscv_vsll_vx_u64, LMUL_1024)(lane_ids, 2, VEC_ELEM_NUM); + auto shifted = + RVVI(__riscv_vsrl_vv_u64, LMUL_1024)(packed, shifts, VEC_ELEM_NUM); + auto idx64 = + RVVI(__riscv_vand_vx_u64, LMUL_1024)(shifted, 0xF, VEC_ELEM_NUM); + auto idx32 = RVVI(__riscv_vnsrl_wx_u32, LMUL_512)(idx64, 0, VEC_ELEM_NUM); + reg = RVVI(__riscv_vrgather_vv_f32, LMUL_512)(lut.reg, idx32, VEC_ELEM_NUM); + } explicit FP32Vec16(const FP16Vec16& v); #ifdef __riscv_zvfbfmin @@ -641,6 +664,10 @@ struct FP32Vec16 : public Vec { explicit FP32Vec16(const BF16Vec16& v) : reg(v.reg_fp32) {}; #endif + // FP8 stub: dead code on RISC-V (fp8 KV cache is x86-only), needed for + // load_b_pair_vec template to compile on all platforms. + explicit FP32Vec16(const BF16Vec32&, int) : FP32Vec16() {} + FP32Vec16 operator+(const FP32Vec16& b) const { return FP32Vec16( RVVI(__riscv_vfadd_vv_f32, LMUL_512)(reg, b.reg, VEC_ELEM_NUM)); @@ -891,6 +918,30 @@ inline void fma(FP32Vec16& acc, const FP32Vec16& a, const FP32Vec16& b) { acc = acc.fma(a, b); } +template +static void interleave_save_16b(const VecT& vec0, const VecT& vec1, void* ptr) { + alignas(64) uint16_t values0[VecT::VEC_ELEM_NUM]; + alignas(64) uint16_t values1[VecT::VEC_ELEM_NUM]; + vec0.save(values0); + vec1.save(values1); + + auto* packed = reinterpret_cast(ptr); + for (int32_t i = 0; i < VecT::VEC_ELEM_NUM; ++i) { + packed[i] = static_cast(values0[i]) | + (static_cast(values1[i]) << 16); + } +} + +static void interleave_save(const FP16Vec16& vec0, const FP16Vec16& vec1, + void* ptr) { + interleave_save_16b(vec0, vec1, ptr); +} + +static void interleave_save(const BF16Vec16& vec0, const BF16Vec16& vec1, + void* ptr) { + interleave_save_16b(vec0, vec1, ptr); +} + #ifdef __riscv_zvfbfmin template <> inline void storeFP32(float v, c10::BFloat16* ptr) { diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 35350cf247..29f86ba0eb 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -518,7 +518,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def("dynamic_per_token_scaled_fp8_quant() -> ()", placeholder_op); // WNA16 -#if defined(__AVX512F__) +#if defined(__AVX512F__) || defined(__riscv_v) ops.def( "cpu_gemm_wna16(Tensor input, Tensor q_weight, Tensor(a2!) output, " "Tensor scales, Tensor? zeros, Tensor? g_idx, Tensor? bias, SymInt "