diff --git a/.agents/skills/transformers-upgrade/SKILL.md b/.agents/skills/transformers-upgrade/SKILL.md index 847b292cc5..959d5281f2 100644 --- a/.agents/skills/transformers-upgrade/SKILL.md +++ b/.agents/skills/transformers-upgrade/SKILL.md @@ -183,9 +183,8 @@ Add `tests/models/test_initialization.py -k ""` where a GPU is available, and a model eval for any model whose resolved config changed. Models in `vllm/models/` are high priority: whenever their config or model -code changes, run GSM8k (`tests/evals/gsm8k/`) and GPQA Diamond -(`tests/evals/gpt_oss/`) on `main` and the branch, even when the dumps show -no diff, and put both scores in the PR. FlashInfer autotuning picks kernel +code changes, run GSM8k (`tests/evals/gsm8k/`) on `main` and the branch, +even when the dumps show no diff, and put the score in the PR. FlashInfer tactics afresh on every server start, which moved DeepSeek-V4-Flash GSM8k by about 1 point; pass `--kernel-config '{"enable_flashinfer_autotune": false}'` to both servers so greedy outputs are comparable. diff --git a/CMakeLists.txt b/CMakeLists.txt index c84e2b9eed..e756127a6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,11 +168,6 @@ define_extension_target( # if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND NOT VLLM_TARGET_DEVICE STREQUAL "rocm") - if (VLLM_TARGET_DEVICE STREQUAL "cpu") - include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake) - else() - return() - endif() return() endif() @@ -415,7 +410,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/cuda_view.cu" "csrc/cuda_utils_kernels.cu" "csrc/activation_kernels.cu" - "csrc/muse_spark_mul_tanh_soft_cap_kernel.cu" "csrc/ngram_embedding_kernels.cu" "csrc/vocab_parallel_embedding_kernels.cu" "csrc/quantization/activation_kernels.cu" @@ -424,9 +418,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/quantization/w8a8/fp8/per_token_group_quant.cu" "csrc/quantization/gptq/q_gemm.cu" "csrc/pos_encoding_kernels.cu" - "csrc/fused_muse_spark_qknorm_rope_kv_insert_kernel.cu" "csrc/fused_qknorm_rope_kernel.cu" - "csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu" "csrc/layernorm_kernels.cu" "csrc/layernorm_quant_kernels.cu" "csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu" @@ -439,14 +431,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/cache_kernels_fused.cu" "csrc/custom_all_gather_reduce_scatter.cu" "csrc/custom_all_gather_reduce_scatter_ops.cpp" - "csrc/custom_all_reduce.cu" - "csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu" - "csrc/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu") - - if(VLLM_GPU_LANG STREQUAL "CUDA") - list(APPEND VLLM_STABLE_EXT_SRC - "csrc/fused_qknorm_rope_cache_kernel.cu") - endif() + "csrc/custom_all_reduce.cu") if(VLLM_GPU_LANG STREQUAL "CUDA" AND DEFINED CMAKE_CUDA_COMPILER_VERSION AND @@ -509,8 +494,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/quantization/w8a8/cutlass/scaled_mm_entry.cu" "csrc/quantization/fp4/nvfp4_quant_entry.cu" "csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu" - "csrc/quantization/awq/gemm_kernels.cu" - "csrc/minimax_reduce_rms_kernel.cu") + "csrc/quantization/awq/gemm_kernels.cu") # # Machete kernels @@ -581,30 +565,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() endif() - # The fused MiniMax-M3 KV insert quantizes NVFP4 with SM100-family-only - # cvt instructions, so its SM10x/11x code is built for the family targets. - set(MINIMAX_M3_FUSED_SRC - "csrc/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu") - list(REMOVE_ITEM VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") set_gencode_flags_for_srcs( SRCS "${VLLM_STABLE_EXT_SRC}" CUDA_ARCHS "${CUDA_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS - "10.0f;10.7f;11.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS - "10.0a;10.1a;10.3a" "${CUDA_ARCHS}") - endif() - foreach(_ARCH ${CUDA_ARCHS}) - if(_ARCH VERSION_LESS 10.0 OR _ARCH VERSION_GREATER_EQUAL 12.0) - list(APPEND MINIMAX_M3_FUSED_ARCHS "${_ARCH}") - endif() - endforeach() - set_gencode_flags_for_srcs( - SRCS "${MINIMAX_M3_FUSED_SRC}" - CUDA_ARCHS "${MINIMAX_M3_FUSED_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") if(COOPERATIVE_TOPK_ARCHS) list(APPEND VLLM_STABLE_EXT_SRC @@ -753,24 +716,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") " in CUDA target architectures") endif() - # DeepSeek V3 fused A GEMM kernel (requires SM 9.0+, Hopper and later) - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(DSV3_FUSED_A_GEMM_ARCHS "9.0a;10.0f;10.7f;11.0f;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(DSV3_FUSED_A_GEMM_ARCHS "9.0a;10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}") - endif() - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND DSV3_FUSED_A_GEMM_ARCHS) - set(DSV3_FUSED_A_GEMM_SRCS "csrc/dsv3_fused_a_gemm.cu") - set_gencode_flags_for_srcs( - SRCS "${DSV3_FUSED_A_GEMM_SRCS}" - CUDA_ARCHS "${DSV3_FUSED_A_GEMM_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${DSV3_FUSED_A_GEMM_SRCS}") - message(STATUS "Building dsv3_fused_a_gemm for archs: ${DSV3_FUSED_A_GEMM_ARCHS}") - else() - message(STATUS "Not building dsv3_fused_a_gemm as no compatible archs found " - "in CUDA target architectures.") - endif() - # FP32 router GEMM (H=3072, E=256, M<=32). Requires SM90+ and CUDA >= 12.0. cuda_archs_sm90plus(FP32_ROUTER_GEMM_ARCHS "${CUDA_ARCHS}") if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND FP32_ROUTER_GEMM_ARCHS) @@ -1150,24 +1095,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "Building fused GDN decode for archs: ${FUSED_GDN_DECODE_ARCHS}") endif() - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(KIMI_K3_ATTN_RES_ARCHS - "10.0f" "${CUDA_ARCHS}") - endif() - if(KIMI_K3_ATTN_RES_ARCHS) - set(KIMI_K3_ATTN_RES_SRC - "csrc/kimi_k3/attn_res_kernel.cu") - set_gencode_flags_for_srcs( - SRCS "${KIMI_K3_ATTN_RES_SRC}" - CUDA_ARCHS "${KIMI_K3_ATTN_RES_ARCHS}") - set_property(SOURCE ${KIMI_K3_ATTN_RES_SRC} APPEND PROPERTY - COMPILE_OPTIONS - "$<$:--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math>") - list(APPEND VLLM_STABLE_EXT_SRC "${KIMI_K3_ATTN_RES_SRC}") - message(STATUS - "Building Kimi K3 AttnRes for archs: ${KIMI_K3_ATTN_RES_ARCHS}") - endif() - # Hadacore kernels cuda_archs_loose_intersection(HADACORE_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}") if(HADACORE_ARCHS) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake deleted file mode 100644 index c4d111e77b..0000000000 --- a/cmake/cpu_extension.cmake +++ /dev/null @@ -1,698 +0,0 @@ -include(FetchContent) -include(CheckCXXCompilerFlag) - -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_EXTENSIONS ON) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(MACOSX_FOUND TRUE) -endif() - - -# -# Define environment variables for special configurations -# -set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86}) -set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16}) -set(ENABLE_ARM_I8MM $ENV{VLLM_CPU_ARM_I8MM}) -set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16}) - -include_directories("${CMAKE_SOURCE_DIR}/csrc") - -set (ENABLE_NUMA TRUE) - -# -# Check the compile flags -# -if(MACOSX_FOUND) - # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is - # needed: _C is a dynamic_lookup bundle and resolves libomp from torch. - list(APPEND CXX_COMPILE_FLAGS - "-Xpreprocessor" "-fopenmp" - "-DVLLM_CPU_EXTENSION") -else() - list(APPEND CXX_COMPILE_FLAGS - "-fopenmp" - "-DVLLM_CPU_EXTENSION") - - # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0) - # and create a local shim dir with it. When PyTorch is built from source or packaged - # by a distro (common on RISC-V, s390x, Fedora/RHEL aarch64), no vendored libgomp - # exists and the shim dir is empty; fall back to the system libgomp in that case. - vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR) - - if(VLLM_TORCH_GOMP_SHIM_DIR) - find_library(OPEN_MP - NAMES gomp - PATHS "${VLLM_TORCH_GOMP_SHIM_DIR}" - NO_DEFAULT_PATH - REQUIRED - ) - # Use the same libgomp as PyTorch at runtime - set(ENV{LD_LIBRARY_PATH} "${VLLM_TORCH_GOMP_SHIM_DIR}:$ENV{LD_LIBRARY_PATH}") - else() - # Fall back to system / toolchain libgomp - find_library(OPEN_MP NAMES gomp REQUIRED) - endif() -endif() - -if (NOT MACOSX_FOUND) - execute_process(COMMAND cat /proc/cpuinfo - RESULT_VARIABLE CPUINFO_RET - OUTPUT_VARIABLE CPUINFO) - if (NOT CPUINFO_RET EQUAL 0) - message(FATAL_ERROR "Failed to check CPU features via /proc/cpuinfo") - endif() -endif() - - -function (find_isa CPUINFO TARGET OUT) - string(FIND ${CPUINFO} ${TARGET} ISA_FOUND) - if(NOT ISA_FOUND EQUAL -1) - set(${OUT} ON PARENT_SCOPE) - else() - set(${OUT} OFF PARENT_SCOPE) - endif() -endfunction() - - -function(check_sysctl TARGET OUT) - execute_process(COMMAND sysctl -n "${TARGET}" - RESULT_VARIABLE SYSCTL_RET - OUTPUT_VARIABLE SYSCTL_INFO - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(SYSCTL_RET EQUAL 0 AND - (SYSCTL_INFO STREQUAL "1" OR SYSCTL_INFO GREATER 0)) - set(${OUT} ON PARENT_SCOPE) - else() - set(${OUT} OFF PARENT_SCOPE) - endif() -endfunction() - -if (MACOSX_FOUND AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - message(STATUS "Apple Silicon Detected") - set(APPLE_SILICON_FOUND TRUE) - set(ENABLE_NUMA OFF) - check_sysctl(hw.optional.neon ASIMD_FOUND) - check_sysctl(hw.optional.arm.FEAT_BF16 ARM_BF16_FOUND) - check_sysctl(hw.optional.arm.FEAT_I8MM ARM_I8MM_FOUND) -else() - find_isa(${CPUINFO} "Power11" POWER11_FOUND) - find_isa(${CPUINFO} "POWER10" POWER10_FOUND) - find_isa(${CPUINFO} "POWER9" POWER9_FOUND) - find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support - find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support - find_isa(${CPUINFO} "i8mm" ARM_I8MM_FOUND) # Check for ARM I8MM support - find_isa(${CPUINFO} "S390" S390_FOUND) - find_isa(${CPUINFO} "zvfhmin" RVV_FP16_FOUND) # Check for RISC-V Vector FP16 support - find_isa(${CPUINFO} "zvfbfmin" RVV_BF16_FOUND) # Check for RISC-V Vector BF16 support - - # Support cross-compilation by allowing override via environment variables - if (ENABLE_ARM_BF16) - set(ARM_BF16_FOUND ON) - message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable") - endif() - if (ENABLE_ARM_I8MM) - set(ARM_I8MM_FOUND ON) - message(STATUS - "ARM I8MM support enabled via VLLM_CPU_ARM_I8MM environment variable") - endif() - # Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin - # in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1 - # overrides the detection result. - if (ENABLE_RVV_BF16) - set(RVV_BF16_FOUND ON) - message(STATUS "RVV BF16 support enabled via VLLM_CPU_RVV_BF16 environment variable") - endif() -endif() - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA) - set(ENABLE_X86_ISA ON) - if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND - CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)) - message(FATAL_ERROR "X86 backend requires gcc/g++ >= 12.3") - endif() - # Work around a GCC 15 optimizer crash in oneDNN (tree-ssa-pre pass). - # Keep this scoped to GCC 15+ so older toolchains are unaffected. - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND - CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) - list(APPEND CXX_COMPILE_FLAGS "-fno-tree-pre") - endif() - list(APPEND CXX_COMPILE_FLAGS "-mf16c") - list(APPEND CXX_COMPILE_FLAGS_AVX512 ${CXX_COMPILE_FLAGS}) - list(APPEND CXX_COMPILE_FLAGS_AVX2 ${CXX_COMPILE_FLAGS}) - list(APPEND CXX_COMPILE_FLAGS_AVX512 - "-mavx512f" - "-mavx512vl" - "-mavx512bw" - "-mavx512dq") - list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX - ${CXX_COMPILE_FLAGS_AVX512} - "-mamx-bf16" - "-mamx-tile" - "-mavx512bf16" - "-mavx512vnni") - # Some packaged GCC 14 builds still do not accept -mamx-fp8, so gate on - # actual flag support rather than compiler version alone. - # Option to enable/disable AMX-FP8 support (default: ON) - option(ENABLE_AMX_FP8 "Enable AMX-FP8 support" ON) - - if (ENABLE_AMX_FP8) - check_cxx_compiler_flag("-mamx-fp8" COMPILER_SUPPORTS_AMX_FP8_FLAG) - if (COMPILER_SUPPORTS_AMX_FP8_FLAG) - list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX "-mamx-fp8") - set(AMX_FP8_SUPPORTED TRUE) - else() - set(AMX_FP8_SUPPORTED FALSE) - message(STATUS "AMX-FP8 disabled: compiler does not support -mamx-fp8 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") - endif() - else() - set(AMX_FP8_SUPPORTED FALSE) - message(STATUS "AMX-FP8 disabled by ENABLE_AMX_FP8=OFF") - endif() - - # AVX10.2 arrived in GCC 15, so the target attribute, the fp8 convert - # intrinsics and __builtin_cpu_supports("avx10.2") are all hard errors on - # the 12.3+ the backend otherwise supports. Probe instead of assuming. - check_cxx_compiler_flag("-mavx10.2" COMPILER_SUPPORTS_AVX10_2_FLAG) - if (COMPILER_SUPPORTS_AVX10_2_FLAG) - set(AVX10_2_SUPPORTED TRUE) - else() - set(AVX10_2_SUPPORTED FALSE) - message(STATUS "AVX10.2 disabled: compiler does not support -mavx10.2 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") - endif() - - list(APPEND CXX_COMPILE_FLAGS_AVX2 - "-mavx2") -elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) - message(STATUS "PowerPC detected") - if (POWER9_FOUND) - list(APPEND CXX_COMPILE_FLAGS - "-mvsx" - "-mcpu=power9" - "-mtune=power9") - elseif (POWER10_FOUND OR POWER11_FOUND) - list(APPEND CXX_COMPILE_FLAGS - "-mvsx" - "-mcpu=power10" - "-mtune=power10") - endif() - -elseif (ASIMD_FOUND) - message(STATUS "ARMv8 or later architecture detected") - if(ARM_BF16_FOUND) - message(STATUS "BF16 extension detected") - set(MARCH_FLAGS "-march=armv8.2-a+bf16+dotprod+fp16") - add_compile_definitions(ARM_BF16_SUPPORT) - else() - message(WARNING "BF16 functionality is not available") - set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16") - endif() - if(ARM_I8MM_FOUND) - message(STATUS "I8MM extension detected") - string(APPEND MARCH_FLAGS "+i8mm") - add_compile_definitions(ARM_I8MM_SUPPORT) - endif() - list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) -elseif (S390_FOUND) - message(STATUS "S390 detected") - # Check for S390 VXE support - list(APPEND CXX_COMPILE_FLAGS - "-mvx" - "-mzvector" - "-march=native" - "-mtune=native") -elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") - message(STATUS "RISC-V detected") - if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0) - message(FATAL_ERROR - "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'") - endif() - # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo - # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build. - # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV. - if(NOT DEFINED VLLM_RVV_VLEN) - # Auto-detect: find the largest zvlb in /proc/cpuinfo isa line. - # Skip when cross-compiling — /proc/cpuinfo describes the build host. - if(CMAKE_CROSSCOMPILING) - message(STATUS "Cross-compiling: skipping VLEN auto-detection from /proc/cpuinfo") - elseif(EXISTS /proc/cpuinfo) - file(READ /proc/cpuinfo _cpuinfo) - set(_best 0) - foreach(_n IN ITEMS 128 256 512 1024) - if(_cpuinfo MATCHES "zvl${_n}b") - set(_best ${_n}) - endif() - endforeach() - # Only VLEN=128 and VLEN=256 are supported by the RVV kernels. - if(_best GREATER 256) - message(WARNING - "Detected VLEN=${_best} but only 128/256 are supported; " - "clamping to 256") - set(_best 256) - endif() - if(_best GREATER 0) - set(VLLM_RVV_VLEN ${_best}) - endif() - endif() - # If auto-detect failed (no /proc/cpuinfo or no zvlb reported) - # but the compiler supports RVV, require explicit specification. - if(NOT DEFINED VLLM_RVV_VLEN AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) - message(FATAL_ERROR - "RISC-V RVV is available but VLEN could not be auto-detected. " - "Please specify VLEN explicitly via CMAKE_ARGS:\n" - " CMAKE_ARGS='-DVLLM_RVV_VLEN=128' (for VLEN=128 hardware)\n" - " CMAKE_ARGS='-DVLLM_RVV_VLEN=256' (for VLEN=256 hardware, e.g. Spacemit X100)") - endif() - endif() - if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0) - message(STATUS "RISC-V RVV VLEN=${VLLM_RVV_VLEN}") - # Sources gate FP16/BF16 paths on the compiler-provided - # __riscv_zvfh / __riscv_zvfbfmin macros, which GCC and clang - # define automatically when those extensions appear in -march. - if(RVV_BF16_FOUND) - message(STATUS "BF16 extension detected") - set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) - elseif(RVV_FP16_FOUND) - message(WARNING "BF16 functionality is not available.") - set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) - else() - message(STATUS "compile riscv with scalar (no FP16/BF16)") - set(MARCH_FLAGS -march=rv64gc) - endif() - else() - message(STATUS "compile riscv with scalar") - set(MARCH_FLAGS -march=rv64gc) - endif() - list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) -else() - message(FATAL_ERROR "vLLM CPU backend requires X86, Power9+ ISA, S390X ISA, ARMv8 or RISC-V support.") -endif() - - -# Build oneDNN for GEMM kernels -if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND OR RVV_FP16_FOUND OR RVV_BF16_FOUND OR S390_FOUND) - # Fetch and build Arm Compute Library (ACL) as oneDNN's backend for AArch64 - # TODO [fadara01]: remove this once ACL can be fetched and built automatically as a dependency of oneDNN - set(ONEDNN_AARCH64_USE_ACL OFF CACHE BOOL "") - if(ASIMD_FOUND) - # Set number of parallel build processes - include(ProcessorCount) - ProcessorCount(NPROC) - if(NOT NPROC) - set(NPROC 4) - endif() - - # Fetch and populate ACL - if(DEFINED ENV{ACL_ROOT_DIR} AND IS_DIRECTORY "$ENV{ACL_ROOT_DIR}") - message(STATUS "Using ACL from specified source directory: $ENV{ACL_ROOT_DIR}") - else() - message(STATUS "Downloading Arm Compute Library (ACL) from GitHub") - FetchContent_Populate(arm_compute - SUBBUILD_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-subbuild" - SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-src" - GIT_REPOSITORY https://github.com/ARM-software/ComputeLibrary.git - GIT_TAG v52.6.0 - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE - ) - set(ENV{ACL_ROOT_DIR} "${arm_compute_SOURCE_DIR}") - set(ACL_LIB_DIR "$ENV{ACL_ROOT_DIR}/build") - endif() - - # Build ACL with CMake - set(_cmake_config_cmd - ${CMAKE_COMMAND} -G Ninja -B build - -DARM_COMPUTE_BUILD_SHARED_LIB=OFF - -DCMAKE_BUILD_TYPE=Release - -DARM_COMPUTE_ARCH=armv8.2-a - -DARM_COMPUTE_ENABLE_ASSERTS=OFF - -DARM_COMPUTE_ENABLE_CPPTHREADS=OFF - -DARM_COMPUTE_ENABLE_OPENMP=ON - -DARM_COMPUTE_ENABLE_WERROR=OFF - -DARM_COMPUTE_BUILD_EXAMPLES=OFF - -DARM_COMPUTE_BUILD_TESTING=OFF) - set(_cmake_build_cmd - ${CMAKE_COMMAND} --build build -- -j${NPROC} - ) - - execute_process( - COMMAND ${_cmake_config_cmd} - WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" - ) - execute_process( - COMMAND ${_cmake_build_cmd} - WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" - RESULT_VARIABLE _acl_rc - ) - - if(NOT _acl_rc EQUAL 0) - message(FATAL_ERROR "ACL SCons build failed (exit ${_acl_rc}).") - endif() - message(STATUS "Arm Compute Library (ACL) built successfully.") - - # VLLM/oneDNN settings for ACL - set(ONEDNN_AARCH64_USE_ACL ON CACHE BOOL "" FORCE) - add_compile_definitions(VLLM_USE_ACL) - endif() - - set(FETCHCONTENT_SOURCE_DIR_ONEDNN "$ENV{FETCHCONTENT_SOURCE_DIR_ONEDNN}" CACHE PATH "Path to a local oneDNN source directory.") - - if(FETCHCONTENT_SOURCE_DIR_ONEDNN) - message(STATUS "Using oneDNN from specified source directory: ${FETCHCONTENT_SOURCE_DIR_ONEDNN}") - FetchContent_Declare( - oneDNN - SOURCE_DIR ${FETCHCONTENT_SOURCE_DIR_ONEDNN} - ) - else() - message(STATUS "Downloading oneDNN from GitHub") - if(ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) - message(STATUS "aarch64 detected: using pinned oneDNN commit 9c5be1cc59e368aebf0909e6cf20f981ea61462a") - FetchContent_Declare( - oneDNN - GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG 9c5be1cc59e368aebf0909e6cf20f981ea61462a - GIT_PROGRESS TRUE - GIT_SHALLOW FALSE - ) - else() - FetchContent_Declare( - oneDNN - GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG v3.13 - GIT_PROGRESS TRUE - GIT_SHALLOW TRUE - ) - endif() - endif() - - set(ONEDNN_LIBRARY_TYPE "STATIC") - set(ONEDNN_BUILD_DOC "OFF") - set(ONEDNN_BUILD_EXAMPLES "OFF") - set(ONEDNN_BUILD_TESTS "OFF") - set(ONEDNN_ENABLE_WORKLOAD "INFERENCE") - set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER") - set(ONEDNN_BUILD_GRAPH "OFF") - set(ONEDNN_ENABLE_JIT_PROFILING "ON") - set(ONEDNN_ENABLE_ITT_TASKS "ON") - set(ONEDNN_ENABLE_MAX_CPU_ISA "ON") - set(ONEDNN_ENABLE_CPU_ISA_HINTS "ON") - set(ONEDNN_VERBOSE "ON") - set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - - set(VLLM_BUILD_TYPE ${CMAKE_BUILD_TYPE}) - set(CMAKE_BUILD_TYPE "Release") # remove oneDNN debug symbols to reduce size - - if(S390_FOUND) - FetchContent_GetProperties(oneDNN) - if(NOT onednn_POPULATED) - FetchContent_Populate(oneDNN) - # Patch s390x helpers.h: ALWAYS_INLINE on operator+= breaks C++20/GCC14 - file(READ "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" _helpers_content) - string(REPLACE - "vec_type_t &ALWAYS_INLINE operator+=" - "ALWAYS_INLINE vec_type_t &operator+=" - _helpers_content "${_helpers_content}") - file(WRITE "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" "${_helpers_content}") - add_subdirectory("${onednn_SOURCE_DIR}" "${onednn_BINARY_DIR}") - endif() - else() - FetchContent_MakeAvailable(oneDNN) - endif() - set(CMAKE_BUILD_TYPE ${VLLM_BUILD_TYPE}) - add_library(dnnl_ext OBJECT "csrc/cpu/dnnl_helper.cpp") - target_include_directories( - dnnl_ext - PUBLIC ${oneDNN_SOURCE_DIR}/include - PUBLIC ${oneDNN_BINARY_DIR}/include - PRIVATE ${oneDNN_SOURCE_DIR}/src - ) - target_link_libraries(dnnl_ext dnnl torch) - if (ENABLE_X86_ISA) - target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS_AVX2} -fPIC) - else() - target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS} -fPIC) - endif() - list(APPEND LIBS dnnl_ext) - - - set(USE_ONEDNN ON) -else() - set(USE_ONEDNN OFF) -endif() - -# TODO: Refactor this -if (ENABLE_X86_ISA) - message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) compile flags: ${CXX_COMPILE_FLAGS_AVX512_AMX}") - message(STATUS "CPU extension (AVX512F) compile flags: ${CXX_COMPILE_FLAGS_AVX512}") - message(STATUS "CPU extension (AVX2) compile flags: ${CXX_COMPILE_FLAGS_AVX2}") -else() - message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}") -endif() - -if(ENABLE_NUMA) - list(APPEND LIBS numa) -else() - message(STATUS "NUMA is disabled") - add_compile_definitions(-DVLLM_NUMA_DISABLED) -endif() - -# check if the pytorch wheel ships libopenblas.so. -set(VLLM_OPENBLAS_LIB "") -if (NOT ENABLE_X86_ISA) - file(GLOB _VLLM_TORCH_OPENBLAS_LIBS - "${TORCH_INSTALL_PREFIX}/lib/libopenblas*.so*") - # Note: we don't link openblas directly to _C extension, as it's available through libtorch.so - if (_VLLM_TORCH_OPENBLAS_LIBS) - list(GET _VLLM_TORCH_OPENBLAS_LIBS 0 VLLM_OPENBLAS_LIB) - message(STATUS "CPU OpenBLAS library: ${VLLM_OPENBLAS_LIB}") - endif() -endif() - -# -# Generate CPU attention dispatch header -# -message(STATUS "Generating CPU attention dispatch header") -execute_process( - COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/csrc/cpu/generate_cpu_attn_dispatch.py - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csrc/cpu - RESULT_VARIABLE GEN_RESULT -) -if(NOT GEN_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to generate CPU attention dispatch header") -endif() - -# -# _C extension -# -set(VLLM_EXT_SRC - "csrc/cpu/activation.cpp" - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp") - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND - VLLM_RVV_VLEN GREATER 0 AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (S390_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/shm.cpp" - "csrc/cpu/activation_lut_bf16.cpp" - "csrc/cpu/cpu_tanhf_neon.hpp" - ${VLLM_EXT_SRC}) - if (ARM_BF16_FOUND) - set(VLLM_EXT_SRC "csrc/cpu/sgl-kernels/conv.cpp" ${VLLM_EXT_SRC}) - if (ARM_I8MM_FOUND) - set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC}) - endif() - endif() -endif() - -if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_wna16.cpp" - "csrc/cpu/shm.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (POWER10_FOUND OR POWER11_FOUND) - set(VLLM_EXT_SRC - "csrc/cpu/cpu_fused_moe_int8.cpp" - ${VLLM_EXT_SRC}) -endif() - -if(USE_ONEDNN) - set(VLLM_EXT_SRC - "csrc/cpu/dnnl_kernels.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") - set(VLLM_EXT_SRC - "csrc/cpu/sgl-kernels/gemm_int4.cpp" - ${VLLM_EXT_SRC}) -endif() - -if (ENABLE_X86_ISA) - set(VLLM_EXT_SRC_SGL - "csrc/cpu/sgl-kernels/conv.cpp" - "csrc/cpu/sgl-kernels/gemm.cpp" - "csrc/cpu/sgl-kernels/gemm_int8.cpp" - "csrc/cpu/sgl-kernels/gemm_fp8.cpp" - "csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp" - "csrc/cpu/sgl-kernels/gemm_int4.cpp" - "csrc/cpu/sgl-kernels/moe.cpp" - "csrc/cpu/sgl-kernels/moe_int8.cpp" - "csrc/cpu/sgl-kernels/moe_int4.cpp" - "csrc/cpu/sgl-kernels/moe_fp8.cpp" - "csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp" - "csrc/cpu/sgl-kernels/bmm.cpp" - "csrc/cpu/sgl-kernels/decode.cpp" - "csrc/cpu/sgl-kernels/extend.cpp" - "csrc/cpu/sgl-kernels/mla_cache.cpp" - "csrc/cpu/sgl-kernels/mhc.cpp" - "csrc/cpu/sgl-kernels/store_cache.cpp" - "csrc/cpu/sgl-kernels/flash_mla.cpp" - "csrc/cpu/sgl-kernels/compressor.cpp" - "csrc/cpu/sgl-kernels/paged_mqa_logits.cpp" - "csrc/cpu/sgl-kernels/topk.cpp" - "csrc/cpu/sgl-kernels/indexer.cpp") - - set(VLLM_EXT_SRC_AVX512 - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/shm.cpp" - "csrc/cpu/cpu_wna16.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/dnnl_kernels.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp" - # TODO: Remove these files - "csrc/cpu/activation.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp") - - set(VLLM_EXT_SRC_AVX2 - "csrc/cpu/sgl-kernels/fla.cpp" - "csrc/cpu/cpu_fused_moe.cpp" - "csrc/cpu/utils.cpp" - "csrc/cpu/spec_decode_utils.cpp" - "csrc/cpu/cpu_attn.cpp" - "csrc/cpu/cpu_isa.cpp" - "csrc/cpu/mamba_cpu.cpp" - "csrc/cpu/dnnl_kernels.cpp" - "csrc/cpu/sampling_kernels.cpp" - "csrc/cpu/torch_bindings.cpp" - # TODO: Remove these files - "csrc/cpu/activation.cpp" - "csrc/cpu/layernorm.cpp" - "csrc/cpu/mla_decode.cpp" - "csrc/cpu/pos_encoding.cpp" - "csrc/cpu/dynamic_4bit_int_moe_cpu.cpp") - - message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) source files: ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL}") - message(STATUS "CPU extension (AVX512F) source files: ${VLLM_EXT_SRC_AVX512}") - message(STATUS "CPU extension (AVX2) source files: ${VLLM_EXT_SRC_AVX2}") - - set(_C_LIBS numa dnnl_ext) - set(_C_AVX512_LIBS numa dnnl_ext) - set(_C_AVX2_LIBS numa dnnl_ext) - - # AMX + AVX512F + AVX512BF16 + AVX512VNNI - define_extension_target( - _C - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL} - LIBRARIES ${_C_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512_AMX} - USE_SABI 3 - WITH_SOABI - ) - - # For AMX kernels - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXBF16") - if (AMX_FP8_SUPPORTED) - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXFP8") - message(STATUS "AMX-FP8 (Diamond Rapids) enabled") - endif() - - # For the sgl-kernels AVX10.2 fp8 quantize paths - if (AVX10_2_SUPPORTED) - target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX10_2") - message(STATUS "AVX10.2 enabled") - endif() - - # AVX512F - define_extension_target( - _C_AVX512 - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX512} - LIBRARIES ${_C_AVX512_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512} - USE_SABI 3 - WITH_SOABI - ) - - # AVX2 - define_extension_target( - _C_AVX2 - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC_AVX2} - LIBRARIES ${_C_AVX2_LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX2} - USE_SABI 3 - WITH_SOABI - ) -else() - message(STATUS "CPU extension source files: ${VLLM_EXT_SRC}") - # - # Define extension targets - # - define_extension_target( - _C - DESTINATION vllm - LANGUAGE CXX - SOURCES ${VLLM_EXT_SRC} - LIBRARIES ${LIBS} - COMPILE_FLAGS ${CXX_COMPILE_FLAGS} - USE_SABI 3 - WITH_SOABI - ) - if (VLLM_OPENBLAS_LIB) - target_compile_definitions(_C PRIVATE VLLM_HAS_OPENBLAS) - endif() -endif() - -message(STATUS "Enabling C extension.") diff --git a/csrc/cpu/activation.cpp b/csrc/cpu/activation.cpp deleted file mode 100644 index 2f06813a19..0000000000 --- a/csrc/cpu/activation.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "cpu_types.hpp" - -namespace { -template -void activation_kernel(int num_tokens, int d, scalar_t* __restrict__ input, - scalar_t* __restrict__ output) { - using scalar_vec_t = vec_op::vec_t; - constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num(); - - TORCH_CHECK(d % VEC_ELEM_NUM == 0); - -#pragma omp parallel for - for (int i = 0; i < num_tokens; ++i) { - for (int j = 0; j < d; j += VEC_ELEM_NUM) { - int start = i * d; - if constexpr (is_gated) { - start *= 2; - } - - const scalar_vec_t x(input + start + j); - const vec_op::FP32Vec8 f32_x(x); - vec_op::FP32Vec8 f32_ans = func(f32_x); - - if constexpr (is_gated) { - const scalar_vec_t y(input + start + d + j); - const vec_op::FP32Vec8 f32_y(y); - f32_ans = f32_y * f32_ans; - } - - const scalar_vec_t result(f32_ans); - result.save(output + i * d + j); - } - } -} - -FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 zeros(0.0); - const vec_op::FP32Vec8 ones(1.0); - return x / (ones + (zeros - x).exp()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(0.79788456f); - const vec_op::FP32Vec8 w2(0.044715f); - const vec_op::FP32Vec8 w3(0.5); - const vec_op::FP32Vec8 x3 = x * x * x; - const vec_op::FP32Vec8 t = (w1 * (x + w2 * x3)).tanh(); - return w3 * x * (ones + t); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(0.79788456f); - const vec_op::FP32Vec8 w2(0.044715f); - const vec_op::FP32Vec8 w3(0.5); - const vec_op::FP32Vec8 t = (x * w1 * (ones + x * w2 * x)).tanh(); - return w3 * x * (ones + t); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_quick_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 zeros(0.0); - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(1.702f); - return x / (ones + (zeros - w1 * x).exp()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(M_SQRT1_2); - const vec_op::FP32Vec8 w2(0.5); - return x * w2 * (ones + (x * w1).er()); -} - -FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8& x) { - const vec_op::FP32Vec8 ones(1.0); - const vec_op::FP32Vec8 w1(M_SQRT2 * M_2_SQRTPI * 0.5); - const vec_op::FP32Vec8 w2(0.5); - const vec_op::FP32Vec8 w3(0.044715); - const vec_op::FP32Vec8 x_3 = x * x * x; - const vec_op::FP32Vec8 inner = w1 * (x + x_3 * w3); - return x * w2 * (ones + inner.tanh()); -} -}; // namespace - -void silu_and_mul(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "silu_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(silu_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(silu_and_mul_impl) - }); -} - -void gelu_and_mul(torch::Tensor& out, // [..., d] - torch::Tensor& input) // [..., 2 * d] -{ - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl) - }); -} - -void gelu_tanh_and_mul(torch::Tensor& out, // [..., d] - torch::Tensor& input) // [..., 2 * d] -{ - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1) / 2; - - VLLM_DISPATCH_FLOATING_TYPES( - input.scalar_type(), "gelu_tanh_and_mul_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_tanh_and_mul_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), - out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_tanh_and_mul_impl) - }); -} - -void gelu_tanh(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_tanh_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_tanh_impl) - }); -} - -void gelu_new(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_new_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_new_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_new_impl) - }); -} - -void gelu_fast(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_fast_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_fast_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_fast_impl) - }); -} - -void gelu_quick(torch::Tensor& out, torch::Tensor& input) { - int num_tokens = input.numel() / input.size(-1); - int d = input.size(-1); - - VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_quick_impl", [&] { - CPU_KERNEL_GUARD_IN(gelu_quick_impl) - activation_kernel( - num_tokens, d, input.data_ptr(), out.data_ptr()); - CPU_KERNEL_GUARD_OUT(gelu_quick_impl) - }); -} diff --git a/csrc/cpu/activation_lut_bf16.cpp b/csrc/cpu/activation_lut_bf16.cpp deleted file mode 100644 index 0ff2567e1e..0000000000 --- a/csrc/cpu/activation_lut_bf16.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "cpu_types.hpp" - -#include -#include -#include -#include - -#include -#include -#include - -constexpr uint32_t ActivationLutSize = 1u << 16; - -at::Tensor gelu_reference(const at::Tensor& x) { return at::gelu(x, "none"); } - -void maybe_init_activation_lut_bf16( - uint16_t* lut, std::once_flag& once, - at::Tensor (*activation)(const at::Tensor&)) { - std::call_once(once, [&]() { - auto lut_input = - at::empty({static_cast(ActivationLutSize)}, - at::TensorOptions().device(at::kCPU).dtype(at::kFloat)); - auto* lut_input_ptr = lut_input.data_ptr(); -#pragma omp parallel for - for (uint32_t i = 0; i < ActivationLutSize; ++i) { - lut_input_ptr[i] = c10::detail::f32_from_bits(static_cast(i)); - } - - auto lut_output = activation(lut_input); - const auto* lut_output_ptr = lut_output.data_ptr(); -#pragma omp parallel for - for (uint32_t i = 0; i < ActivationLutSize; ++i) { - lut[i] = c10::detail::round_to_nearest_even(lut_output_ptr[i]); - } - }); -} - -void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, - const uint16_t* lut, const char* op_name) { - TORCH_CHECK(input.scalar_type() == at::kBFloat16, op_name, - ": input must be bfloat16"); - TORCH_CHECK(out.scalar_type() == at::kBFloat16, op_name, - ": out must be bfloat16"); - TORCH_CHECK(input.is_contiguous(), op_name, ": input must be contiguous"); - TORCH_CHECK(out.is_contiguous(), op_name, ": out must be contiguous"); - - const auto* src = - reinterpret_cast(input.data_ptr()); - auto* dst = reinterpret_cast(out.data_ptr()); - const int64_t n = input.numel(); - - CPU_KERNEL_GUARD_IN(activation_lut_bf16_impl) -#pragma omp parallel for - for (int64_t i = 0; i < n; ++i) { - dst[i] = lut[src[i]]; - } - CPU_KERNEL_GUARD_OUT(activation_lut_bf16_impl) -} - -void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, - const std::string& activation) { - if (activation == "gelu") { - static std::array lut{}; - static std::once_flag once; - maybe_init_activation_lut_bf16(lut.data(), once, gelu_reference); - activation_lut_bf16(out, input, lut.data(), "gelu_lut"); - return; - } - - TORCH_CHECK(false, "Unsupported activation: ", activation); -} diff --git a/csrc/cpu/cpu_arch_macros.h b/csrc/cpu/cpu_arch_macros.h deleted file mode 100644 index ccec4c2e87..0000000000 --- a/csrc/cpu/cpu_arch_macros.h +++ /dev/null @@ -1,197 +0,0 @@ -#ifndef CPU_ARCH_MACROS_H -#define CPU_ARCH_MACROS_H - -// x86_64 -#ifdef __x86_64__ - #define FAST_SPINNING _mm_pause(); - - #ifdef __AVX512F__ - #define DEFINE_FAST_EXP \ - const __m512 vec_factorial_1 = _mm512_set1_ps(0.999999701f); \ - const __m512 vec_factorial_2 = _mm512_set1_ps(0.499991506f); \ - const __m512 vec_factorial_3 = _mm512_set1_ps(0.166676521f); \ - const __m512 vec_factorial_4 = _mm512_set1_ps(0.0418978221f); \ - const __m512 vec_factorial_5 = _mm512_set1_ps(0.00828929059f); \ - const __m512 vec_exp_log2ef = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); \ - const __m512 vec_half = _mm512_set1_ps(0.5f); \ - const __m512 vec_one = _mm512_set1_ps(1.f); \ - const __m512 vec_zero = _mm512_set1_ps(0.f); \ - const __m512 vec_two = _mm512_set1_ps(2.f); \ - const __m512 vec_ln2f = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x3f317218)); \ - const __m512 vec_ln_flt_min = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50)); \ - const __m512 vec_ln_flt_max = \ - _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218)); \ - const __m512i vec_127 = _mm512_set1_epi32(0x0000007f); \ - const int n_mantissa_bits = 23; \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) __attribute__(( \ - always_inline)) { \ - __m512 values = vec.reg; \ - auto less_ln_flt_min_mask = \ - _mm512_cmp_ps_mask(values, vec_ln_flt_min, 1 /*_CMP_LT_OS*/); \ - auto vec_src = _mm512_min_ps(values, vec_ln_flt_max); \ - vec_src = _mm512_max_ps(vec_src, vec_ln_flt_min); \ - auto vec_fx = _mm512_fmadd_ps(vec_src, vec_exp_log2ef, vec_half); \ - auto vec_fx_i = _mm512_cvt_roundps_epi32( \ - vec_fx, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); \ - vec_fx = _mm512_cvtepi32_ps(vec_fx_i); \ - auto vec_exp_poly = _mm512_fnmadd_ps(vec_fx, vec_ln2f, vec_src); \ - auto vec_res = \ - _mm512_fmadd_ps(vec_exp_poly, vec_factorial_5, vec_factorial_4); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_3); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_2); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_1); \ - vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_one); \ - auto vec_exp_number = _mm512_sub_ps(vec_fx, vec_one); \ - auto vec_exp_number_i = _mm512_cvtps_epi32(vec_exp_number); \ - auto vec_two_pow_n_i = _mm512_add_epi32(vec_exp_number_i, vec_127); \ - vec_two_pow_n_i = _mm512_slli_epi32(vec_two_pow_n_i, n_mantissa_bits); \ - auto vec_two_pow_n = _mm512_castsi512_ps(vec_two_pow_n_i); \ - vec_two_pow_n = _mm512_mask_blend_ps(less_ln_flt_min_mask, \ - vec_two_pow_n, vec_zero); \ - vec_res = _mm512_mul_ps(vec_res, vec_two_pow_n); \ - vec_res = _mm512_mul_ps(vec_res, vec_two); \ - vec_op::FP32Vec16 res(vec_res); \ - return res; \ - }; - #endif - -#endif - -#ifdef __aarch64__ - // Implementation of neon_expf copied from Arm Optimized Routines (expf - // AdvSIMD) - // https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/expf.c - // - // Additional fast exponential intended for cases where outputs will be - // downcasted to FP16 / BF16 (e.g. attention softmax). Accurate within 1 ULP - // for FP16 Accurate within 1 ULP for BF16 for inputs in [-87.683, 88.376] & - // clamps inputs outside this range to 0 / inf. Implementation is similar to - // exp_u20, but: - // - uses a third degree polynomial approximation for exp(r) instead of a - // fifth degree one, with coefficients re-tuned. - // - does not split natural log (ln) into high / low parts - // - clamps exp(x) to 0 for x < -87.683113f and inf for x > 88.3762589f - // exp(x) = 2^n (exp(r)) - // r = x - n*ln2, with n = round(x/ln2) - // exp(r) ~ poly(r) = 1 + r + r^2 * (c3 + c2 * r) - // n = round(x / ln2), r = x - n*ln2 - #include - #define DEFINE_FAST_EXP \ - const float32x4_t inv_ln2 = vdupq_n_f32(0x1.715476p+0f); \ - const float ln2_hi = 0x1.62e4p-1f; \ - const float ln2_lo = 0x1.7f7d1cp-20f; \ - const float c0 = 0x1.0e4020p-7f; \ - const float c2 = 0x1.555e66p-3f; \ - const float32x4_t ln2_c02 = {ln2_hi, ln2_lo, c0, c2}; \ - const uint32x4_t exponent_bias = vdupq_n_u32(0x3f800000); \ - const float32x4_t c1 = vdupq_n_f32(0x1.573e2ep-5f); \ - const float32x4_t c3 = vdupq_n_f32(0x1.fffdb6p-2f); \ - const float32x4_t c4 = vdupq_n_f32(0x1.ffffecp-1f); \ - const float32x4_t pos_special_bound = vdupq_n_f32(0x1.5d5e2ap+6f); \ - const float32x4_t neg_special_bound = vnegq_f32(pos_special_bound); \ - const float32x4_t inf = \ - vdupq_n_f32(std::numeric_limits::infinity()); \ - const float32x4_t zero = vdupq_n_f32(0.0f); \ - auto neon_expf = [&](float32x4_t values) __attribute__((always_inline)) { \ - float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \ - float32x4_t r = vfmsq_laneq_f32(values, n, ln2_c02, 0); \ - r = vfmsq_laneq_f32(r, n, ln2_c02, 1); \ - uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \ - float32x4_t scale = vreinterpretq_f32_u32(vaddq_u32(e, exponent_bias)); \ - float32x4_t r2 = vmulq_f32(r, r); \ - float32x4_t p = vfmaq_laneq_f32(c1, r, ln2_c02, 2); \ - float32x4_t q = vfmaq_laneq_f32(c3, r, ln2_c02, 3); \ - q = vfmaq_f32(q, p, r2); \ - p = vmulq_f32(c4, r); \ - float32x4_t poly = vfmaq_f32(p, q, r2); \ - poly = vfmaq_f32(scale, poly, scale); \ - const uint32x4_t hi_mask = vcgeq_f32(values, pos_special_bound); \ - const uint32x4_t lo_mask = vcleq_f32(values, neg_special_bound); \ - poly = vbslq_f32(hi_mask, inf, poly); \ - return vbslq_f32(lo_mask, zero, poly); \ - }; \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { \ - float32x4x4_t result; \ - result.val[0] = neon_expf(vec.reg.val[0]); \ - result.val[1] = neon_expf(vec.reg.val[1]); \ - result.val[2] = neon_expf(vec.reg.val[2]); \ - result.val[3] = neon_expf(vec.reg.val[3]); \ - return vec_op::FP32Vec16(result); \ - }; \ - const float32x4_t lower_bound = vdupq_n_f32(-0x1.5ebb82p+6f); \ - const float32x4_t upper_bound = vdupq_n_f32(0x1.61814ap+6f); \ - constexpr float ln2 = 0x1.62e43p-1f; \ - constexpr float f_c2 = 0x1.5592ecp-3f; \ - const float32x4_t f_c3 = vdupq_n_f32(0x1.017d34p-1f); \ - auto neon_expf_f16 = [&](float32x4_t values) __attribute__(( \ - always_inline)) { \ - const uint32x4_t lt_lower = vcltq_f32(values, lower_bound); \ - const uint32x4_t gt_upper = vcgtq_f32(values, upper_bound); \ - float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \ - float32x4_t r = vfmsq_n_f32(values, n, ln2); \ - uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \ - float32x4_t r2 = vmulq_f32(r, r); \ - float32x4_t q = vfmaq_n_f32(f_c3, r, f_c2); \ - float32x4_t s = vaddq_f32(vdupq_n_f32(1.0f), r); \ - float32x4_t p = vfmaq_f32(s, q, r2); \ - float32x4_t y = \ - vreinterpretq_f32_u32(vaddq_u32(vreinterpretq_u32_f32(p), e)); \ - y = vbslq_f32(lt_lower, vdupq_n_f32(0.0f), y); \ - y = vbslq_f32(gt_upper, vdupq_n_f32(INFINITY), y); \ - return y; \ - }; \ - auto fast_exp_f16 = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { \ - float32x4x4_t result; \ - result.val[0] = neon_expf_f16(vec.reg.val[0]); \ - result.val[1] = neon_expf_f16(vec.reg.val[1]); \ - result.val[2] = neon_expf_f16(vec.reg.val[2]); \ - result.val[3] = neon_expf_f16(vec.reg.val[3]); \ - return vec_op::FP32Vec16(result); \ - }; - -#endif // __aarch64__ - -// RISC-V RVV -#ifdef __riscv_v - #include - - #ifdef __riscv_zihintpause - #define FAST_SPINNING __riscv_pause(); - #endif - - // FP32Vec16::exp() in cpu_types_riscv.hpp already implements the full - // polynomial approximation for RVV, so we simply delegate to it. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; - -#endif // __riscv_v - -// Power VSX -#ifdef __powerpc__ - // FP32Vec16::exp() in cpu_types_vsx.hpp delegates to FP32Vec8::exp(), which - // implements a vectorised 5-term minimax polynomial using VSX intrinsics. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; \ - auto fast_exp_f16 = fast_exp; - -#endif // __powerpc__ - -// IBM Z (s390x) VXE -#ifdef __s390x__ - // FP32Vec16::exp() in cpu_types_vxe.hpp delegates to FP32Vec8::exp(), which - // implements a vectorised 5-term minimax polynomial using VXE intrinsics. - #define DEFINE_FAST_EXP \ - auto fast_exp = [&](const vec_op::FP32Vec16& vec) \ - __attribute__((always_inline)) { return vec.exp(); }; \ - auto fast_exp_f16 = fast_exp; - -#endif // __s390x__ - -#endif diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp deleted file mode 100644 index afdc0e0eca..0000000000 --- a/csrc/cpu/cpu_attn.cpp +++ /dev/null @@ -1,287 +0,0 @@ -#include "cpu_attn_dispatch_generated.h" - -// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. -extern bool runtime_has_amx_fp8(); - -// Maps kv_cache_dtype string to Fp8KVCacheDataType enum. -// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2. -static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype( - const std::string& kv_cache_dtype) { - if (kv_cache_dtype == "fp8_e5m2") - return cpu_attention::Fp8KVCacheDataType::kFp8E5M2; - if (kv_cache_dtype == "fp8_e4m3" || kv_cache_dtype == "fp8") - return cpu_attention::Fp8KVCacheDataType::kFp8E4M3; - return cpu_attention::Fp8KVCacheDataType::kAuto; -} - -bool cpu_attn_has_isa(const std::string& isa) { - if (isa == "rvv") { -#if defined(__riscv) && defined(__riscv_v_min_vlen) && \ - (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256) - return true; -#else - return false; -#endif - } - if (isa == "amx_fp8") { -#ifdef CPU_CAPABILITY_AMXFP8 - // Guard with runtime CPUID check: the binary may be compiled with - // -mamx-fp8 (CPU_CAPABILITY_AMXFP8 defined) but run on a CPU that - // has AMX-BF16 without AMX-FP8 (e.g. Sapphire Rapids, Emerald Rapids). - return runtime_has_amx_fp8(); -#else - return false; -#endif - } - return false; -} - -torch::Tensor get_scheduler_metadata( - const int64_t num_req, const int64_t num_heads_q, - const int64_t num_heads_kv, const int64_t head_dim, - const torch::Tensor& seq_lens, at::ScalarType dtype, - const torch::Tensor& query_start_loc, const bool causal, - const int64_t window_size, const std::string& isa_hint, - const bool enable_kv_split, - const std::optional& dynamic_causal, - const std::string& kv_cache_dtype) { - cpu_attention::ISA isa; - if (isa_hint == "amx") { - isa = cpu_attention::ISA::AMX; - } else if (isa_hint == "amx_fp8") { - isa = cpu_attention::ISA::AMX_FP8; - } else if (isa_hint == "vec") { - isa = cpu_attention::ISA::VEC; - } else if (isa_hint == "vec16") { - isa = cpu_attention::ISA::VEC16; - } else if (isa_hint == "neon") { - isa = cpu_attention::ISA::NEON; - } else if (isa_hint == "vxe") { - isa = cpu_attention::ISA::VXE; - } else if (isa_hint == "rvv") { - isa = cpu_attention::ISA::RVV; - } else if (isa_hint == "vsx") { - isa = cpu_attention::ISA::VSX; - } else { - TORCH_CHECK(false, "Unsupported CPU attention ISA hint: " + isa_hint); - } - - cpu_attention::AttentionScheduler::ScheduleInput input; - input.num_reqs = num_req; - input.num_heads_q = num_heads_q; - input.num_heads_kv = num_heads_kv; - input.head_dim = head_dim; - input.query_start_loc = query_start_loc.data_ptr(); - input.seq_lens = seq_lens.data_ptr(); - - input.sliding_window_size = window_size; - input.causal = causal; - input.isa = isa; - input.enable_kv_split = enable_kv_split; - input.dynamic_causal = dynamic_causal.has_value() - ? dynamic_causal->data_ptr() - : nullptr; - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() { - CPU_ATTN_DISPATCH(head_dim, isa, kv_cache_idx, [&]() { - input.elem_size = sizeof(attn_impl::kv_cache_t); - input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t); - input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t); - input.output_buffer_elem_size = - sizeof(attn_impl::partial_output_buffer_t); - input.max_num_q_per_iter = attn_impl::MaxQHeadNumPerIteration; - input.kv_block_alignment = attn_impl::BlockSizeAlignment; - }); - }); - - cpu_attention::AttentionScheduler scheduler; - torch::Tensor metadata = scheduler.schedule(input); - return metadata; -} - -void cpu_attn_reshape_and_cache( - const torch::Tensor& key, // [token_num, head_num, head_size] - const torch::Tensor& value, // [token_num, head_num, head_size] - torch::Tensor& - key_cache, // [num_blocks, num_kv_heads, block_size, head_size] - torch::Tensor& - value_cache, // [num_blocks, num_kv_heads, block_size, head_size] - const torch::Tensor& slot_mapping, const std::string& isa, - const double k_scale = 1.0, const double v_scale = 1.0, - const std::string& kv_cache_dtype = "auto") { - TORCH_CHECK_EQ(key.dim(), 3); - TORCH_CHECK_EQ(value.dim(), 3); - TORCH_CHECK_EQ(key_cache.dim(), 4); - TORCH_CHECK_EQ(value_cache.dim(), 4); - TORCH_CHECK_EQ(key.stride(2), 1); - TORCH_CHECK_EQ(value.stride(2), 1); - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - const bool is_fp8 = (kv_cache_idx != 0); - - if (is_fp8) { - TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte, - "key_cache must be uint8 for FP8 path"); - TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte, - "value_cache must be uint8 for FP8 path"); - TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path"); - TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path"); - } - - const float k_inv = is_fp8 ? 1.0f / static_cast(k_scale) : 0.0f; - const float v_inv = is_fp8 ? 1.0f / static_cast(v_scale) : 0.0f; - - const int64_t token_num = key.size(0); - const int64_t head_num = key.size(1); - const int64_t head_dim = key.size(2); - const int64_t num_blocks = key_cache.size(0); - const int64_t num_blocks_stride = key_cache.stride(0); - const int64_t cache_head_num_stride = key_cache.stride(1); - const int64_t block_size = key_cache.size(2); - const int64_t block_size_stride = key_cache.stride(2); - - cpu_attention::ISA isa_tag = [&]() { - if (isa == "amx") { - return cpu_attention::ISA::AMX; - } else if (isa == "amx_fp8") { - return cpu_attention::ISA::AMX_FP8; - } else if (isa == "vec") { - return cpu_attention::ISA::VEC; - } else if (isa == "vec16") { - return cpu_attention::ISA::VEC16; - } else if (isa == "neon") { - return cpu_attention::ISA::NEON; - } else if (isa == "vxe") { - return cpu_attention::ISA::VXE; - } else if (isa == "rvv") { - return cpu_attention::ISA::RVV; - } else if (isa == "vsx") { - return cpu_attention::ISA::VSX; - } else { - TORCH_CHECK(false, "Invalid ISA type: " + isa); - } - }(); - - if (is_fp8) { - TORCH_CHECK(isa_tag == cpu_attention::ISA::AMX || - isa_tag == cpu_attention::ISA::AMX_FP8 || - isa_tag == cpu_attention::ISA::VEC, - "FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA"); - } - - VLLM_DISPATCH_FLOATING_TYPES( - key.scalar_type(), "cpu_attn_reshape_and_cache", [&]() { - CPU_ATTN_DISPATCH(head_dim, isa_tag, kv_cache_idx, [&]() { - using kv_t = typename attn_impl::kv_cache_t; - attn_impl::reshape_and_cache( - key.data_ptr(), value.data_ptr(), - reinterpret_cast(key_cache.data_ptr()), - reinterpret_cast(value_cache.data_ptr()), - slot_mapping.data_ptr(), token_num, key.stride(0), - value.stride(0), head_num, key.stride(1), value.stride(1), - num_blocks, num_blocks_stride, cache_head_num_stride, block_size, - block_size_stride, k_inv, v_inv); - }); - }); -} - -void cpu_attention_with_kv_cache( - const torch::Tensor& query, // [num_tokens, num_heads, head_size] - const torch::Tensor& - key_cache, // [num_blocks, num_kv_heads, block_size, head_size] - const torch::Tensor& - value_cache, // [num_blocks, num_kv_heads, block_size, head_size] - torch::Tensor& output, // [num_tokens, num_heads, head_size] - const torch::Tensor& query_start_loc, // [num_tokens + 1] - const torch::Tensor& seq_lens, // [num_tokens] - const double scale, const bool causal, - const std::optional& alibi_slopes, // [num_heads] - const int64_t sliding_window, - const torch::Tensor& block_table, // [num_tokens, max_block_num] - const double softcap, const torch::Tensor& scheduler_metadata, - const std::optional& s_aux, // [num_heads] - const std::optional& dynamic_causal, // [num_reqs] - const double k_scale = 1.0, const double v_scale = 1.0, - const std::string& kv_cache_dtype = "auto") { - TORCH_CHECK_EQ(query.dim(), 3); - TORCH_CHECK_EQ(query.stride(2), 1); - TORCH_CHECK_EQ(key_cache.dim(), 4); - TORCH_CHECK_EQ(value_cache.dim(), 4); - - const int64_t kv_cache_idx = - static_cast(parse_fp8_kv_dtype(kv_cache_dtype)); - const bool is_fp8 = (kv_cache_idx != 0); - if (is_fp8) { - TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte, - "key_cache must be uint8 for FP8 path"); - TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte, - "value_cache must be uint8 for FP8 path"); - TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path"); - TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path"); - } - - cpu_attention::AttentionInput input; - input.metadata = reinterpret_cast( - scheduler_metadata.data_ptr()); - input.num_tokens = query.size(0); - input.num_heads = query.size(1); - input.num_kv_heads = key_cache.size(1); - input.block_size = key_cache.size(2); - input.query = query.data_ptr(); - input.query_num_tokens_stride = query.stride(0); - input.query_num_heads_stride = query.stride(1); - input.cache_num_blocks_stride = key_cache.stride(0); - input.cache_num_kv_heads_stride = key_cache.stride(1); - input.blt_num_tokens_stride = block_table.stride(0); - input.key_cache = key_cache.data_ptr(); - input.value_cache = value_cache.data_ptr(); - input.output = output.data_ptr(); - input.query_start_loc = query_start_loc.data_ptr(); - input.seq_lens = seq_lens.data_ptr(); - input.block_table = block_table.data_ptr(); - input.alibi_slopes = - alibi_slopes.has_value() ? alibi_slopes->data_ptr() : nullptr; - // Attention sinks may be bf16 (native path) or fp32. Anything that is not - // bf16 must be provided as fp32 and is executed in full float precision. - if (s_aux.has_value()) { - TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 || - s_aux->scalar_type() == at::ScalarType::Float, - "cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ", - s_aux->scalar_type(), " must be bfloat16 or float32"); - input.s_aux = s_aux->data_ptr(); - input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16; - } else { - input.s_aux = nullptr; - input.s_aux_is_bf16 = false; - } - input.dynamic_causal = dynamic_causal.has_value() - ? dynamic_causal->data_ptr() - : nullptr; - input.scale = scale; - input.causal = causal; - input.sliding_window_size = sliding_window; - input.softcap = static_cast(softcap); - - if (is_fp8) { - input.k_scale_fp8 = static_cast(k_scale); - input.v_scale_fp8 = static_cast(v_scale); - TORCH_CHECK(input.metadata->isa == cpu_attention::ISA::AMX || - input.metadata->isa == cpu_attention::ISA::AMX_FP8 || - input.metadata->isa == cpu_attention::ISA::VEC, - "FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA"); - } - - VLLM_DISPATCH_FLOATING_TYPES( - query.scalar_type(), "cpu_attention_with_kv_cache", [&]() { - CPU_ATTN_DISPATCH( - query.size(2), input.metadata->isa, kv_cache_idx, [&]() { - TORCH_CHECK_EQ(input.block_size % attn_impl::BlockSizeAlignment, - 0); - cpu_attention::AttentionMainLoop mainloop; - mainloop(&input); - }); - }); -} diff --git a/csrc/cpu/cpu_attn_amx.hpp b/csrc/cpu/cpu_attn_amx.hpp deleted file mode 100644 index 6a0341085d..0000000000 --- a/csrc/cpu/cpu_attn_amx.hpp +++ /dev/null @@ -1,636 +0,0 @@ -#ifndef CPU_ATTN_AMX_HPP -#define CPU_ATTN_AMX_HPP - -#include "cpu_attn_fp8.hpp" -#include "cpu_attn_impl.hpp" - -namespace cpu_attention { -namespace { -// AMX specific -constexpr static int64_t AMX_TILE_ROW_BYTES = 64; -constexpr static int64_t AMX_TILE_ROW_NUM = 16; -constexpr static int64_t AMX_TILE_BYTES = AMX_TILE_ROW_BYTES * AMX_TILE_ROW_NUM; - -typedef struct __tile_config { - uint8_t palette_id = 1; - uint8_t start_row = 0; - uint8_t reserved_0[14] = {0}; - uint16_t colsb[16] = {0}; - uint8_t rows[16] = {0}; -} __tilecfg; - -// 2-2-4 pattern, for 16 < m <= 32 -// TILE 0, 1: load A matrix, row num should be 16, m - 16 -// TILE 2, 3: load B matrix, row num should be 16 -// TILE 4, 5, 6, 7: store results C matrix, row num should be 16, 16, -// m - 16, m - 16 -// q_buffer_t: A (Q/P) tile type; kv_cache_t: B (K/V cache) tile type. -template -class TileGemm224 { - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, void* __restrict__ a_tile, - void* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm224"); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm224"); - } -}; - -// Dequantize one FP8 tile (AMX_TILE_ROW_NUM rows x 32 cols) to BF16. -template -FORCE_INLINE void deq_tile_amx(const uint8_t* src, c10::BFloat16* dst) { - for (int r = 0; r < AMX_TILE_ROW_NUM; ++r) { - if constexpr (std::is_same_v) { - vec_op::BF16Vec32(src + r * 32, vec_op::fp8_bf16_e4m3_tag{}) - .save(dst + r * 32); - } else { - vec_op::BF16Vec32(src + r * 32, vec_op::fp8_bf16_e5m2_tag{}) - .save(dst + r * 32); - } - } -} - -// For FP8: dequant src into scratch and return scratch. -// For BF16: return src directly (scratch is unused; the compiler elides it). -template -FORCE_INLINE const c10::BFloat16* prepare_b_tile(const kv_cache_t* src, - c10::BFloat16* scratch) { - if constexpr (std::is_same_v || - std::is_same_v) { - deq_tile_amx(reinterpret_cast(src), scratch); - return scratch; - } else { - return reinterpret_cast(src); - } -} - -// Handles both BF16 and FP8 KV cache (2-2-4 pattern). -template -class TileGemm224 { - static_assert(std::is_same_v || - std::is_same_v || - std::is_same_v, - "kv_cache_t must be BFloat16, Float8_e4m3fn, or Float8_e5m2"); - - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - static constexpr int64_t tile_elems = AMX_TILE_BYTES / sizeof(c10::BFloat16); - // BF16 path: scratch_elems=1 so the scratch array is eliminated by the - // compiler. - static constexpr int64_t scratch_elems = fp8_kv ? tile_elems : 1; - - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, - c10::BFloat16* __restrict__ a_tile, - kv_cache_t* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - const int32_t k_times = - dynamic_k_size / (AMX_TILE_ROW_NUM * 4 / sizeof(c10::BFloat16)); - - c10::BFloat16* __restrict__ a_tile_0 = a_tile; - c10::BFloat16* __restrict__ a_tile_1 = a_tile + lda * AMX_TILE_ROW_NUM; - const int64_t a_tile_stride = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return AMX_TILE_ROW_BYTES; - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return lda * sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - - kv_cache_t* __restrict__ b_tile_2 = b_tile; - kv_cache_t* __restrict__ b_tile_3 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // k_cache is prepacked - return b_tile + (k_size * AMX_TILE_ROW_BYTES / 4); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // v_cache is prepacked - return b_tile + (block_size * AMX_TILE_ROW_BYTES / 4); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - // k_cache, v_cache are prepacked - const int32_t b_tile_stride = AMX_TILE_ROW_BYTES; - - // logits_buffer, output_buffer are not prepacked - float* __restrict__ c_tile_4 = c_tile; - float* __restrict__ c_tile_5 = - c_tile_4 + AMX_TILE_ROW_BYTES / sizeof(float); - float* __restrict__ c_tile_6 = c_tile + AMX_TILE_ROW_NUM * ldc; - float* __restrict__ c_tile_7 = - c_tile_6 + AMX_TILE_ROW_BYTES / sizeof(float); - const int32_t c_tile_stride = ldc * sizeof(float); - - if (accum_c) { - _tile_loadd(4, c_tile_4, c_tile_stride); - _tile_loadd(5, c_tile_5, c_tile_stride); - _tile_loadd(6, c_tile_6, c_tile_stride); - _tile_loadd(7, c_tile_7, c_tile_stride); - } else { - _tile_zero(4); - _tile_zero(5); - _tile_zero(6); - _tile_zero(7); - } - - alignas(64) c10::BFloat16 scratch_2[scratch_elems]; - alignas(64) c10::BFloat16 scratch_3[scratch_elems]; - for (int32_t k = 0; k < k_times; ++k) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_tile_stride); - _tile_dpbf16ps(4, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_tile_stride); - _tile_dpbf16ps(5, 0, 3); - _tile_loadd(1, a_tile_1, a_tile_stride); - _tile_dpbf16ps(6, 1, 2); - _tile_dpbf16ps(7, 1, 3); - - // update ptrs - if constexpr (phase == AttentionGemmPhase::QK) { - // Q buffer is prepacked - a_tile_0 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - a_tile_1 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // P buffer is not prepacked - a_tile_0 += AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - a_tile_1 += AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - b_tile_2 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_3 += AMX_TILE_BYTES / sizeof(c10::BFloat16); - } - - _tile_stored(4, c_tile_4, c_tile_stride); - _tile_stored(5, c_tile_5, c_tile_stride); - _tile_stored(6, c_tile_6, c_tile_stride); - _tile_stored(7, c_tile_7, c_tile_stride); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - const int32_t m_0 = AMX_TILE_ROW_NUM; - const int32_t m_1 = m - AMX_TILE_ROW_NUM; - config.rows[0] = m_0; - config.rows[1] = m_1; - config.rows[2] = AMX_TILE_ROW_NUM; - config.rows[3] = AMX_TILE_ROW_NUM; - config.rows[4] = m_0; - config.rows[5] = m_0; - config.rows[6] = m_1; - config.rows[7] = m_1; - _tile_loadconfig(&config); - } -}; - -// 1-2-2 pattern, for 0 < m <= 16 -// TILE 0, (1): load A matrix, use extra 1 tile for prefetch, row num should -// be m, m -// TILE 2, 3, (4, 5): load B matrix, use extra 2 tiles for prefetch, row num -// should be 16 -// TILE 6, 7: store results C matrix, row num should be m -// q_buffer_t: A (Q/P) tile type; kv_cache_t: B (K/V cache) tile type. -template -class TileGemm122 { - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, void* __restrict__ a_tile, - void* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm122"); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - TORCH_CHECK(false, "Unsupported kv cache type for TileGemm122"); - } -}; - -// Handles both BF16 and FP8 KV cache (1-2-2 pattern). -template -class TileGemm122 { - static_assert(std::is_same_v || - std::is_same_v || - std::is_same_v, - "kv_cache_t must be BFloat16, Float8_e4m3fn, or Float8_e5m2"); - - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - static constexpr int64_t tile_elems = AMX_TILE_BYTES / sizeof(c10::BFloat16); - static constexpr int64_t scratch_elems = fp8_kv ? tile_elems : 1; - - public: - template - FORCE_INLINE static void gemm(const int32_t m_size, - c10::BFloat16* __restrict__ a_tile, - kv_cache_t* __restrict__ b_tile, - float* __restrict__ c_tile, const int64_t lda, - const int64_t ldb, const int64_t ldc, - const int32_t block_size, - const int32_t dynamic_k_size, - const bool accum_c) { - c10::BFloat16* __restrict__ a_tile_0 = a_tile; - c10::BFloat16* __restrict__ a_tile_1 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return a_tile + AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return a_tile + AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - const int64_t a_tile_stride = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - // q_buffer is prepacked - return AMX_TILE_ROW_BYTES; - } else if constexpr (phase == AttentionGemmPhase::PV) { - // logits_buffer is row-major - return lda * sizeof(c10::BFloat16); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - - kv_cache_t* __restrict__ b_tile_2 = b_tile; - kv_cache_t* __restrict__ b_tile_3 = [&]() { - if constexpr (phase == AttentionGemmPhase::QK) { - return b_tile + (k_size * AMX_TILE_ROW_BYTES / 4); - } else if constexpr (phase == AttentionGemmPhase::PV) { - return b_tile + (block_size * AMX_TILE_ROW_BYTES / 4); - } else { - TORCH_CHECK(false, "Unreachable"); - } - }(); - kv_cache_t* __restrict__ b_tile_4 = - b_tile_2 + AMX_TILE_BYTES / sizeof(c10::BFloat16); - kv_cache_t* __restrict__ b_tile_5 = - b_tile_3 + AMX_TILE_BYTES / sizeof(c10::BFloat16); - int64_t b_stride = AMX_TILE_ROW_BYTES; - - float* __restrict__ c_tile_6 = c_tile; - float* __restrict__ c_tile_7 = c_tile + AMX_TILE_ROW_BYTES / sizeof(float); - int64_t c_stride = ldc * sizeof(float); - - const int32_t k_times = - dynamic_k_size / (AMX_TILE_ROW_NUM * 4 / sizeof(c10::BFloat16)); - const int32_t k_group_times = k_times / 2; - const bool has_tail = (k_times % 2 == 1); - - if (accum_c) { - _tile_loadd(6, c_tile_6, c_stride); - _tile_loadd(7, c_tile_7, c_stride); - } else { - _tile_zero(6); - _tile_zero(7); - } - - alignas(64) c10::BFloat16 scratch_2[scratch_elems]; - alignas(64) c10::BFloat16 scratch_3[scratch_elems]; - alignas(64) c10::BFloat16 scratch_4[scratch_elems]; - alignas(64) c10::BFloat16 scratch_5[scratch_elems]; - for (int32_t k = 0; k < k_group_times; ++k) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - const c10::BFloat16* load_4 = prepare_b_tile(b_tile_4, scratch_4); - const c10::BFloat16* load_5 = prepare_b_tile(b_tile_5, scratch_5); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_stride); - _tile_dpbf16ps(6, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_stride); - _tile_dpbf16ps(7, 0, 3); - _tile_loadd(1, a_tile_1, a_tile_stride); - _tile_stream_loadd(4, const_cast(load_4), b_stride); - _tile_dpbf16ps(6, 1, 4); - _tile_stream_loadd(5, const_cast(load_5), b_stride); - _tile_dpbf16ps(7, 1, 5); - - // update ptrs - if constexpr (phase == AttentionGemmPhase::QK) { - // Q buffer is prepacked - a_tile_0 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - a_tile_1 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - } else if constexpr (phase == AttentionGemmPhase::PV) { - // P buffer is not prepacked - a_tile_0 += 2 * AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - a_tile_1 += 2 * AMX_TILE_ROW_BYTES / sizeof(c10::BFloat16); - } - b_tile_2 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_3 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_4 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - b_tile_5 += 2 * AMX_TILE_BYTES / sizeof(c10::BFloat16); - } - - if (has_tail) { - const c10::BFloat16* load_2 = prepare_b_tile(b_tile_2, scratch_2); - const c10::BFloat16* load_3 = prepare_b_tile(b_tile_3, scratch_3); - - _tile_loadd(0, a_tile_0, a_tile_stride); - _tile_stream_loadd(2, const_cast(load_2), b_stride); - _tile_dpbf16ps(6, 0, 2); - _tile_stream_loadd(3, const_cast(load_3), b_stride); - _tile_dpbf16ps(7, 0, 3); - } - - _tile_stored(6, c_tile_6, c_stride); - _tile_stored(7, c_tile_7, c_stride); - } - - FORCE_INLINE static void init_tile_config(int32_t m, __tilecfg& config) { - config.rows[0] = m; - config.rows[1] = m; - config.rows[2] = AMX_TILE_ROW_NUM; - config.rows[3] = AMX_TILE_ROW_NUM; - config.rows[4] = AMX_TILE_ROW_NUM; - config.rows[5] = AMX_TILE_ROW_NUM; - config.rows[6] = m; - config.rows[7] = m; - _tile_loadconfig(&config); - } -}; - -} // namespace - -template -class AttentionImpl { - static constexpr bool fp8_kv = - std::is_same_v || - std::is_same_v; - - public: - using query_t = scalar_t; - using q_buffer_t = scalar_t; - using kv_cache_t = kv_cache_scalar_t; - using logits_buffer_t = float; - using partial_output_buffer_t = float; - using prob_buffer_t = scalar_t; - - constexpr static int64_t BlockSizeAlignment = - 32; // AMX_TILE_ROW_NUM = 16 tokens/tile; 32 = 2 tiles - constexpr static int64_t HeadDimAlignment = - 2 * (AMX_TILE_ROW_BYTES / 4); // headdim num unit of PV phase - constexpr static int64_t MaxQHeadNumPerIteration = 32; - constexpr static int64_t HeadDim = head_dim; - constexpr static ISA ISAType = ISA::AMX; - constexpr static bool scale_on_logits = true; - - float k_scale = 1.0f; - float v_scale = 1.0f; - - public: - AttentionImpl() : current_q_head_num_(0) { - // Use all columns in AMX tiles - vec_op::unroll_loop([&](int i) { amx_tile_config_.colsb[i] = 64; }); - } - - ~AttentionImpl() { _tile_release(); } - - void init_from_input(const AttentionInput* input) { - if constexpr (fp8_kv) { - k_scale = input->k_scale_fp8; - v_scale = input->v_scale_fp8; - } - } - - float get_output_v_scale() const noexcept { - if constexpr (fp8_kv) { - // AMX dequant places FP8 payload into a BF16 field (exponent bias 127). - // Correction = 2^(127 - FP8_bias): E4M3 bias=7 → 2^120, E5M2 bias=15 → - // 2^112. - constexpr float bias = - std::is_same_v ? 0x1p112f : 0x1p120f; - return v_scale * bias; - } - return 1.0f; - } - - template