From 7021be66e8c351fa819fd07b4053e946f11c1147 Mon Sep 17 00:00:00 2001 From: Chris Leonard Date: Fri, 12 Jun 2026 00:22:37 -0400 Subject: [PATCH] [11a/n] Migrate Marlin kernels to torch stable ABI (#45176) Signed-off-by: Chris Leonard --- CMakeLists.txt | 278 +++++------ .../moe/marlin_moe_wna16/kernel.h | 4 +- .../moe/marlin_moe_wna16/marlin_template.h | 8 +- .../gptq_allspark/allspark_utils.cuh | 2 +- .../quantization/marlin/.gitignore | 0 .../quantization/marlin/awq_marlin_repack.cu | 80 ++-- .../quantization/marlin/dequant.h | 0 .../quantization/marlin/generate_kernels.py | 2 +- .../quantization/marlin/gptq_marlin_repack.cu | 91 ++-- .../quantization/marlin/kernel.h | 0 .../quantization/marlin/marlin.cu | 443 ++++++++++-------- .../quantization/marlin/marlin.cuh | 8 - .../quantization/marlin/marlin_dtypes.cuh | 0 .../marlin/marlin_int4_fp8_preprocess.cu | 118 +++++ .../quantization/marlin/marlin_mma.h | 0 .../quantization/marlin/marlin_template.h | 0 csrc/libtorch_stable/torch_bindings.cpp | 29 ++ .../marlin/marlin_int4_fp8_preprocess.cu | 106 ----- csrc/torch_bindings.cpp | 29 -- 19 files changed, 626 insertions(+), 572 deletions(-) rename csrc/{ => libtorch_stable}/quantization/marlin/.gitignore (100%) rename csrc/{ => libtorch_stable}/quantization/marlin/awq_marlin_repack.cu (77%) rename csrc/{ => libtorch_stable}/quantization/marlin/dequant.h (100%) rename csrc/{ => libtorch_stable}/quantization/marlin/generate_kernels.py (99%) rename csrc/{ => libtorch_stable}/quantization/marlin/gptq_marlin_repack.cu (77%) rename csrc/{ => libtorch_stable}/quantization/marlin/kernel.h (100%) rename csrc/{ => libtorch_stable}/quantization/marlin/marlin.cu (61%) rename csrc/{ => libtorch_stable}/quantization/marlin/marlin.cuh (93%) rename csrc/{ => libtorch_stable}/quantization/marlin/marlin_dtypes.cuh (100%) create mode 100644 csrc/libtorch_stable/quantization/marlin/marlin_int4_fp8_preprocess.cu rename csrc/{ => libtorch_stable}/quantization/marlin/marlin_mma.h (100%) rename csrc/{ => libtorch_stable}/quantization/marlin/marlin_template.h (100%) delete mode 100644 csrc/quantization/marlin/marlin_int4_fp8_preprocess.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a48ddca68..c03360a5d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,145 +358,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA") SRCS "${VLLM_EXT_SRC}" CUDA_ARCHS "${CUDA_ARCHS}") - # Only build Marlin kernels if we are building for at least some compatible archs. - # Keep building Marlin for 9.0 as there are some group sizes and shapes that - # are not supported by Machete yet. - - # marlin arches for fp16 output - # Family-conditional 12.0f (one cubin for SM12x family) requires CUDA >= 13.0; - # fall back to architecture-specific 12.0a;12.1a on CUDA < 13.0 (e.g. 12.8). - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}") - endif() - # marlin has limited support for turing - cuda_archs_loose_intersection(MARLIN_SM75_ARCHS "7.5" "${CUDA_ARCHS}") - # marlin arches for bf16 output (we need 9.0 for bf16 atomicAdd PTX) - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}") - endif() - # marlin arches for fp8 input - # - sm80 doesn't support fp8 computation - # - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction - # so we only enable fp8 computation for SM89 (e.g. RTX 40x0) and 12.0 (e.g. RTX 50x0) - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0a;12.1a" "${CUDA_ARCHS}") - endif() - # marlin arches for other files - cuda_archs_loose_intersection(MARLIN_OTHER_ARCHS "7.5;8.0+PTX" "${CUDA_ARCHS}") - - if (MARLIN_OTHER_ARCHS) - - # - # For the Marlin kernels we automatically generate sources for various - # preselected input type pairs and schedules. - # Generate sources: - set(MARLIN_GEN_SCRIPT - ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/marlin/generate_kernels.py) - file(MD5 ${MARLIN_GEN_SCRIPT} MARLIN_GEN_SCRIPT_HASH) - list(JOIN CUDA_ARCHS "," CUDA_ARCHS_STR) - set(MARLIN_GEN_SCRIPT_HASH_AND_ARCH "${MARLIN_GEN_SCRIPT_HASH}(ARCH:${CUDA_ARCHS_STR})") - - message(STATUS "Marlin generation script hash: ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH}") - message(STATUS "Last run Marlin generate script hash: $CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH}") - - if (NOT DEFINED CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH} - OR NOT $CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH} STREQUAL ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH}) - execute_process( - COMMAND ${CMAKE_COMMAND} -E env - PYTHONPATH=$ENV{PYTHONPATH} - ${Python_EXECUTABLE} ${MARLIN_GEN_SCRIPT} ${CUDA_ARCHS_STR} - RESULT_VARIABLE marlin_generation_result - OUTPUT_VARIABLE marlin_generation_result - OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log - ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log - ) - - if (NOT marlin_generation_result EQUAL 0) - message(FATAL_ERROR "Marlin generation failed." - " Result: \"${marlin_generation_result}\"" - "\nCheck the log for details: " - "${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log") - else() - set(MARLIN_GEN_SCRIPT_HASH_AND_ARCH ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH} - CACHE STRING "Last run Marlin generate script hash and arch" FORCE) - message(STATUS "Marlin generation completed successfully.") - endif() - else() - message(STATUS "Marlin generation script has not changed, skipping generation.") - endif() - - if (MARLIN_ARCHS) - file(GLOB MARLIN_TEMPLATE_KERNEL_SRC "csrc/quantization/marlin/sm80_kernel_*_float16.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_TEMPLATE_KERNEL_SRC}" - CUDA_ARCHS "${MARLIN_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_TEMPLATE_KERNEL_SRC} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_EXT_SRC ${MARLIN_TEMPLATE_KERNEL_SRC}) - - file(GLOB MARLIN_TEMPLATE_BF16_KERNEL_SRC "csrc/quantization/marlin/sm80_kernel_*_bfloat16.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_TEMPLATE_BF16_KERNEL_SRC}" - CUDA_ARCHS "${MARLIN_BF16_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_TEMPLATE_BF16_KERNEL_SRC} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_EXT_SRC ${MARLIN_TEMPLATE_BF16_KERNEL_SRC}) - endif() - - if (MARLIN_SM75_ARCHS) - file(GLOB MARLIN_TEMPLATE_SM75_KERNEL_SRC "csrc/quantization/marlin/sm75_kernel_*.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_TEMPLATE_SM75_KERNEL_SRC}" - CUDA_ARCHS "${MARLIN_SM75_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_TEMPLATE_SM75_KERNEL_SRC} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_EXT_SRC ${MARLIN_TEMPLATE_SM75_KERNEL_SRC}) - endif() - - if (MARLIN_FP8_ARCHS) - file(GLOB MARLIN_TEMPLATE_FP8_KERNEL_SRC "csrc/quantization/marlin/sm89_kernel_*.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_TEMPLATE_FP8_KERNEL_SRC}" - CUDA_ARCHS "${MARLIN_FP8_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_TEMPLATE_FP8_KERNEL_SRC} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_EXT_SRC ${MARLIN_TEMPLATE_FP8_KERNEL_SRC}) - endif() - - set(MARLIN_SRCS - "csrc/quantization/marlin/marlin.cu" - "csrc/quantization/marlin/marlin_int4_fp8_preprocess.cu" - "csrc/quantization/marlin/gptq_marlin_repack.cu" - "csrc/quantization/marlin/awq_marlin_repack.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_SRCS}" - CUDA_ARCHS "${MARLIN_OTHER_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_SRCS} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_EXT_SRC "${MARLIN_SRCS}") - - message(STATUS "Building Marlin kernels for archs: ${MARLIN_OTHER_ARCHS}") - else() - message(STATUS "Not building Marlin kernels as no compatible archs found" - " in CUDA target architectures") - endif() - # Expert-specialization MXFP8 blockscaled grouped kernels (SM100+). if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) cuda_archs_loose_intersection(ES_MXFP8_GROUPED_MM_ARCHS "10.0f;11.0f" "${CUDA_ARCHS}") @@ -676,6 +537,145 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") SRCS "${VLLM_STABLE_EXT_SRC}" CUDA_ARCHS "${CUDA_ARCHS}") + # Only build Marlin kernels if we are building for at least some compatible archs. + # Keep building Marlin for 9.0 as there are some group sizes and shapes that + # are not supported by Machete yet. + + # marlin arches for fp16 output + # Family-conditional 12.0f (one cubin for SM12x family) requires CUDA >= 13.0; + # fall back to architecture-specific 12.0a;12.1a on CUDA < 13.0 (e.g. 12.8). + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) + cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0f" "${CUDA_ARCHS}") + else() + cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}") + endif() + # marlin has limited support for turing + cuda_archs_loose_intersection(MARLIN_SM75_ARCHS "7.5" "${CUDA_ARCHS}") + # marlin arches for bf16 output (we need 9.0 for bf16 atomicAdd PTX) + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) + cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0f" "${CUDA_ARCHS}") + else() + cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}") + endif() + # marlin arches for fp8 input + # - sm80 doesn't support fp8 computation + # - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction + # so we only enable fp8 computation for SM89 (e.g. RTX 40x0) and 12.0 (e.g. RTX 50x0) + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) + cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0f" "${CUDA_ARCHS}") + else() + cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0a;12.1a" "${CUDA_ARCHS}") + endif() + # marlin arches for other files + cuda_archs_loose_intersection(MARLIN_OTHER_ARCHS "7.5;8.0+PTX" "${CUDA_ARCHS}") + + if (MARLIN_OTHER_ARCHS) + + # + # For the Marlin kernels we automatically generate sources for various + # preselected input type pairs and schedules. + # Generate sources: + set(MARLIN_GEN_SCRIPT + ${CMAKE_CURRENT_SOURCE_DIR}/csrc/libtorch_stable/quantization/marlin/generate_kernels.py) + file(MD5 ${MARLIN_GEN_SCRIPT} MARLIN_GEN_SCRIPT_HASH) + list(JOIN CUDA_ARCHS "," CUDA_ARCHS_STR) + set(MARLIN_GEN_SCRIPT_HASH_AND_ARCH "${MARLIN_GEN_SCRIPT_HASH}(ARCH:${CUDA_ARCHS_STR})") + + message(STATUS "Marlin generation script hash: ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH}") + message(STATUS "Last run Marlin generate script hash: $CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH}") + + if (NOT DEFINED CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH} + OR NOT $CACHE{MARLIN_GEN_SCRIPT_HASH_AND_ARCH} STREQUAL ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH}) + execute_process( + COMMAND ${CMAKE_COMMAND} -E env + PYTHONPATH=$ENV{PYTHONPATH} + ${Python_EXECUTABLE} ${MARLIN_GEN_SCRIPT} ${CUDA_ARCHS_STR} + RESULT_VARIABLE marlin_generation_result + OUTPUT_VARIABLE marlin_generation_result + OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log + ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log + ) + + if (NOT marlin_generation_result EQUAL 0) + message(FATAL_ERROR "Marlin generation failed." + " Result: \"${marlin_generation_result}\"" + "\nCheck the log for details: " + "${CMAKE_CURRENT_BINARY_DIR}/marlin_generation.log") + else() + set(MARLIN_GEN_SCRIPT_HASH_AND_ARCH ${MARLIN_GEN_SCRIPT_HASH_AND_ARCH} + CACHE STRING "Last run Marlin generate script hash and arch" FORCE) + message(STATUS "Marlin generation completed successfully.") + endif() + else() + message(STATUS "Marlin generation script has not changed, skipping generation.") + endif() + + if (MARLIN_ARCHS) + file(GLOB MARLIN_TEMPLATE_KERNEL_SRC "csrc/libtorch_stable/quantization/marlin/sm80_kernel_*_float16.cu") + set_gencode_flags_for_srcs( + SRCS "${MARLIN_TEMPLATE_KERNEL_SRC}" + CUDA_ARCHS "${MARLIN_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) + set_source_files_properties(${MARLIN_TEMPLATE_KERNEL_SRC} + PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") + endif() + list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_KERNEL_SRC}) + + file(GLOB MARLIN_TEMPLATE_BF16_KERNEL_SRC "csrc/libtorch_stable/quantization/marlin/sm80_kernel_*_bfloat16.cu") + set_gencode_flags_for_srcs( + SRCS "${MARLIN_TEMPLATE_BF16_KERNEL_SRC}" + CUDA_ARCHS "${MARLIN_BF16_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) + set_source_files_properties(${MARLIN_TEMPLATE_BF16_KERNEL_SRC} + PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") + endif() + list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_BF16_KERNEL_SRC}) + endif() + + if (MARLIN_SM75_ARCHS) + file(GLOB MARLIN_TEMPLATE_SM75_KERNEL_SRC "csrc/libtorch_stable/quantization/marlin/sm75_kernel_*.cu") + set_gencode_flags_for_srcs( + SRCS "${MARLIN_TEMPLATE_SM75_KERNEL_SRC}" + CUDA_ARCHS "${MARLIN_SM75_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) + set_source_files_properties(${MARLIN_TEMPLATE_SM75_KERNEL_SRC} + PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") + endif() + list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_SM75_KERNEL_SRC}) + endif() + + if (MARLIN_FP8_ARCHS) + file(GLOB MARLIN_TEMPLATE_FP8_KERNEL_SRC "csrc/libtorch_stable/quantization/marlin/sm89_kernel_*.cu") + set_gencode_flags_for_srcs( + SRCS "${MARLIN_TEMPLATE_FP8_KERNEL_SRC}" + CUDA_ARCHS "${MARLIN_FP8_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) + set_source_files_properties(${MARLIN_TEMPLATE_FP8_KERNEL_SRC} + PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") + endif() + list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_FP8_KERNEL_SRC}) + endif() + + set(MARLIN_SRCS + "csrc/libtorch_stable/quantization/marlin/marlin.cu" + "csrc/libtorch_stable/quantization/marlin/marlin_int4_fp8_preprocess.cu" + "csrc/libtorch_stable/quantization/marlin/gptq_marlin_repack.cu" + "csrc/libtorch_stable/quantization/marlin/awq_marlin_repack.cu") + set_gencode_flags_for_srcs( + SRCS "${MARLIN_SRCS}" + CUDA_ARCHS "${MARLIN_OTHER_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) + set_source_files_properties(${MARLIN_SRCS} + PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") + endif() + list(APPEND VLLM_STABLE_EXT_SRC "${MARLIN_SRCS}") + + message(STATUS "Building Marlin kernels for archs: ${MARLIN_OTHER_ARCHS}") + else() + message(STATUS "Not building Marlin kernels as no compatible archs found" + " in CUDA target architectures") + endif() + # DeepSeek V3 fused A GEMM kernel (requires SM 9.0+, Hopper and later) if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) cuda_archs_loose_intersection(DSV3_FUSED_A_GEMM_ARCHS "9.0a;10.0f;11.0f" "${CUDA_ARCHS}") diff --git a/csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h b/csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h index 09ed1a470b..783736ab50 100644 --- a/csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h +++ b/csrc/libtorch_stable/moe/marlin_moe_wna16/kernel.h @@ -3,8 +3,8 @@ #define MARLIN_NAMESPACE_NAME marlin_moe_wna16 #endif -#include "quantization/marlin/marlin.cuh" -#include "quantization/marlin/marlin_dtypes.cuh" +#include "libtorch_stable/quantization/marlin/marlin.cuh" +#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh" #include "core/scalar_type.hpp" #define MARLIN_KERNEL_PARAMS \ diff --git a/csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h b/csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h index 9858df9457..04f90101be 100644 --- a/csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h +++ b/csrc/libtorch_stable/moe/marlin_moe_wna16/marlin_template.h @@ -23,10 +23,10 @@ #define MARLIN_NAMESPACE_NAME marlin_moe_wna16 #endif -#include "quantization/marlin/marlin.cuh" -#include "quantization/marlin/marlin_dtypes.cuh" -#include "quantization/marlin/dequant.h" -#include "quantization/marlin/marlin_mma.h" +#include "libtorch_stable/quantization/marlin/marlin.cuh" +#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh" +#include "libtorch_stable/quantization/marlin/dequant.h" +#include "libtorch_stable/quantization/marlin/marlin_mma.h" #include "core/scalar_type.hpp" #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh index ce96c2d11f..ac33d5f2ce 100644 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh +++ b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh @@ -6,7 +6,7 @@ #include -#include "quantization/marlin/marlin_dtypes.cuh" +#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh" using marlin::MarlinScalarType2; namespace allspark { diff --git a/csrc/quantization/marlin/.gitignore b/csrc/libtorch_stable/quantization/marlin/.gitignore similarity index 100% rename from csrc/quantization/marlin/.gitignore rename to csrc/libtorch_stable/quantization/marlin/.gitignore diff --git a/csrc/quantization/marlin/awq_marlin_repack.cu b/csrc/libtorch_stable/quantization/marlin/awq_marlin_repack.cu similarity index 77% rename from csrc/quantization/marlin/awq_marlin_repack.cu rename to csrc/libtorch_stable/quantization/marlin/awq_marlin_repack.cu index 307bae6738..55ce5b4e73 100644 --- a/csrc/quantization/marlin/awq_marlin_repack.cu +++ b/csrc/libtorch_stable/quantization/marlin/awq_marlin_repack.cu @@ -1,6 +1,13 @@ #include "marlin.cuh" -#include "core/registration.h" +#include +#include +#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" namespace marlin { @@ -218,56 +225,55 @@ __global__ void awq_marlin_repack_kernel( b_q_weight_ptr, out_ptr, size_k, size_n); \ } -torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, - int64_t size_n, int64_t num_bits, - bool is_a_8bit) { +torch::stable::Tensor awq_marlin_repack(torch::stable::Tensor& b_q_weight, + int64_t size_k, int64_t size_n, + int64_t num_bits, bool is_a_8bit) { // Verify compatibility with marlin tile of 16x64 - TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k, - " is not divisible by tile_k_size = ", marlin::tile_k_size); - TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n, - " is not divisible by tile_n_size = ", marlin::tile_n_size); + STD_TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k, + " is not divisible by tile_k_size = ", marlin::tile_k_size); + STD_TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n, + " is not divisible by tile_n_size = ", marlin::tile_n_size); - TORCH_CHECK(num_bits == 4 || num_bits == 8, - "num_bits must be 4 or 8. Got = ", num_bits); + STD_TORCH_CHECK(num_bits == 4 || num_bits == 8, + "num_bits must be 4 or 8. Got = ", num_bits); int const pack_factor = 32 / num_bits; // Verify B - TORCH_CHECK(b_q_weight.size(0) == size_k, - "b_q_weight.size(0) = ", b_q_weight.size(0), - " is not size_k = ", size_k); - TORCH_CHECK((size_n / pack_factor) == b_q_weight.size(1), - "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1), - ", size_n = ", size_n, ", pack_factor = ", pack_factor); + STD_TORCH_CHECK(b_q_weight.size(0) == size_k, + "b_q_weight.size(0) = ", b_q_weight.size(0), + " is not size_k = ", size_k); + STD_TORCH_CHECK((size_n / pack_factor) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", b_q_weight.size(1), + ", size_n = ", size_n, ", pack_factor = ", pack_factor); // Verify device and strides - TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); - TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); - TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + STD_TORCH_CHECK(b_q_weight.is_cuda(), "b_q_weight is not on GPU"); + STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + STD_TORCH_CHECK( + b_q_weight.scalar_type() == torch::headeronly::ScalarType::Int, + "b_q_weight type is not kInt"); + + const int32_t device_index = b_q_weight.get_device_index(); + torch::stable::accelerator::DeviceGuard device_guard(device_index); + const cudaStream_t stream = get_current_cuda_stream(device_index); // Alloc buffers - const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); - auto options = torch::TensorOptions() - .dtype(b_q_weight.dtype()) - .device(b_q_weight.device()); - torch::Tensor out = torch::empty( + torch::stable::Tensor out = torch::stable::empty( {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, - options); + b_q_weight.scalar_type(), std::nullopt, b_q_weight.device()); // Get ptrs uint32_t const* b_q_weight_ptr = - reinterpret_cast(b_q_weight.data_ptr()); - uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + reinterpret_cast(b_q_weight.const_data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.mutable_data_ptr()); - // Get dev info - int dev = b_q_weight.get_device(); - cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); int blocks; - cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, device_index); int max_shared_mem = 0; cudaDeviceGetAttribute(&max_shared_mem, - cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); - TORCH_CHECK(max_shared_mem > 0); + cudaDevAttrMaxSharedMemoryPerBlockOptin, device_index); + STD_TORCH_CHECK(max_shared_mem > 0); if (false) { } @@ -276,13 +282,13 @@ torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, CALL_IF(4, true) CALL_IF(8, true) else { - TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, - ", is_a_8bit = ", is_a_8bit); + STD_TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, + ", is_a_8bit = ", is_a_8bit); } return out; } -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("awq_marlin_repack", &awq_marlin_repack); +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { + m.impl("awq_marlin_repack", TORCH_BOX(&awq_marlin_repack)); } diff --git a/csrc/quantization/marlin/dequant.h b/csrc/libtorch_stable/quantization/marlin/dequant.h similarity index 100% rename from csrc/quantization/marlin/dequant.h rename to csrc/libtorch_stable/quantization/marlin/dequant.h diff --git a/csrc/quantization/marlin/generate_kernels.py b/csrc/libtorch_stable/quantization/marlin/generate_kernels.py similarity index 99% rename from csrc/quantization/marlin/generate_kernels.py rename to csrc/libtorch_stable/quantization/marlin/generate_kernels.py index 7b316037ec..2a03847989 100644 --- a/csrc/quantization/marlin/generate_kernels.py +++ b/csrc/libtorch_stable/quantization/marlin/generate_kernels.py @@ -303,7 +303,7 @@ def generate_new_kernels(): if not SUPPORT_FP8 and kernel_selector_str != FILE_HEAD_COMMENT: kernel_selector_str += ( "else if (a_type == vllm::kFE4M3fn)\n" - " TORCH_CHECK(false, " + " STD_TORCH_CHECK(false, " '"marlin kernel with fp8 activation is not built.");' ) diff --git a/csrc/quantization/marlin/gptq_marlin_repack.cu b/csrc/libtorch_stable/quantization/marlin/gptq_marlin_repack.cu similarity index 77% rename from csrc/quantization/marlin/gptq_marlin_repack.cu rename to csrc/libtorch_stable/quantization/marlin/gptq_marlin_repack.cu index 796e6c5359..cafa212bcc 100644 --- a/csrc/quantization/marlin/gptq_marlin_repack.cu +++ b/csrc/libtorch_stable/quantization/marlin/gptq_marlin_repack.cu @@ -1,6 +1,13 @@ #include "marlin.cuh" -#include "core/registration.h" +#include +#include +#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" namespace marlin { @@ -275,64 +282,66 @@ __global__ void gptq_marlin_repack_kernel( b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \ } -torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, - int64_t size_k, int64_t size_n, - int64_t num_bits, bool is_a_8bit) { +torch::stable::Tensor gptq_marlin_repack(torch::stable::Tensor& b_q_weight, + torch::stable::Tensor& perm, + int64_t size_k, int64_t size_n, + int64_t num_bits, bool is_a_8bit) { // Verify compatibility with marlin tile of 16x64 - TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k, - " is not divisible by tile_k_size = ", marlin::tile_k_size); - TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n, - " is not divisible by tile_n_size = ", marlin::tile_n_size); + STD_TORCH_CHECK(size_k % marlin::tile_k_size == 0, "size_k = ", size_k, + " is not divisible by tile_k_size = ", marlin::tile_k_size); + STD_TORCH_CHECK(size_n % marlin::tile_n_size == 0, "size_n = ", size_n, + " is not divisible by tile_n_size = ", marlin::tile_n_size); - TORCH_CHECK(num_bits == 4 || num_bits == 8, - "num_bits must be 4 or 8. Got = ", num_bits); + STD_TORCH_CHECK(num_bits == 4 || num_bits == 8, + "num_bits must be 4 or 8. Got = ", num_bits); int const pack_factor = 32 / num_bits; // Verify B - TORCH_CHECK((size_k / pack_factor) == b_q_weight.size(0), - "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0), - ", size_k = ", size_k, ", pack_factor = ", pack_factor); - TORCH_CHECK(b_q_weight.size(1) == size_n, - "b_q_weight.size(1) = ", b_q_weight.size(1), - " is not size_n = ", size_n); + STD_TORCH_CHECK((size_k / pack_factor) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0), + ", size_k = ", size_k, ", pack_factor = ", pack_factor); + STD_TORCH_CHECK(b_q_weight.size(1) == size_n, + "b_q_weight.size(1) = ", b_q_weight.size(1), + " is not size_n = ", size_n); // Verify device and strides - TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); - TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); - TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + STD_TORCH_CHECK(b_q_weight.is_cuda(), "b_q_weight is not on GPU"); + STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + STD_TORCH_CHECK( + b_q_weight.scalar_type() == torch::headeronly::ScalarType::Int, + "b_q_weight type is not kInt"); - TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); - TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); - TORCH_CHECK(perm.dtype() == at::kInt, "perm type is not at::kInt"); + STD_TORCH_CHECK(perm.is_cuda(), "perm is not on GPU"); + STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + STD_TORCH_CHECK(perm.scalar_type() == torch::headeronly::ScalarType::Int, + "perm type is not at::kInt"); + + const int32_t device_index = b_q_weight.get_device_index(); + torch::stable::accelerator::DeviceGuard device_guard(device_index); + const cudaStream_t stream = get_current_cuda_stream(device_index); // Alloc buffers - const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); - auto options = torch::TensorOptions() - .dtype(b_q_weight.dtype()) - .device(b_q_weight.device()); - torch::Tensor out = torch::empty( + torch::stable::Tensor out = torch::stable::empty( {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, - options); + b_q_weight.scalar_type(), std::nullopt, b_q_weight.device()); // Detect if there is act_order bool has_perm = perm.size(0) != 0; // Get ptrs uint32_t const* b_q_weight_ptr = - reinterpret_cast(b_q_weight.data_ptr()); - uint32_t const* perm_ptr = reinterpret_cast(perm.data_ptr()); - uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + reinterpret_cast(b_q_weight.const_data_ptr()); + uint32_t const* perm_ptr = + reinterpret_cast(perm.const_data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.mutable_data_ptr()); - // Get dev info - int dev = b_q_weight.get_device(); - cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); int blocks; - cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, device_index); int max_shared_mem = 0; cudaDeviceGetAttribute(&max_shared_mem, - cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); - TORCH_CHECK(max_shared_mem > 0); + cudaDevAttrMaxSharedMemoryPerBlockOptin, device_index); + STD_TORCH_CHECK(max_shared_mem > 0); if (false) { } @@ -345,13 +354,13 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, CALL_IF(8, false, true) else { - TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, - ", has_perm = ", has_perm, ", is_a_8bit = ", is_a_8bit); + STD_TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, + ", has_perm = ", has_perm, ", is_a_8bit = ", is_a_8bit); } return out; } -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("gptq_marlin_repack", &gptq_marlin_repack); +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { + m.impl("gptq_marlin_repack", TORCH_BOX(&gptq_marlin_repack)); } diff --git a/csrc/quantization/marlin/kernel.h b/csrc/libtorch_stable/quantization/marlin/kernel.h similarity index 100% rename from csrc/quantization/marlin/kernel.h rename to csrc/libtorch_stable/quantization/marlin/kernel.h diff --git a/csrc/quantization/marlin/marlin.cu b/csrc/libtorch_stable/quantization/marlin/marlin.cu similarity index 61% rename from csrc/quantization/marlin/marlin.cu rename to csrc/libtorch_stable/quantization/marlin/marlin.cu index 721c206c33..63fea239e4 100644 --- a/csrc/quantization/marlin/marlin.cu +++ b/csrc/libtorch_stable/quantization/marlin/marlin.cu @@ -24,7 +24,15 @@ #endif #include "kernel.h" -#include "core/registration.h" + +#include +#include +#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ static_assert(std::is_same::value || \ @@ -46,19 +54,22 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr, } // namespace marlin -torch::Tensor marlin_gemm( - torch::Tensor& a, std::optional c_or_none, - torch::Tensor& b_q_weight, - std::optional const& b_bias_or_none, torch::Tensor& b_scales, - std::optional const& b_zeros_or_none, - std::optional const& g_idx_or_none, - std::optional const& perm_or_none, torch::Tensor& workspace, - vllm::ScalarTypeId const& b_type_id, int64_t size_m, int64_t size_n, - int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce, - bool is_zp_float) { - TORCH_CHECK_NOT_IMPLEMENTED(false, - "marlin_gemm(..) requires CUDA_ARCH >= 7.5"); - return torch::empty({1, 1}); +torch::stable::Tensor marlin_gemm( + torch::stable::Tensor& a, std::optional c_or_none, + torch::stable::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::stable::Tensor& b_scales, + std::optional const& a_scales_or_none, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::stable::Tensor& workspace, vllm::ScalarTypeId const& b_type_id, + int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full, + bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) { + STD_TORCH_CHECK_NOT_IMPLEMENTED(false, + "marlin_gemm(..) requires CUDA_ARCH >= 7.5"); + return torch::stable::empty({1, 1}); } #else @@ -323,18 +334,18 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, int thread_n_init, int sms, bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) { bool is_a_8bit = a_type.size_bits() == 8; - TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, - ", ", prob_n, ", ", prob_k, "]"); + STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", + prob_m, ", ", prob_n, ", ", prob_k, "]"); int group_blocks = 0; if (has_act_order) { if (is_k_full) { - TORCH_CHECK(group_size != -1); + STD_TORCH_CHECK(group_size != -1); group_blocks = group_size / 16; - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, - " is not divisible by group_blocks = ", group_blocks); + STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, + " is not divisible by group_blocks = ", group_blocks); } else { - TORCH_CHECK(group_size == 0); + STD_TORCH_CHECK(group_size == 0); group_blocks = 0; } } else { @@ -342,8 +353,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, group_blocks = -1; } else { group_blocks = group_size / 16; - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, - " is not divisible by group_blocks = ", group_blocks); + STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k, + " is not divisible by group_blocks = ", group_blocks); } } @@ -384,25 +395,25 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, int max_shared_mem = 0; cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); - TORCH_CHECK(max_shared_mem > 0); + STD_TORCH_CHECK(max_shared_mem > 0); int major_capability, minor_capability; cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor, dev); cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor, dev); - TORCH_CHECK(major_capability * 10 + minor_capability >= 75, - "marlin kernel only support Turing or newer GPUs."); + STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 75, + "marlin kernel only support Turing or newer GPUs."); int stages = 4; if (major_capability == 7 && minor_capability == 5) { stages = 2; - TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8, - "Turing only support FP16 or INT8 activation."); + STD_TORCH_CHECK(a_type == vllm::kFloat16 || a_type == vllm::kS8, + "Turing only support FP16 or INT8 activation."); } if (a_type == vllm::kFE4M3fn) { - TORCH_CHECK(major_capability * 10 + minor_capability >= 89, - "FP8 only support Ada Lovelace or newer GPUs."); - TORCH_CHECK( + STD_TORCH_CHECK(major_capability * 10 + minor_capability >= 89, + "FP8 only support Ada Lovelace or newer GPUs."); + STD_TORCH_CHECK( major_capability * 10 + minor_capability == 89 || major_capability == 12, "Marlin W4A8-FP8 only support SM89 or SM12x device (It is slower than " @@ -432,10 +443,10 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, if (thread_k != -1 && thread_n != -1) { thread_tfg = thread_config_t{thread_k, thread_n, default_threads}; exec_cfg = exec_config_t{1, thread_tfg}; - TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, - " is not divisible by thread_n = ", thread_n); - TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, - " is not divisible by thread_k = ", thread_k); + STD_TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, + " is not divisible by thread_n = ", thread_n); + STD_TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, + " is not divisible by thread_k = ", thread_k); } else { // Auto config exec_cfg = determine_exec_config( @@ -474,7 +485,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, int thread_k_blocks = thread_k / 16; int thread_n_blocks = thread_n / 16; - TORCH_CHECK( + STD_TORCH_CHECK( is_valid_config(thread_tfg, thread_m_blocks, prob_m_split, prob_n, prob_k, num_bits, group_size, has_act_order, is_k_full, has_zp, is_zp_float, is_a_8bit, stages, @@ -495,14 +506,15 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, num_threads, is_zp_float, stages); if (kernel == MarlinDefault) { - TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, - ", ", prob_k, "]", ", has_act_order = ", has_act_order, - ", num_groups = ", num_groups, ", group_size = ", group_size, - ", prob_m_split = ", prob_m_split, - ", thread_m_blocks = ", thread_m_blocks, - ", thread_n_blocks = ", thread_n_blocks, - ", thread_k_blocks = ", thread_k_blocks, - ", num_threads = ", num_threads, ", num_bits = ", num_bits); + STD_TORCH_CHECK( + false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ", + prob_k, "]", ", has_act_order = ", has_act_order, + ", num_groups = ", num_groups, ", group_size = ", group_size, + ", prob_m_split = ", prob_m_split, + ", thread_m_blocks = ", thread_m_blocks, + ", thread_n_blocks = ", thread_n_blocks, + ", thread_k_blocks = ", thread_k_blocks, + ", num_threads = ", num_threads, ", num_bits = ", num_bits); } cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, @@ -530,71 +542,76 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias, } // namespace marlin -torch::Tensor marlin_gemm( - torch::Tensor& a, std::optional c_or_none, - torch::Tensor& b_q_weight, - std::optional const& b_bias_or_none, torch::Tensor& b_scales, - std::optional const& a_scales_or_none, - std::optional const& global_scale_or_none, - std::optional const& b_zeros_or_none, - std::optional const& g_idx_or_none, - std::optional const& perm_or_none, torch::Tensor& workspace, - vllm::ScalarTypeId const& b_type_id, int64_t size_m, int64_t size_n, - int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce, - bool is_zp_float) { +torch::stable::Tensor marlin_gemm( + torch::stable::Tensor& a, std::optional c_or_none, + torch::stable::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::stable::Tensor& b_scales, + std::optional const& a_scales_or_none, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::stable::Tensor& workspace, vllm::ScalarTypeId const& b_type_id, + int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full, + bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) { vllm::ScalarTypeId a_type_id, c_type_id, s_type_id; - auto c_dtype = a.dtype(); - if (a.scalar_type() == at::ScalarType::Half) { + auto c_scalar_type = a.scalar_type(); + if (a.scalar_type() == torch::headeronly::ScalarType::Half) { a_type_id = vllm::kFloat16.id(); c_type_id = vllm::kFloat16.id(); - } else if (a.scalar_type() == at::ScalarType::BFloat16) { + } else if (a.scalar_type() == torch::headeronly::ScalarType::BFloat16) { a_type_id = vllm::kBFloat16.id(); c_type_id = vllm::kBFloat16.id(); } else { - c_dtype = b_scales.dtype(); - if (b_scales.scalar_type() == at::ScalarType::Half) { + c_scalar_type = b_scales.scalar_type(); + if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) { c_type_id = vllm::kFloat16.id(); - } else if (b_scales.scalar_type() == at::ScalarType::BFloat16) { + } else if (b_scales.scalar_type() == + torch::headeronly::ScalarType::BFloat16) { c_type_id = vllm::kBFloat16.id(); } else { c_type_id = vllm::kBFloat16.id(); - TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4"); - torch::Tensor c = c_or_none.value(); - c_dtype = c.dtype(); + STD_TORCH_CHECK(c_or_none.has_value(), "c must be passed for W4A8-FP4"); + torch::stable::Tensor c = c_or_none.value(); + c_scalar_type = c.scalar_type(); - if (c.scalar_type() == at::ScalarType::Half) { + if (c.scalar_type() == torch::headeronly::ScalarType::Half) { c_type_id = vllm::kFloat16.id(); - } else if (c.scalar_type() == at::ScalarType::BFloat16) { + } else if (c.scalar_type() == torch::headeronly::ScalarType::BFloat16) { c_type_id = vllm::kBFloat16.id(); } else { - TORCH_CHECK(false, "unsupported c dtype"); + STD_TORCH_CHECK(false, "unsupported c dtype"); } } - if (a.scalar_type() == at::ScalarType::Float8_e4m3fn) { + if (a.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn) { a_type_id = vllm::kFE4M3fn.id(); - } else if (a.scalar_type() == at::ScalarType::Char) { + } else if (a.scalar_type() == torch::headeronly::ScalarType::Char) { a_type_id = vllm::kS8.id(); } else { - TORCH_CHECK(false, "unsupported `a` scalar_type"); + STD_TORCH_CHECK(false, "unsupported `a` scalar_type"); } } s_type_id = c_type_id; if (b_type_id == vllm::kFE2M1f.id()) { - if (b_scales.scalar_type() == at::ScalarType::Float8_e4m3fn) { + if (b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e4m3fn) { s_type_id = vllm::kFE4M3fn.id(); - } else if (b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) { + } else if (b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e8m0fnu) { s_type_id = vllm::kFE8M0fnu.id(); } else { - TORCH_CHECK(false, - "When b_type = float4_e2m1f, b_scale scalar type must be", - "float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4)."); + STD_TORCH_CHECK( + false, "When b_type = float4_e2m1f, b_scale scalar type must be", + "float8_e4m3fn (for NVFP4) or float8_e8m0fnu (for MXFP4)."); } } else if (b_type_id == vllm::kFE4M3fn.id() && - b_scales.scalar_type() == at::ScalarType::Float8_e8m0fnu) { + b_scales.scalar_type() == + torch::headeronly::ScalarType::Float8_e8m0fnu) { s_type_id = vllm::kFE8M0fnu.id(); } @@ -606,54 +623,58 @@ torch::Tensor marlin_gemm( int pack_factor = 32 / b_type.size_bits(); // Verify A - TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), - ", size_m = ", size_m); - TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), - ", size_k = ", size_k); + STD_TORCH_CHECK(a.size(0) == size_m, + "Shape mismatch: a.size(0) = ", a.size(0), + ", size_m = ", size_m); + STD_TORCH_CHECK(a.size(1) == size_k, + "Shape mismatch: a.size(1) = ", a.size(1), + ", size_k = ", size_k); // Verify B - TORCH_CHECK( + STD_TORCH_CHECK( size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, "size_k = ", size_k, " is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); - TORCH_CHECK((size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(0), - "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0), - ", size_k = ", size_k, - ", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); - TORCH_CHECK( + STD_TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0), + ", size_k = ", size_k, + ", tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); + STD_TORCH_CHECK( b_q_weight.size(1) % MARLIN_NAMESPACE_NAME::tile_size == 0, "b_q_weight.size(1) = ", b_q_weight.size(1), " is not divisible by tile_size = ", MARLIN_NAMESPACE_NAME::tile_size); int actual_size_n = (b_q_weight.size(1) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; - TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, - ", actual_size_n = ", actual_size_n); + STD_TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, + ", actual_size_n = ", actual_size_n); // Verify device and strides - TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); - TORCH_CHECK(a.stride(1) == 1, "A.stride(1) is not 1"); + STD_TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + STD_TORCH_CHECK(a.stride(1) == 1, "A.stride(1) is not 1"); // We use int4 (16 bytes) to load A, so A must aligned to 16 bytes - TORCH_CHECK(a.stride(0) % 8 == 0, "A.stride(0) must divisible by 8"); - TORCH_CHECK(((uint64_t)a.data_ptr()) % 16 == 0, "A must aligned to 16 bytes"); + STD_TORCH_CHECK(a.stride(0) % 8 == 0, "A.stride(0) must divisible by 8"); + STD_TORCH_CHECK(((uint64_t)a.const_data_ptr()) % 16 == 0, + "A must aligned to 16 bytes"); - TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); - TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + STD_TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + STD_TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); - TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); - TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + STD_TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + STD_TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); - torch::Tensor a_scales; - auto options = torch::TensorOptions().dtype(c_dtype).device(a.device()); - auto options_fp32 = - torch::TensorOptions().dtype(at::kFloat).device(a.device()); + torch::stable::Tensor a_scales; + const auto device = a.device(); if (a_scales_or_none.has_value()) { a_scales = a_scales_or_none.value(); - TORCH_CHECK(a_type.size_bits() == 8, - "a_scales can only be used for 8bit activation."); + STD_TORCH_CHECK(a_type.size_bits() == 8, + "a_scales can only be used for 8bit activation."); } else { - a_scales = torch::empty({0}, options_fp32); - TORCH_CHECK(a_type.size_bits() != 8, - "the a_scales parameter must be passed for 8bit activation."); + a_scales = torch::stable::empty({0}, torch::headeronly::ScalarType::Float, + std::nullopt, device); + STD_TORCH_CHECK( + a_type.size_bits() != 8, + "the a_scales parameter must be passed for 8bit activation."); } // thread_k: `k` size of a thread_tile in `weights` (can usually be left as @@ -664,84 +685,93 @@ torch::Tensor marlin_gemm( int thread_n = -1; // sms: number of SMs to use for the kernel int sms = -1; - cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); + const int32_t device_index = a.get_device_index(); + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, device_index); // Alloc buffers - const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); - torch::Tensor c; + torch::stable::accelerator::DeviceGuard device_guard(device_index); + torch::stable::Tensor c; if (c_or_none.has_value()) { c = c_or_none.value(); - TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); - TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); - TORCH_CHECK(c.size(0) == size_m, "Shape mismatch: c.size(0) = ", c.size(0), - ", size_m = ", size_m); - TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), - ", size_n = ", size_n); + STD_TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + STD_TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + STD_TORCH_CHECK(c.size(0) == size_m, + "Shape mismatch: c.size(0) = ", c.size(0), + ", size_m = ", size_m); + STD_TORCH_CHECK(c.size(1) == size_n, + "Shape mismatch: c.size(1) = ", c.size(1), + ", size_n = ", size_n); } else { - c = torch::empty({size_m, size_n}, options); + c = torch::stable::empty({size_m, size_n}, c_scalar_type, std::nullopt, + device); } if (size_m == 0) return c; // Alloc C tmp buffer that is going to be used for the global reduce - torch::Tensor c_tmp; + torch::stable::Tensor c_tmp; if (use_fp32_reduce) { int max_m_block_size = (size_m + 16 - 1) / 16 * 16; max_m_block_size = min(max_m_block_size, 64); int max_c_tmp_size = sms * max_m_block_size * MARLIN_NAMESPACE_NAME::max_thread_n; - c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + c_tmp = torch::stable::empty({max_c_tmp_size}, + torch::headeronly::ScalarType::Float, + std::nullopt, device); } else { - c_tmp = torch::empty({0}, options_fp32); + c_tmp = torch::stable::empty({0}, torch::headeronly::ScalarType::Float, + std::nullopt, device); } // Detect groupsize and act_order int num_groups = -1; int group_size = -1; - int rank = b_scales.sizes().size(); - TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2"); - TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1), - " is not size_n = ", size_n); + int rank = b_scales.dim(); + STD_TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2"); + STD_TORCH_CHECK(b_scales.size(1) == size_n, + "b_scales dim 1 = ", b_scales.size(1), + " is not size_n = ", size_n); num_groups = b_scales.size(0); - torch::Tensor g_idx, perm, a_tmp; + torch::stable::Tensor g_idx, perm, a_tmp; if (g_idx_or_none.has_value() && perm_or_none.has_value()) { g_idx = g_idx_or_none.value(); perm = perm_or_none.value(); - TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); - TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); - TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); - TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); // Verify g_idx and perm - TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) || - (g_idx.size(-1) == size_k && perm.size(-1) == size_k), - "Unexpected g_idx.size(-1) = ", g_idx.size(-1), - " and perm.size(-1) = ", perm.size(-1), - ", where size_k = ", size_k); + STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) || + (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", g_idx.size(-1), + " and perm.size(-1) = ", perm.size(-1), + ", where size_k = ", size_k); } else { - g_idx = torch::empty({0}, options); - perm = torch::empty({0}, options); - a_tmp = torch::empty({0}, options); + g_idx = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); + perm = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); + a_tmp = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); } bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; if (has_act_order) { - a_tmp = torch::empty({size_m, size_k}, options); + a_tmp = torch::stable::empty({size_m, size_k}, c_scalar_type, std::nullopt, + device); if (is_k_full) { - TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); - TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, - ", is not divisible by num_groups = ", num_groups); + STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, + ", is not divisible by num_groups = ", num_groups); group_size = size_k / num_groups; } else { group_size = 0; } } else { - a_tmp = torch::empty({0}, options); + a_tmp = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); if (num_groups > 1) { - TORCH_CHECK( + STD_TORCH_CHECK( size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(0) = ", b_scales.size(0)); group_size = size_k / num_groups; @@ -750,109 +780,114 @@ torch::Tensor marlin_gemm( } } - torch::Tensor global_scale; + torch::stable::Tensor global_scale; if (global_scale_or_none.has_value()) { global_scale = global_scale_or_none.value(); - TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn, - "global_scale can only be used for nvfp4 format."); + STD_TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn, + "global_scale can only be used for nvfp4 format."); } else { - global_scale = torch::empty({0}, options_fp32); - TORCH_CHECK(!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn), - "the global_scale parameter must be passed for nvfp4 format."); + global_scale = torch::stable::empty( + {0}, torch::headeronly::ScalarType::Float, std::nullopt, device); + STD_TORCH_CHECK( + !(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn), + "the global_scale parameter must be passed for nvfp4 format."); } bool has_bias = b_bias_or_none.has_value(); - torch::Tensor b_bias; + torch::stable::Tensor b_bias; if (has_bias) { b_bias = b_bias_or_none.value(); - TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU"); - TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous"); - TORCH_CHECK(b_bias.size(0) == size_n, "b_bias.size(0) != size_n"); - TORCH_CHECK(b_bias.stride(0) == 1, "b_bias.stride(0) != 1"); + STD_TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU"); + STD_TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous"); + STD_TORCH_CHECK(b_bias.size(0) == size_n, "b_bias.size(0) != size_n"); + STD_TORCH_CHECK(b_bias.stride(0) == 1, "b_bias.stride(0) != 1"); } else { - b_bias = torch::empty({0}, options); + b_bias = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); } - torch::Tensor b_zeros; + torch::stable::Tensor b_zeros; if (b_zeros_or_none.has_value()) { b_zeros = b_zeros_or_none.value(); - TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); - TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + STD_TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + STD_TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); } else { - b_zeros = torch::empty({0}, options); + b_zeros = torch::stable::empty({0}, c_scalar_type, std::nullopt, device); } bool has_zp = b_zeros.size(-1) > 0; if (has_zp) { - TORCH_CHECK( + STD_TORCH_CHECK( b_type == vllm::kU4 || b_type == vllm::kU8, "b_type must be u4 or u8 when has_zp = True. Got = ", b_type.str()); } else { - TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 || - b_type == vllm::kS4 || b_type == vllm::kS8 || - b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f, - "b_type must be uint4b8, uint8b128, int4, int8, " - "float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ", - b_type.str()); + STD_TORCH_CHECK(b_type == vllm::kU4B8 || b_type == vllm::kU8B128 || + b_type == vllm::kS4 || b_type == vllm::kS8 || + b_type == vllm::kFE4M3fn || b_type == vllm::kFE2M1f, + "b_type must be uint4b8, uint8b128, int4, int8, " + "float8_e4m3fn or float4_e2m1f when has_zp = False. Got = ", + b_type.str()); } if (has_zp && is_zp_float) { - TORCH_CHECK(a.scalar_type() == at::ScalarType::Half, - "Computation type must be float16 (half) when using float zero " - "points."); + STD_TORCH_CHECK( + a.scalar_type() == torch::headeronly::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); } // Verify b_zeros if (has_zp) { - int rank = b_zeros.sizes().size(); - TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2"); + int rank = b_zeros.dim(); + STD_TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2"); if (is_zp_float) { - TORCH_CHECK(b_zeros.size(1) == size_n, - "b_zeros dim 1 = ", b_zeros.size(1), - " is not size_n = ", size_n); - TORCH_CHECK(num_groups == b_zeros.size(0), - "b_zeros dim 0 = ", b_zeros.size(0), - " is not num_groups = ", num_groups); - TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + STD_TORCH_CHECK(b_zeros.size(1) == size_n, + "b_zeros dim 1 = ", b_zeros.size(1), + " is not size_n = ", size_n); + STD_TORCH_CHECK(num_groups == b_zeros.size(0), + "b_zeros dim 0 = ", b_zeros.size(0), + " is not num_groups = ", num_groups); + STD_TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); } else { - TORCH_CHECK(b_zeros.size(0) == num_groups, - "b_zeros dim 0 = ", b_zeros.size(0), - " is not num_groups = ", num_groups); - TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor, - "b_zeros dim 1 = ", b_zeros.size(1), - " is not size_n / pack_factor = ", size_n / pack_factor); + STD_TORCH_CHECK(b_zeros.size(0) == num_groups, + "b_zeros dim 0 = ", b_zeros.size(0), + " is not num_groups = ", num_groups); + STD_TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor, + "b_zeros dim 1 = ", b_zeros.size(1), + " is not size_n / pack_factor = ", size_n / pack_factor); } } // Verify workspace size - TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, - "size_n = ", size_n, ", is not divisible by min_thread_n = ", - MARLIN_NAMESPACE_NAME::min_thread_n); + STD_TORCH_CHECK(size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", size_n, ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); int min_workspace_size = sms; - TORCH_CHECK(workspace.numel() >= min_workspace_size, - "workspace.numel = ", workspace.numel(), - " is below min_workspace_size = ", min_workspace_size); + STD_TORCH_CHECK(workspace.numel() >= min_workspace_size, + "workspace.numel = ", workspace.numel(), + " is below min_workspace_size = ", min_workspace_size); - int dev = a.get_device(); - - TORCH_CHECK(a_scales.scalar_type() == at::ScalarType::Float, - "scalar type of a_scales must be float"); - TORCH_CHECK(global_scale.scalar_type() == at::ScalarType::Float, - "scalar type of global_scale must be float"); + STD_TORCH_CHECK( + a_scales.scalar_type() == torch::headeronly::ScalarType::Float, + "scalar type of a_scales must be float"); + STD_TORCH_CHECK( + global_scale.scalar_type() == torch::headeronly::ScalarType::Float, + "scalar type of global_scale must be float"); if (a_type.size_bits() == 16) { - TORCH_CHECK( + STD_TORCH_CHECK( a.scalar_type() == c.scalar_type(), "scalar type of a must be the same with c for 16 bit activation"); } marlin::marlin_mm( - a.data_ptr(), b_q_weight.data_ptr(), c.data_ptr(), c_tmp.data_ptr(), - b_bias.data_ptr(), a_scales.data_ptr(), b_scales.data_ptr(), - global_scale.data_ptr(), b_zeros.data_ptr(), g_idx.data_ptr(), - perm.data_ptr(), a_tmp.data_ptr(), size_m, size_n, size_k, a.stride(0), - workspace.data_ptr(), a_type, b_type, c_type, s_type, has_bias, - has_act_order, is_k_full, has_zp, num_groups, group_size, dev, - at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, + a.const_data_ptr(), b_q_weight.const_data_ptr(), c.mutable_data_ptr(), + c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(), + a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(), + global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(), + g_idx.mutable_data_ptr(), perm.mutable_data_ptr(), + a_tmp.mutable_data_ptr(), size_m, size_n, size_k, a.stride(0), + workspace.mutable_data_ptr(), a_type, b_type, c_type, s_type, has_bias, + has_act_order, is_k_full, has_zp, num_groups, group_size, device_index, + get_current_cuda_stream(device_index), thread_k, thread_n, sms, use_atomic_add, use_fp32_reduce, is_zp_float); return c; @@ -860,6 +895,6 @@ torch::Tensor marlin_gemm( #endif -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("marlin_gemm", &marlin_gemm); +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { + m.impl("marlin_gemm", TORCH_BOX(&marlin_gemm)); } diff --git a/csrc/quantization/marlin/marlin.cuh b/csrc/libtorch_stable/quantization/marlin/marlin.cuh similarity index 93% rename from csrc/quantization/marlin/marlin.cuh rename to csrc/libtorch_stable/quantization/marlin/marlin.cuh index d3a9156834..bfb65e874b 100644 --- a/csrc/quantization/marlin/marlin.cuh +++ b/csrc/libtorch_stable/quantization/marlin/marlin.cuh @@ -2,14 +2,6 @@ #ifndef _marlin_cuh #define _marlin_cuh - // These torch headers are only needed by non-stable callers (e.g. ops.cu). - // Guard them so that stable ABI targets can still include marlin.cuh - // for Vec, constants, and cp_async helpers without pulling in torch/all.h. - #ifndef TORCH_TARGET_VERSION - #include - #include - #include - #endif #include #include #include diff --git a/csrc/quantization/marlin/marlin_dtypes.cuh b/csrc/libtorch_stable/quantization/marlin/marlin_dtypes.cuh similarity index 100% rename from csrc/quantization/marlin/marlin_dtypes.cuh rename to csrc/libtorch_stable/quantization/marlin/marlin_dtypes.cuh diff --git a/csrc/libtorch_stable/quantization/marlin/marlin_int4_fp8_preprocess.cu b/csrc/libtorch_stable/quantization/marlin/marlin_int4_fp8_preprocess.cu new file mode 100644 index 0000000000..f8ef6b12a0 --- /dev/null +++ b/csrc/libtorch_stable/quantization/marlin/marlin_int4_fp8_preprocess.cu @@ -0,0 +1,118 @@ + +#include "marlin.cuh" + +#include +#include +#include +#include +#include +#include + +#include "libtorch_stable/torch_utils.h" + +// for only non-zp format (like gptq) +__global__ void marlin_int4_fp8_preprocess_kernel_without_zp( + // qweight: (size_k * size_n // 8,) + const int32_t* __restrict__ qweight, + // output: same shape with qweight + int32_t* __restrict__ output) { + int32_t val = qweight[blockIdx.x * 32 + threadIdx.x]; + int32_t new_val = 0; + +#pragma unroll + for (int32_t i = 0; i < 8; i++) { + int32_t single_val = val & 0xF; + single_val = single_val >= 8 ? single_val - 8 : 15 - single_val; + new_val |= single_val << (i * 4); + val >>= 4; + } + + output[blockIdx.x * 32 + threadIdx.x] = new_val; +} + +// for awq format only (with zp and with awq weight layout) +__global__ void marlin_int4_fp8_preprocess_kernel_awq( + // AWQ qweight: (size_k, size_n // 8) + const int32_t* __restrict__ qweight, + // output: same shape with qweight + int32_t* __restrict__ output, + // AWQ zeros: (size_k // group_size, size_n // 8) + const int32_t* __restrict__ qzeros, int32_t size_n, int32_t size_k, + int32_t group_size) { + int32_t val = + qweight[(blockIdx.x * 32 + threadIdx.x) * size_n / 8 + blockIdx.y]; + int32_t zero = + qzeros[(blockIdx.x * 32 + threadIdx.x) / group_size * size_n / 8 + + blockIdx.y]; + int32_t new_val = 0; + +#pragma unroll + for (int32_t i = 0; i < 8; i++) { + int32_t single_val = val & 0xF; + int32_t single_zero = zero & 0xF; + + single_val = + single_val >= single_zero ? single_val - single_zero : 15 - single_val; + new_val |= single_val << (i * 4); + val >>= 4; + zero >>= 4; + } + + output[(blockIdx.x * 32 + threadIdx.x) * size_n / 8 + blockIdx.y] = new_val; +} + +torch::stable::Tensor marlin_int4_fp8_preprocess( + torch::stable::Tensor& qweight, + std::optional qzeros_or_none, bool inplace) { + STD_TORCH_CHECK(qweight.is_cuda(), "qweight is not on GPU"); + STD_TORCH_CHECK(qweight.scalar_type() == torch::headeronly::ScalarType::Int, + "qweight.dtype != torch.int32"); + + const int32_t device_index = qweight.get_device_index(); + torch::stable::accelerator::DeviceGuard device_guard(device_index); + const cudaStream_t stream = get_current_cuda_stream(device_index); + + torch::stable::Tensor output = + inplace ? qweight : torch::stable::empty_like(qweight); + + if (!qzeros_or_none.has_value()) { + STD_TORCH_CHECK(qweight.numel() * 8 % 256 == 0, + "qweight.numel() * 8 % 256 != 0"); + + int blocks = qweight.numel() * 8 / 256; + marlin_int4_fp8_preprocess_kernel_without_zp<<>>( + reinterpret_cast(qweight.const_data_ptr()), + reinterpret_cast(output.mutable_data_ptr())); + } else { + int32_t size_k = qweight.size(0); + int32_t size_n = qweight.size(1) * 8; + torch::stable::Tensor qzeros = qzeros_or_none.value(); + + STD_TORCH_CHECK(size_k % 32 == 0, "size_k % 32 != 0"); + STD_TORCH_CHECK(qzeros.is_cuda(), "qzeros is not on GPU"); + STD_TORCH_CHECK(qzeros.scalar_type() == torch::headeronly::ScalarType::Int, + "qweight.dtype != torch.int32"); + STD_TORCH_CHECK(qzeros.get_device_index() == device_index, + "qzeros is not on the same device with qweight"); + + int32_t group_size = qweight.size(0) / qzeros.size(0); + STD_TORCH_CHECK(qweight.size(1) == qzeros.size(1), + "qweight.size(1) != qzeros.size(1)"); + STD_TORCH_CHECK(qweight.size(0) % qzeros.size(0) == 0, + "qweight.size(0) % qzeros.size(0) != 0"); + STD_TORCH_CHECK(group_size % 8 == 0, "group_size % 8 != 0"); + + dim3 blocks(size_k / 32, size_n / 8); + marlin_int4_fp8_preprocess_kernel_awq<<>>( + reinterpret_cast(qweight.const_data_ptr()), + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast(qzeros.const_data_ptr()), size_n, + size_k, group_size); + } + + return output; +} + +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { + m.impl("marlin_int4_fp8_preprocess", TORCH_BOX(&marlin_int4_fp8_preprocess)); +} diff --git a/csrc/quantization/marlin/marlin_mma.h b/csrc/libtorch_stable/quantization/marlin/marlin_mma.h similarity index 100% rename from csrc/quantization/marlin/marlin_mma.h rename to csrc/libtorch_stable/quantization/marlin/marlin_mma.h diff --git a/csrc/quantization/marlin/marlin_template.h b/csrc/libtorch_stable/quantization/marlin/marlin_template.h similarity index 100% rename from csrc/quantization/marlin/marlin_template.h rename to csrc/libtorch_stable/quantization/marlin/marlin_template.h diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 816f266504..204feed4a2 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -33,6 +33,35 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // TODO: Remove this once ROCm upgrade to torch 2.11. ops.def("get_cuda_view_from_cpu_tensor(Tensor cpu_tensor) -> Tensor"); + + // Marlin GEMM + ops.def( + "marlin_gemm(Tensor a, Tensor? c_or_none, Tensor b_q_weight, " + "Tensor? b_bias_or_none,Tensor b_scales, " + "Tensor? a_scales, Tensor? global_scale, Tensor? b_zeros_or_none, " + "Tensor? " + "g_idx_or_none, Tensor? perm_or_none, Tensor workspace, int b_type_id, " + "SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, " + "bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) -> Tensor"); + // conditionally compiled so impl registrations are in source file + + // gptq_marlin repack from GPTQ. + ops.def( + "gptq_marlin_repack(Tensor b_q_weight, Tensor perm, " + "SymInt size_k, SymInt size_n, int num_bits, bool is_a_8bit) -> Tensor"); + // conditionally compiled so impl registrations are in source file + + // awq_marlin repack from AWQ. + ops.def( + "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, " + "SymInt size_n, int num_bits, bool is_a_8bit) -> Tensor"); + // conditionally compiled so impl registrations are in source file + + // preprocess W-int4A-fp8 weight for marlin kernel + ops.def( + "marlin_int4_fp8_preprocess(Tensor qweight, " + "Tensor? qzeros_or_none, bool inplace) -> Tensor"); + // conditionally compiled so impl registrations are in source file #endif #ifndef USE_ROCM diff --git a/csrc/quantization/marlin/marlin_int4_fp8_preprocess.cu b/csrc/quantization/marlin/marlin_int4_fp8_preprocess.cu deleted file mode 100644 index 7d4c97fb57..0000000000 --- a/csrc/quantization/marlin/marlin_int4_fp8_preprocess.cu +++ /dev/null @@ -1,106 +0,0 @@ - - -#include "marlin.cuh" - -#include "core/registration.h" - -// for only non-zp format (like gptq) -__global__ void marlin_int4_fp8_preprocess_kernel_without_zp( - // qweight: (size_k * size_n // 8,) - const int32_t* __restrict__ qweight, - // output: same shape with qweight - int32_t* __restrict__ output) { - int32_t val = qweight[blockIdx.x * 32 + threadIdx.x]; - int32_t new_val = 0; - -#pragma unroll - for (int32_t i = 0; i < 8; i++) { - int32_t single_val = val & 0xF; - single_val = single_val >= 8 ? single_val - 8 : 15 - single_val; - new_val |= single_val << (i * 4); - val >>= 4; - } - - output[blockIdx.x * 32 + threadIdx.x] = new_val; -} - -// for awq format only (with zp and with awq weight layout) -__global__ void marlin_int4_fp8_preprocess_kernel_awq( - // AWQ qweight: (size_k, size_n // 8) - const int32_t* __restrict__ qweight, - // output: same shape with qweight - int32_t* __restrict__ output, - // AWQ zeros: (size_k // group_size, size_n // 8) - const int32_t* __restrict__ qzeros, int32_t size_n, int32_t size_k, - int32_t group_size) { - int32_t val = - qweight[(blockIdx.x * 32 + threadIdx.x) * size_n / 8 + blockIdx.y]; - int32_t zero = - qzeros[(blockIdx.x * 32 + threadIdx.x) / group_size * size_n / 8 + - blockIdx.y]; - int32_t new_val = 0; - -#pragma unroll - for (int32_t i = 0; i < 8; i++) { - int32_t single_val = val & 0xF; - int32_t single_zero = zero & 0xF; - - single_val = - single_val >= single_zero ? single_val - single_zero : 15 - single_val; - new_val |= single_val << (i * 4); - val >>= 4; - zero >>= 4; - } - - output[(blockIdx.x * 32 + threadIdx.x) * size_n / 8 + blockIdx.y] = new_val; -} - -torch::Tensor marlin_int4_fp8_preprocess( - torch::Tensor& qweight, std::optional qzeros_or_none, - bool inplace) { - TORCH_CHECK(qweight.device().is_cuda(), "qweight is not on GPU"); - TORCH_CHECK(qweight.scalar_type() == at::ScalarType::Int, - "qweight.dtype != torch.int32"); - - const at::cuda::OptionalCUDAGuard device_guard(device_of(qweight)); - - torch::Tensor output = inplace ? qweight : torch::empty_like(qweight); - - if (!qzeros_or_none.has_value()) { - TORCH_CHECK(qweight.numel() * 8 % 256 == 0, - "qweight.numel() * 8 % 256 != 0"); - - int blocks = qweight.numel() * 8 / 256; - marlin_int4_fp8_preprocess_kernel_without_zp<<>>( - (const int32_t*)qweight.data_ptr(), (int32_t*)output.data_ptr()); - } else { - int32_t size_k = qweight.size(0); - int32_t size_n = qweight.size(1) * 8; - torch::Tensor qzeros = qzeros_or_none.value(); - - TORCH_CHECK(size_k % 32 == 0, "size_k % 32 != 0"); - TORCH_CHECK(qzeros.device().is_cuda(), "qzeros is not on GPU"); - TORCH_CHECK(qzeros.scalar_type() == at::ScalarType::Int, - "qweight.dtype != torch.int32"); - TORCH_CHECK(device_of(qweight) == device_of(qzeros), - "qzeros is not on the same device with qweight"); - - int32_t group_size = qweight.size(0) / qzeros.size(0); - TORCH_CHECK(qweight.size(1) == qzeros.size(1), - "qweight.size(1) != qzeros.size(1)"); - TORCH_CHECK(qweight.size(0) % qzeros.size(0) == 0, - "qweight.size(0) % qzeros.size(0) != 0"); - TORCH_CHECK(group_size % 8 == 0, "group_size % 8 != 0"); - - dim3 blocks(size_k / 32, size_n / 8); - marlin_int4_fp8_preprocess_kernel_awq<<>>( - (const int32_t*)qweight.data_ptr(), (int32_t*)output.data_ptr(), - (const int32_t*)qzeros.data_ptr(), size_n, size_k, group_size); - } - - return output; -} - -TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) { - m.impl("marlin_int4_fp8_preprocess", &marlin_int4_fp8_preprocess); -} diff --git a/csrc/torch_bindings.cpp b/csrc/torch_bindings.cpp index 58524c4c5d..941e4a61c1 100644 --- a/csrc/torch_bindings.cpp +++ b/csrc/torch_bindings.cpp @@ -101,35 +101,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ") -> Tensor"); // conditionally compiled so impl registration is in source file - // Marlin Optimized Quantized GEMM (supports GPTQ, AWQ, FP8, NVFP4, MXFP4). - ops.def( - "marlin_gemm(Tensor a, Tensor? c_or_none, Tensor b_q_weight, " - "Tensor? b_bias_or_none,Tensor b_scales, " - "Tensor? a_scales, Tensor? global_scale, Tensor? b_zeros_or_none, " - "Tensor? " - "g_idx_or_none, Tensor? perm_or_none, Tensor workspace, int b_type_id, " - "SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, " - "bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) -> Tensor"); - // conditionally compiled so impl registration is in source file - - // gptq_marlin repack from GPTQ. - ops.def( - "gptq_marlin_repack(Tensor b_q_weight, Tensor perm, " - "SymInt size_k, SymInt size_n, int num_bits, bool is_a_8bit) -> Tensor"); - // conditionally compiled so impl registrations are in source file - - // awq_marlin repack from AWQ. - ops.def( - "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, " - "SymInt size_n, int num_bits, bool is_a_8bit) -> Tensor"); - // conditionally compiled so impl registrations are in source file - - // preprocess W-int4A-fp8 weight for marlin kernel - ops.def( - "marlin_int4_fp8_preprocess(Tensor qweight, " - "Tensor? qzeros_or_none, bool inplace) -> Tensor"); - // conditionally compiled so impl registrations are in source file - #endif }