diff --git a/CMakeLists.txt b/CMakeLists.txt index 529ba55c51..d956e29e39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,6 +629,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/libtorch_stable/quantization/w8a8/fp8/common.cu" "csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu" "csrc/libtorch_stable/quantization/w8a8/int8/per_token_group_quant.cu" + "csrc/libtorch_stable/permute_cols.cu" "csrc/libtorch_stable/quantization/gptq/q_gemm.cu" "csrc/libtorch_stable/quantization/gguf/gguf_kernel.cu" "csrc/libtorch_stable/pos_encoding_kernels.cu" @@ -657,9 +658,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_entry.cu" "csrc/libtorch_stable/quantization/fp4/nvfp4_quant_entry.cu" "csrc/libtorch_stable/quantization/fp4/nvfp4_scaled_mm_entry.cu" - "csrc/libtorch_stable/permute_cols.cu" - "csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu" - "csrc/libtorch_stable/quantization/w8a8/int8/per_token_group_quant.cu" "csrc/libtorch_stable/quantization/awq/gemm_kernels.cu" "csrc/libtorch_stable/minimax_reduce_rms_kernel.cu") diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 3571726f61..6ebec95449 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -24,10 +24,10 @@ void per_token_group_quant_int8(const torch::stable::Tensor& input, int64_t group_size, double eps, double int8_min, double int8_max); -#ifndef USE_ROCM torch::stable::Tensor permute_cols(torch::stable::Tensor const& A, torch::stable::Tensor const& perm); +#ifndef USE_ROCM bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability); bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability); bool cutlass_group_gemm_supported(int64_t cuda_device_capability); diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 6c699ffd98..e7d1b3669f 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -27,13 +27,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "per_token_group_quant_int8(Tensor input, Tensor! output_q, Tensor! " "output_s, int group_size, float eps, float int8_min, float int8_max) -> " "()"); + ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor"); #ifndef USE_ROCM // TODO: Remove this once ROCm upgrade to torch 2.11. ops.def("get_cuda_view_from_cpu_tensor(Tensor cpu_tensor) -> Tensor"); - - ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor"); #endif #ifndef USE_ROCM @@ -562,9 +561,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { ops.impl("per_token_group_quant_int8", TORCH_BOX(&per_token_group_quant_int8)); -#ifndef USE_ROCM ops.impl("permute_cols", TORCH_BOX(&permute_cols)); -#endif #ifndef USE_ROCM // CUTLASS scaled_mm ops