From a5e0045433236d3b3c717ac2876578b4ed20b281 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:28:44 -0400 Subject: [PATCH] [Refactor] Remove dead code quantization 2 (#52821) Signed-off-by: yewentao256 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../quantization/gptq/matrix_view.cuh | 34 --- .../quantization/gptq/q_gemm.cu | 63 ----- .../quantization/gptq/qdq_4.cuh | 42 --- .../quantization/gptq/qdq_util.cuh | 9 - .../gptq_allspark/allspark_utils.cuh | 22 -- .../machete/machete_prepacked_layout.cuh | 8 - .../w8a8/cutlass/c3x/scaled_mm.cuh | 61 ---- .../w8a8/fp8/nvidia/quant_utils.cuh | 260 ------------------ .../schemes/compressed_tensors_w8a8_fp8.py | 4 - .../model_executor/layers/quantization/fp8.py | 2 - .../quantization/inc/schemes/inc_scheme.py | 16 -- .../layers/quantization/quark/quark_moe.py | 2 - .../quark/schemes/quark_ocp_mx.py | 2 - .../layers/quantization/utils/w8a8_utils.py | 1 - 14 files changed, 526 deletions(-) diff --git a/csrc/libtorch_stable/quantization/gptq/matrix_view.cuh b/csrc/libtorch_stable/quantization/gptq/matrix_view.cuh index 2b6719fbdc..3d82e43931 100644 --- a/csrc/libtorch_stable/quantization/gptq/matrix_view.cuh +++ b/csrc/libtorch_stable/quantization/gptq/matrix_view.cuh @@ -27,12 +27,6 @@ class MatrixView_half { __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } - __device__ __forceinline__ half2 item_half2(int row, int column) const { - return ((half2*)data)[(row * width + column) / 2]; - } - __device__ __forceinline__ half2 item_half2half2(int row, int column) const { - return __half2half2(data[row * width + column]); - } __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; } @@ -83,9 +77,6 @@ class MatrixView_half_rw { __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } - __device__ __forceinline__ half2 item_half2(int row, int column) const { - return ((half2*)data)[(row * width + column) / 2]; - } __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; } @@ -141,31 +132,6 @@ class MatrixView_q4_row { } }; -class MatrixView_q4_column { - public: - const uint32_t* data; - const int height; - const int width; - - __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, - const int height, - const int width) - : data(data), height(height), width(width) {} - - __device__ __forceinline__ int item(int row, int column) const { - int shift = (row & 0x07) * 4; - return (data[row / 8 * width + column] >> shift) & 0x0f; - } - - __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { - return data[row / 8 * width + column]; - } - __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, - int column) { - return &data[row / 8 * width + column]; - } -}; - class MatrixView_q2_row { public: const uint32_t* data; diff --git a/csrc/libtorch_stable/quantization/gptq/q_gemm.cu b/csrc/libtorch_stable/quantization/gptq/q_gemm.cu index e3f79c5a6b..af0b9837b7 100644 --- a/csrc/libtorch_stable/quantization/gptq/q_gemm.cu +++ b/csrc/libtorch_stable/quantization/gptq/q_gemm.cu @@ -52,15 +52,6 @@ __host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm( #define rocblas_hgemm __compat_hipblasHgemm #endif -__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, - const half2 g_result) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); - return __hadd2(result, g_result); -} - __forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) { half2 result = {}; const half2* a2_ptr = (const half2*)a_ptr; @@ -69,36 +60,6 @@ __forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) { return __half2float(__low2half(result)) + __half2float(__high2half(result)); } -__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, - const half2 g_result, - const half qs_h) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); -} - -__forceinline__ __device__ half2 dot22_16(half2 (&dq)[8], const half* a_ptr, - const half2 g_result, - const half qs_h) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result); - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); -} - -__forceinline__ __device__ half2 dot22_32(half2 (&dq)[16], const half* a_ptr, - const half2 g_result, - const half qs_h) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result); - return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); -} - __forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr, const float g_result, const float qs_f) { @@ -111,30 +72,6 @@ __forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr, return fma(result_f, qs_f, g_result); } -__forceinline__ __device__ float dot22_16_f(half2 (&dq)[8], const half* a_ptr, - const float g_result, - const float qs_f) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result); - float result_f = - __half2float(__low2half(result)) + __half2float(__high2half(result)); - return fma(result_f, qs_f, g_result); -} - -__forceinline__ __device__ float dot22_32_f(half2 (&dq)[16], const half* a_ptr, - const float g_result, - const float qs_f) { - half2 result = {}; - const half2* a2_ptr = (const half2*)a_ptr; -#pragma unroll - for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result); - float result_f = - __half2float(__low2half(result)) + __half2float(__high2half(result)); - return fma(result_f, qs_f, g_result); -} - __forceinline__ __device__ half dot22_8_h(half2 (&dq)[4], const half* a_ptr, const half g_result, const half qs_h) { diff --git a/csrc/libtorch_stable/quantization/gptq/qdq_4.cuh b/csrc/libtorch_stable/quantization/gptq/qdq_4.cuh index 7f65d2d281..b482cb9744 100644 --- a/csrc/libtorch_stable/quantization/gptq/qdq_4.cuh +++ b/csrc/libtorch_stable/quantization/gptq/qdq_4.cuh @@ -28,48 +28,6 @@ __forceinline__ __device__ void shuffle_4bit_8(uint32_t* q, int stride) { q[0] = qb; } -__forceinline__ __device__ void dequant_4bit_8(const uint32_t q_0, - half2 (&dq)[4], int stride, - const uint32_t zero) { - const uint32_t c0 = 0x64006400; - const half y16_ = __float2half_rn(1.0f / 16.0f); - const half2 y16 = __halves2half2(y16_, y16_); - const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); - const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero)); - const half2 z1 = __half2half2(z1_.as_half); - const half2 z16 = __half2half2(z16_); - - uint32_t qa = q_0; - half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024 - half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024 - qa >>= 8; - half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024 - half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024 - - dq[0] = __hadd2(q0.as_half2, z1); - dq[1] = __hfma2(q1.as_half2, y16, z16); - dq[2] = __hadd2(q2.as_half2, z1); - dq[3] = __hfma2(q3.as_half2, y16, z16); -} - -__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale( - const uint32_t zero, const half scale, half2 (&z1z16)[2], - half2 (&y1y16)[2]) { - half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); - half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); - - half2 scale2 = __half2half2(scale); - - z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half)); - z1z16[1] = __hmul2(scale2, __half2half2(z16)); - - const half y1 = __float2half_rn(1.0f); - const half y16 = __float2half_rn(1.0f / 16.0f); - - y1y16[0] = __hmul2(scale2, __half2half2(y1)); - y1y16[1] = __hmul2(scale2, __half2half2(y16)); -} - __forceinline__ __device__ void dequant_4bit_8_prep_zero(const uint32_t zero, half2 (&z1z16)[2], half2 (&y1y16)[2]) { diff --git a/csrc/libtorch_stable/quantization/gptq/qdq_util.cuh b/csrc/libtorch_stable/quantization/gptq/qdq_util.cuh index 9426408fec..39f4583774 100644 --- a/csrc/libtorch_stable/quantization/gptq/qdq_util.cuh +++ b/csrc/libtorch_stable/quantization/gptq/qdq_util.cuh @@ -22,15 +22,6 @@ union half_uint16 { __device__ half_uint16(half val) : as_half(val) {} }; -// Max_scale premultiplied by 1/256 - -__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) { - int qs_i = qs + 1; - half qs_h = __int2half_rn(qs_i * qs_i); - qs_h = __hmul(qs_h, max_scale); - return qs_h; -} - __forceinline__ __device__ half dq(const int q, const int qzero, const half scale) { return __hmul(__int2half_rn(q - qzero), scale); diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh index ac33d5f2ce..628f6551ea 100644 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh +++ b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_utils.cuh @@ -306,28 +306,6 @@ __device__ __forceinline__ void cp_async(const uint32_t smem_addr, #endif } -template -__device__ __forceinline__ void cp_async_ca(const uint32_t smem_addr, - const void* gmem_ptr, - const int src_in_bytes, - bool guard) { - static_assert( - (SIZE_IN_BYTES == 4 || SIZE_IN_BYTES == 8 || SIZE_IN_BYTES == 16), - "Size is not supported"); -#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800 - asm volatile( - "{.reg.pred p;\n" - " setp.ne.b32 p, %4, 0;\n" - #if __CUDACC_VER_MINOR__ >= 4 - " @p cp.async.ca.shared.global.L2::256B [%0], [%1], %2, %3;}\n" - #else - " @p cp.async.ca.shared.global [%0], [%1], %2, %3;}\n" - #endif - ::"r"(smem_addr), - "l"(gmem_ptr), "n"(SIZE_IN_BYTES), "r"(src_in_bytes), "r"((int)guard)); -#endif -} - __device__ __forceinline__ void cp_async_commit_group() { #if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800 asm volatile("cp.async.commit_group;\n"); diff --git a/csrc/libtorch_stable/quantization/machete/machete_prepacked_layout.cuh b/csrc/libtorch_stable/quantization/machete/machete_prepacked_layout.cuh index c16a2ab8a3..9c20bc358f 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_prepacked_layout.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_prepacked_layout.cuh @@ -236,14 +236,6 @@ struct PrepackedLayoutBTemplate { auto tiled_A = zipped_divide(make_layout(shape_mkl), tile); return tiled_A.compose(ppblock_TV_to_NK(), _); } - - // (N, K, L) -> ((athrid, val), (BlocksN, BlocksK), L) - template - CUTE_HOST_DEVICE static auto NKL_to_TVbNbK(Shape_NKL shape_mkl) { - auto TVbNbK_to_NKL_layout = TVbNbK_to_NKL(shape_mkl); - return blocked_product(ppblock_NK_to_TV(), - make_layout(shape<1>(TVbNbK_to_NKL_layout))); - } }; }; // namespace machete diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm.cuh index 7b7d4d7147..cf18b63a88 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm.cuh @@ -84,67 +84,6 @@ struct cutlass_3x_gemm { struct GemmKernel : public KernelType {}; }; -template typename Epilogue_, - typename TileShape, typename ClusterShape, typename KernelSchedule, - typename EpilogueSchedule> -struct cutlass_3x_gemm_sm100 { - using ElementAB = ElementAB_; - using LayoutA = cutlass::layout::RowMajor; - static constexpr int AlignmentA = - 128 / cutlass::sizeof_bits::value; - - using LayoutB = cutlass::layout::ColumnMajor; - static constexpr int AlignmentB = - 128 / cutlass::sizeof_bits::value; - - using ElementC = void; - using LayoutC = cutlass::layout::RowMajor; - static constexpr int AlignmentC = - 128 / cutlass::sizeof_bits::value; - - using ElementD = ElementD_; - using LayoutD = cutlass::layout::RowMajor; - static constexpr int AlignmentD = AlignmentC; - - using ElementAcc = - typename std::conditional, int32_t, - float>::type; - using Epilogue = Epilogue_; - - // MMA type - using ElementAccumulator = float; - - // Epilogue types - using ElementBias = cutlass::half_t; - using ElementCompute = float; - using ElementAux = ElementD; - using LayoutAux = LayoutD; - using ElementAmax = float; - - using EVTCompute = typename Epilogue::EVTCompute; - - using CollectiveEpilogue = - typename cutlass::epilogue::collective::CollectiveBuilder< - cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, TileShape, - ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto, - ElementAccumulator, ElementCompute, ElementC, LayoutC, AlignmentC, - ElementD, LayoutD, AlignmentD, EpilogueSchedule, - EVTCompute>::CollectiveOp; - - using CollectiveMainloop = - typename cutlass::gemm::collective::CollectiveBuilder< - cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, ElementAB, - LayoutA, AlignmentA, ElementAB, LayoutB, AlignmentB, - ElementAccumulator, TileShape, ClusterShape, - cutlass::gemm::collective::StageCountAutoCarveout( - sizeof(typename CollectiveEpilogue::SharedStorage))>, - KernelSchedule>::CollectiveOp; - - using GemmKernel = enable_sm100_to_sm120, CollectiveMainloop, CollectiveEpilogue, void>>; -}; - template typename Epilogue_, typename TileShape, typename ClusterShape, typename KernelSchedule, diff --git a/csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh b/csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh index 296978f384..11a9b0818d 100644 --- a/csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh +++ b/csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh @@ -40,266 +40,6 @@ vec_conversion( #endif } - #if 0 // Disable the following code to reduce the binary size. -// fp8 -> half -template <> -__inline__ __device__ uint16_t vec_conversion( - const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) { - __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type); - return res.x; -} - -// fp8x2 -> half2 -template <> -__inline__ __device__ uint32_t vec_conversion( - const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) { - union { - uint16_t u16[2]; - uint32_t u32; - } tmp; - __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, fp8_type); - tmp.u16[0] = res.x; - tmp.u16[1] = res.y; - return tmp.u32; -} - -// fp8x4 -> half2x2 -template <> -__inline__ __device__ uint2 vec_conversion( - const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) { - union { - uint2 u32x2; - uint32_t u32[2]; - } tmp; - tmp.u32[0] = vec_conversion((uint16_t)a, fp8_type); - tmp.u32[1] = - vec_conversion((uint16_t)(a >> 16U), fp8_type); - return tmp.u32x2; -} - -// fp8x8 -> half2x4 -template <> -__inline__ __device__ uint4 vec_conversion( - const uint2 &a, const __nv_fp8_interpretation_t fp8_type) { - union { - uint4 u64x2; - uint2 u64[2]; - } tmp; - tmp.u64[0] = vec_conversion(a.x, fp8_type); - tmp.u64[1] = vec_conversion(a.y, fp8_type); - return tmp.u64x2; -} - -// fp8 -> __nv_bfloat16 -template <> -__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>( - const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) { - // Note there is no direct convert function from fp8 to bf16. - // fp8 -> half - __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type); - // half -> float -> bf16 - float tmp = half_to_float(res.x); - return __float2bfloat16(tmp); -} - -// fp8x2 -> __nv_bfloat162 -template <> -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>( - const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) { - __nv_bfloat162 res; - res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, fp8_type); - res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), fp8_type); - return res; -} - -// fp8x4 -> bf16_4_t -template <> -__inline__ __device__ bf16_4_t vec_conversion( - const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) { - bf16_4_t res; - res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, fp8_type); - res.y = - vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), fp8_type); - return res; -} - -// fp8x8 -> bf16_8_t -template <> -__inline__ __device__ bf16_8_t vec_conversion( - const uint2 &a, const __nv_fp8_interpretation_t fp8_type) { - bf16_4_t tmp1, tmp2; - tmp1 = vec_conversion(a.x, fp8_type); - tmp2 = vec_conversion(a.y, fp8_type); - bf16_8_t res; - res.x = tmp1.x; - res.y = tmp1.y; - res.z = tmp2.x; - res.w = tmp2.y; - return res; -} - -// fp8 -> float -template <> -__inline__ __device__ float -vec_conversion(const uint8_t &a, - const __nv_fp8_interpretation_t fp8_type) { - // fp8 -> half - uint16_t tmp = vec_conversion(a, fp8_type); - // half -> float - return half_to_float(tmp); -} - -// fp8x2 -> float2 -template <> -__inline__ __device__ float2 vec_conversion( - const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) { - // fp8x2 -> half2 - uint32_t tmp = vec_conversion(a, fp8_type); - // half2 -> float2 - return half2_to_float2(tmp); -} - -// fp8x4 -> float4 -template <> -__inline__ __device__ Float4_ vec_conversion( - const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) { - Float4_ res; - res.x = vec_conversion((uint16_t)a, fp8_type); - res.y = vec_conversion((uint16_t)(a >> 16U), fp8_type); - return res; -} - -// fp8x8 -> float8 -template <> -__inline__ __device__ Float8_ vec_conversion( - const uint2 &a, const __nv_fp8_interpretation_t fp8_type) { - Float4_ tmp1, tmp2; - tmp1 = vec_conversion(a.x, fp8_type); - tmp2 = vec_conversion(a.y, fp8_type); - Float8_ res; - res.x = tmp1.x; - res.y = tmp1.y; - res.z = tmp2.x; - res.w = tmp2.y; - return res; -} - -// half -> fp8 -template <> -__inline__ __device__ uint8_t vec_conversion( - const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) { - __half_raw tmp; - tmp.x = a; - __nv_fp8_storage_t res = - __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, fp8_type); - return (uint8_t)res; -} - -// bf16 -> fp8 -template <> -__inline__ __device__ uint8_t vec_conversion( - const __nv_bfloat16 &a, const __nv_fp8_interpretation_t fp8_type) { - #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 - assert(false); - #else - __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8( - __nv_bfloat16_raw(a), __NV_SATFINITE, fp8_type); - return (uint8_t)res; - #endif -} - -// float -> fp8 -template <> -__inline__ __device__ uint8_t vec_conversion( - const float &a, const __nv_fp8_interpretation_t fp8_type) { - __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, fp8_type); - return (uint8_t)res; -} - -// fp8x4 -> float4 -template <> -__inline__ __device__ float4 vec_conversion( - const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) { - Float4_ tmp = vec_conversion(a, fp8_type); - float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y); - return res; -} - -template <> -__inline__ __device__ uint32_t vec_conversion( - const float2 &a, const __nv_fp8_interpretation_t fp8_type) { - union { - half2 float16; - uint32_t uint32; - }; - - float16 = __float22half2_rn(a); - return uint32; -} - -template <> -__inline__ __device__ uint2 vec_conversion( - const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) { - uint2 b; - float2 val; - val.x = a.x.x; - val.y = a.x.y; - b.x = vec_conversion(val, fp8_type); - - val.x = a.y.x; - val.y = a.y.y; - b.y = vec_conversion(val, fp8_type); - - return b; -} - -template <> -__inline__ __device__ float4 vec_conversion( - const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) { - float4 b; - b.x = a.x.x; - b.y = a.x.y; - b.z = a.y.x; - b.w = a.y.y; - return b; -} - -template <> -__inline__ __device__ uint4 vec_conversion( - const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) { - uint4 b; - b.x = vec_conversion(a.x, fp8_type); - b.y = vec_conversion(a.y, fp8_type); - b.z = vec_conversion(a.z, fp8_type); - b.w = vec_conversion(a.w, fp8_type); - return b; -} - -template <> -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>( - const float2 &a, const __nv_fp8_interpretation_t fp8_type) { - __nv_bfloat162 b; - from_float(b, a); - return b; -} - -template <> -__inline__ __device__ bf16_4_t vec_conversion( - const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) { - bf16_4_t b; - from_float(b, a); - return b; -} - -template <> -__inline__ __device__ bf16_8_t vec_conversion( - const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) { - bf16_8_t b; - from_float(b, a); - return b; -} - #endif - /* Scaled and vectorized conversions, for data exchange between high and low precision domains Convention of the scale in API, e.g: FP8_data = Quantization( High_Precision_data / scale ) s.t. Quantize(HP / scale) => FP8 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index e50778f0ac..cb9cb34158 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -38,9 +38,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticChannelSym, kFp8StaticTensorSym, ) -from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( - cutlass_block_fp8_supported, -) __all__ = ["CompressedTensorsW8A8Fp8"] @@ -67,7 +64,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): self.weight_block_size = self.weight_quant.block_structure if self.weight_block_size is not None: - self.cutlass_block_fp8_supported = cutlass_block_fp8_supported() self.use_aiter_and_is_supported = rocm_aiter_ops.is_linear_fp8_enabled() assert not self.is_static_input_scheme self.act_q_group_shape = GroupShape(1, self.weight_block_size[0]) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 9c69ff93d3..20699484fc 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -67,7 +67,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, ) from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( - cutlass_block_fp8_supported, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, ) @@ -252,7 +251,6 @@ class Fp8LinearMethod(LinearMethodBase): def __init__(self, quant_config: Fp8Config): self.quant_config = quant_config self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False) - self.cutlass_block_fp8_supported = cutlass_block_fp8_supported() self.out_dtype = torch.get_default_dtype() self.input_dtype = get_current_vllm_config().model_config.dtype diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_scheme.py index b8bb263de0..8672972aa1 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_scheme.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase from vllm.model_executor.layers.linear import LinearMethodBase - from vllm.model_executor.layers.quantization import QuantizationMethods from ..config_parser import INCLayerConfig from ..inc import INCConfig @@ -22,7 +21,6 @@ class INCScheme(ABC): - can_handle(): when does this scheme apply? - get_linear_method(): required — how to quantize Linear layers - get_moe_method(): optional — how to quantize MoE layers - - get_kvcache_method(): optional — how to quantize KV cache Schemes that don't support MoE/KVCache inherit the default raise. """ @@ -56,20 +54,6 @@ class INCScheme(ABC): f"Layer config: {layer_config}" ) - def get_kvcache_method( - self, - config: "INCConfig", - layer: "torch.nn.Module", - prefix: str, - layer_config: "INCLayerConfig", - ) -> "QuantizationMethods": - """Optional. Override if this scheme supports KV cache quantization. - Default raises NotImplementedError.""" - raise NotImplementedError( - f"{type(self).__name__} does not support KV cache quantization. " - f"Layer config: {layer_config}" - ) - class INCLinearScheme(ABC): @classmethod diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 1000b42705..d2b286f148 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1064,8 +1064,6 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): else: self.input_dtype = None - self.fp4_dtype = getattr(torch, "float4_e2m1fn_x2", None) - self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype( self.input_dtype, self.weight_dtype ) diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 8b6edd6cae..59499a1060 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -60,8 +60,6 @@ class QuarkOCP_MX(QuarkScheme): input_quant_spec: dict[str, Any] | None, dynamic_mxfp4_quant: bool = False, ): - self.weight_quant_spec = weight_quant_spec - self.input_quant_spec = input_quant_spec self.dynamic_mxfp4_quant = dynamic_mxfp4_quant self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp") self.input_dtype: str | None = None diff --git a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py index b309bf14d9..47776bb3bf 100644 --- a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py @@ -38,7 +38,6 @@ def cutlass_group_gemm_supported() -> bool: return ops.cutlass_group_gemm_supported(capability) -CUTLASS_FP8_SUPPORTED = cutlass_fp8_supported() CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported()