[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
+47
-105
@@ -37,7 +37,7 @@ namespace {
|
||||
using cpu_utils::ISA;
|
||||
using cpu_utils::VecTypeTrait;
|
||||
|
||||
template <typename scalar_t, ISA isa, bool has_zp, bool use_desc_act>
|
||||
template <typename scalar_t, ISA isa, bool has_zp>
|
||||
class Dequantizer4b {
|
||||
public:
|
||||
constexpr static int32_t pack_num = 32 / 4;
|
||||
@@ -47,9 +47,9 @@ class Dequantizer4b {
|
||||
static void dequant(int32_t* __restrict__ q_weight,
|
||||
scalar_t* __restrict__ weight,
|
||||
scalar_t* __restrict__ scales,
|
||||
int32_t* __restrict__ zeros, int32_t* __restrict__ g_idx,
|
||||
const int64_t scales_stride, const int64_t zeros_stride,
|
||||
const int32_t k_size, const int32_t group_size) {
|
||||
int32_t* __restrict__ zeros, const int64_t scales_stride,
|
||||
const int64_t zeros_stride, const int32_t k_size,
|
||||
const int32_t group_size) {
|
||||
vec_op::FP32Vec16 lut;
|
||||
if constexpr (has_zp) {
|
||||
// AWQ
|
||||
@@ -81,30 +81,15 @@ class Dequantizer4b {
|
||||
vec_op::FP32Vec16 wb_0(qwb_0, lut);
|
||||
vec_op::FP32Vec16 wb_1(qwb_1, lut);
|
||||
|
||||
if constexpr (!use_desc_act) {
|
||||
if (group_counter == 0) {
|
||||
scale_0 = vec_op::FP32Vec16(scalar_vec_t(curr_scale));
|
||||
scale_1 = vec_op::FP32Vec16(scale_0);
|
||||
curr_scale += scales_stride;
|
||||
if (group_counter == 0) {
|
||||
scale_0 = vec_op::FP32Vec16(scalar_vec_t(curr_scale));
|
||||
scale_1 = vec_op::FP32Vec16(scale_0);
|
||||
curr_scale += scales_stride;
|
||||
|
||||
if constexpr (has_zp) {
|
||||
zero_0 = vec_op::FP32Vec16(*curr_zeros, lut);
|
||||
zero_1 = vec_op::FP32Vec16(zero_0);
|
||||
curr_zeros += zeros_stride / 2;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
int32_t g_idx_0 = g_idx[k_idx];
|
||||
int32_t g_idx_1 = g_idx[k_idx + 1];
|
||||
scale_0 = vec_op::FP32Vec16(
|
||||
scalar_vec_t(curr_scale + g_idx_0 * scales_stride));
|
||||
scale_1 = vec_op::FP32Vec16(
|
||||
scalar_vec_t(curr_scale + g_idx_1 * scales_stride));
|
||||
if constexpr (has_zp) {
|
||||
zero_0 = vec_op::FP32Vec16(*(curr_zeros + g_idx_0 * zeros_stride / 2),
|
||||
lut);
|
||||
zero_1 = vec_op::FP32Vec16(*(curr_zeros + g_idx_1 * zeros_stride / 2),
|
||||
lut);
|
||||
zero_0 = vec_op::FP32Vec16(*curr_zeros, lut);
|
||||
zero_1 = vec_op::FP32Vec16(zero_0);
|
||||
curr_zeros += zeros_stride / 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,11 +115,9 @@ class Dequantizer4b {
|
||||
// update
|
||||
curr_q_weight += 2;
|
||||
curr_weight += 32;
|
||||
if constexpr (!use_desc_act) {
|
||||
group_counter += 2;
|
||||
if (group_counter == group_size) {
|
||||
group_counter = 0;
|
||||
}
|
||||
group_counter += 2;
|
||||
if (group_counter == group_size) {
|
||||
group_counter = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -145,11 +128,10 @@ template <typename scalar_t, typename dequantizer_t, typename gemm_t>
|
||||
void cpu_gemm_wna16_impl(
|
||||
scalar_t* __restrict__ input, int32_t* __restrict__ q_weight,
|
||||
scalar_t* __restrict__ output, scalar_t* __restrict__ scales,
|
||||
int32_t* __restrict__ zeros, int32_t* __restrict__ g_idx,
|
||||
scalar_t* __restrict__ bias, const int32_t m_size, const int32_t n_size,
|
||||
const int32_t k_size, const int64_t input_stride,
|
||||
const int64_t output_stride, const int64_t scales_group_stride,
|
||||
const int64_t zeros_group_stride, const int32_t group_num,
|
||||
int32_t* __restrict__ zeros, scalar_t* __restrict__ bias,
|
||||
const int32_t m_size, const int32_t n_size, const int32_t k_size,
|
||||
const int64_t input_stride, const int64_t output_stride,
|
||||
const int64_t scales_group_stride, const int64_t zeros_group_stride,
|
||||
const int32_t group_size, const int64_t pack_factor) {
|
||||
constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
|
||||
constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
|
||||
@@ -229,7 +211,7 @@ void cpu_gemm_wna16_impl(
|
||||
int32_t* __restrict__ curr_zeros = zeros + n_start_idx / pack_factor;
|
||||
for (int32_t block_idx = 0; block_idx < n_block_num; ++block_idx) {
|
||||
dequantizer_t::dequant(curr_q_weight, curr_b_buffer, curr_scales,
|
||||
curr_zeros, g_idx, scales_group_stride,
|
||||
curr_zeros, scales_group_stride,
|
||||
zeros_group_stride, k_size, group_size);
|
||||
|
||||
// if (block_idx == 0 && n_start_idx == 0) {
|
||||
@@ -296,8 +278,7 @@ void cpu_gemm_wna16(
|
||||
const torch::Tensor& scales, // [group_num, N]
|
||||
const std::optional<torch::Tensor>&
|
||||
zeros, // [group_num, N / pack_factor], packed as int32
|
||||
const std::optional<torch::Tensor>& g_idx, // [K]
|
||||
const std::optional<torch::Tensor>& bias, // [N]
|
||||
const std::optional<torch::Tensor>& bias, // [N]
|
||||
const int64_t pack_factor, const std::string& isa_hint) {
|
||||
using cpu_utils::ISA;
|
||||
TORCH_CHECK_EQ(pack_factor, 8); // only supports 4bits
|
||||
@@ -314,8 +295,6 @@ void cpu_gemm_wna16(
|
||||
const int64_t output_m_stride = output.stride(0);
|
||||
|
||||
bool has_zp = zeros.has_value();
|
||||
bool use_desc_act = g_idx.has_value();
|
||||
TORCH_CHECK(!(has_zp && use_desc_act));
|
||||
|
||||
ISA isa = [&]() {
|
||||
if (isa_hint == "amx") {
|
||||
@@ -331,109 +310,72 @@ void cpu_gemm_wna16(
|
||||
|
||||
int32_t* zeros_ptr = has_zp ? zeros->data_ptr<int32_t>() : nullptr;
|
||||
const int64_t zeros_group_stride = has_zp ? zeros->stride(0) : 0;
|
||||
int32_t* g_idx_ptr = use_desc_act ? g_idx->data_ptr<int32_t>() : nullptr;
|
||||
|
||||
VLLM_DISPATCH_16B_TYPES(input.scalar_type(), "cpu_gemm_wna16", [&]() {
|
||||
if (isa == ISA::AMX) {
|
||||
using gemm_t = cpu_micro_gemm::MicroGemm<ISA::AMX, scalar_t>;
|
||||
if (has_zp) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, true, false>;
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, true>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
if (use_desc_act) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, false, true>;
|
||||
{
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
return;
|
||||
} else {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, false, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
} else if (isa == ISA::VEC) {
|
||||
using gemm_t = cpu_micro_gemm::MicroGemm<ISA::VEC, scalar_t>;
|
||||
if (has_zp) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, true, false>;
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, true>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
if (use_desc_act) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, false, true>;
|
||||
{
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
return;
|
||||
} else {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, false, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
} else if (isa == ISA::RVV) {
|
||||
using gemm_t = cpu_micro_gemm::MicroGemm<ISA::RVV, scalar_t>;
|
||||
if (has_zp) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, true, false>;
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, true>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
if (use_desc_act) {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false, true>;
|
||||
{
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
return;
|
||||
} else {
|
||||
using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false, false>;
|
||||
cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
|
||||
input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
|
||||
output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
|
||||
g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
|
||||
a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_num, group_size,
|
||||
pack_factor);
|
||||
bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size,
|
||||
b_n_size, a_k_size, a_m_stride, output_m_stride,
|
||||
scales_group_stride, zeros_group_stride, group_size, pack_factor);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,7 +226,6 @@ void placeholder_op() { TORCH_CHECK(false, "Unimplemented"); }
|
||||
void cpu_gemm_wna16(const torch::Tensor& input, const torch::Tensor& q_weight,
|
||||
torch::Tensor& output, const torch::Tensor& scales,
|
||||
const std::optional<torch::Tensor>& zeros,
|
||||
const std::optional<torch::Tensor>& g_idx,
|
||||
const std::optional<torch::Tensor>& bias,
|
||||
const int64_t pack_factor, const std::string& isa_hint);
|
||||
|
||||
@@ -671,7 +670,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
#if defined(__AVX512F__) || defined(__riscv_v) || defined(__s390x__)
|
||||
ops.def(
|
||||
"cpu_gemm_wna16(Tensor input, Tensor q_weight, Tensor(a2!) output, "
|
||||
"Tensor scales, Tensor? zeros, Tensor? g_idx, Tensor? bias, SymInt "
|
||||
"Tensor scales, Tensor? zeros, Tensor? bias, SymInt "
|
||||
"pack_factor, str isa_hint) -> ()");
|
||||
ops.impl("cpu_gemm_wna16", torch::kCPU, &cpu_gemm_wna16);
|
||||
#endif
|
||||
|
||||
@@ -75,14 +75,14 @@ QUANT_CONFIGS = [
|
||||
"b_type": "kU4B8",
|
||||
"thread_configs": THREAD_CONFIGS,
|
||||
"thread_m_blocks": THREAD_M_BLOCKS,
|
||||
"group_blocks": [-1, 0, 2, 4, 8],
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# AWQ-INT8
|
||||
{
|
||||
"b_type": "kU8B128",
|
||||
"thread_configs": THREAD_CONFIGS,
|
||||
"thread_m_blocks": THREAD_M_BLOCKS,
|
||||
"group_blocks": [-1, 0, 2, 4, 8],
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
{
|
||||
|
||||
@@ -7,21 +7,20 @@
|
||||
#include "libtorch_stable/quantization/marlin/marlin_dtypes.cuh"
|
||||
#include "core/scalar_type.hpp"
|
||||
|
||||
#define MARLIN_KERNEL_PARAMS \
|
||||
const int4 *__restrict__ A, const int4 *__restrict__ B, \
|
||||
int4 *__restrict__ C, int4 *__restrict__ C_tmp, \
|
||||
const int4 *__restrict__ b_bias_ptr, \
|
||||
const float *__restrict__ a_scales_ptr, \
|
||||
const int4 *__restrict__ scales_ptr, \
|
||||
const float *__restrict__ global_scale_ptr, \
|
||||
const int4 *__restrict__ zp_ptr, const int *__restrict__ g_idx, \
|
||||
const int32_t *__restrict__ sorted_token_ids_ptr, \
|
||||
const int32_t *__restrict__ expert_ids_ptr, \
|
||||
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
|
||||
const float *__restrict__ topk_weights_ptr, int top_k, \
|
||||
bool mul_topk_weights, int num_groups, int prob_m, int prob_n, \
|
||||
int prob_k, int *locks, bool has_bias, bool use_atomic_add, \
|
||||
bool use_fp32_reduce
|
||||
#define MARLIN_KERNEL_PARAMS \
|
||||
const int4 *__restrict__ A, const int4 *__restrict__ B, \
|
||||
int4 *__restrict__ C, int4 *__restrict__ C_tmp, \
|
||||
const int4 *__restrict__ b_bias_ptr, \
|
||||
const float *__restrict__ a_scales_ptr, \
|
||||
const int4 *__restrict__ scales_ptr, \
|
||||
const float *__restrict__ global_scale_ptr, \
|
||||
const int4 *__restrict__ zp_ptr, \
|
||||
const int32_t *__restrict__ sorted_token_ids_ptr, \
|
||||
const int32_t *__restrict__ expert_ids_ptr, \
|
||||
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
|
||||
const float *__restrict__ topk_weights_ptr, int top_k, \
|
||||
bool mul_topk_weights, int prob_m, int prob_n, int prob_k, int *locks, \
|
||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce
|
||||
|
||||
namespace MARLIN_NAMESPACE_NAME {
|
||||
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
|
||||
|
||||
@@ -50,10 +50,9 @@ template <typename scalar_t, // compute dtype, half or nv_float16
|
||||
// only works when thread_m_blocks == 1
|
||||
const int stages, // number of stages for the async global->shared
|
||||
// fetch pipeline
|
||||
const bool has_act_order, // whether act_order is enabled
|
||||
const int group_blocks, // number of consecutive 16x16 blocks
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float // is zero point of float16 type?
|
||||
const int group_blocks, // number of consecutive 16x16 blocks
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float // is zero point of float16 type?
|
||||
>
|
||||
__global__ void Marlin(
|
||||
const int4* __restrict__ A, // fp16 input matrix of shape mxk
|
||||
@@ -64,14 +63,12 @@ __global__ void Marlin(
|
||||
// (k/groupsize)xn
|
||||
const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
|
||||
// (k/groupsize)x(n/pack_factor)
|
||||
const int* __restrict__ g_idx, // int32 group indices of shape k
|
||||
const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids
|
||||
const int32_t* __restrict__ expert_ids_ptr, // moe expert ids
|
||||
const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens
|
||||
const float* __restrict__ topk_weights_ptr, // moe top weights
|
||||
int top_k, // num of experts per token
|
||||
bool mul_topk_weights, // mul topk weights or not
|
||||
int num_groups, // number of scale groups per output channel
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
@@ -148,29 +145,6 @@ __device__ inline void sub_zp(
|
||||
frag_b[1] = __hsub2(frag_b[1], zp);
|
||||
}
|
||||
|
||||
// Same as above, but for act_order (each K is multiplied individually)
|
||||
template <vllm::ScalarTypeId type_id>
|
||||
__device__ inline void scale4(
|
||||
typename MarlinScalarType<type_id>::FragB& frag_b,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_1,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_2,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_3,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_4, int i) {
|
||||
using scalar_t = typename MarlinScalarType<type_id>::scalar_t;
|
||||
using scalar_t2 = typename MarlinScalarType<type_id>::scalar_t2;
|
||||
|
||||
scalar_t2 s_val_1_2;
|
||||
s_val_1_2.x = reinterpret_cast<scalar_t*>(&frag_s_1)[i];
|
||||
s_val_1_2.y = reinterpret_cast<scalar_t*>(&frag_s_2)[i];
|
||||
|
||||
scalar_t2 s_val_3_4;
|
||||
s_val_3_4.x = reinterpret_cast<scalar_t*>(&frag_s_3)[i];
|
||||
s_val_3_4.y = reinterpret_cast<scalar_t*>(&frag_s_4)[i];
|
||||
|
||||
frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
|
||||
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
|
||||
}
|
||||
|
||||
// Given 2 floats multiply by 2 scales (halves)
|
||||
template <vllm::ScalarTypeId type_id>
|
||||
__device__ inline void scale_float(
|
||||
@@ -264,15 +238,12 @@ __global__ void Marlin(
|
||||
// 4bit packed zero-points of shape
|
||||
// (k/groupsize, n/pack_factor)
|
||||
const int4* __restrict__ zp_ptr,
|
||||
// int32 group indices of shape k
|
||||
const int* __restrict__ g_idx,
|
||||
const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids
|
||||
const int32_t* __restrict__ expert_ids_ptr, // moe expert ids
|
||||
const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens
|
||||
const float* __restrict__ topk_weights_ptr, // moe top weights
|
||||
int top_k, // num of experts per token
|
||||
bool mul_topk_weights, // mul topk weights or not
|
||||
int num_groups, // number of scale groups per output channel
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
@@ -369,12 +340,12 @@ __global__ void Marlin(
|
||||
|
||||
float global_scale_f32 = 1.0f;
|
||||
|
||||
constexpr bool has_act_order = group_blocks == 0;
|
||||
static_assert(group_blocks != 0,
|
||||
"runtime activation ordering is not supported");
|
||||
|
||||
constexpr int pack_factor = 32 / b_type.size_bits();
|
||||
static_assert(thread_m_blocks == 1 || !m_block_size_8);
|
||||
const int group_size =
|
||||
(!has_act_order && group_blocks == -1) ? prob_k : prob_k / num_groups;
|
||||
const int group_size = group_blocks == -1 ? prob_k : group_blocks * 16;
|
||||
const int scales_expert_stride =
|
||||
prob_n * prob_k / group_size / (is_8bit_scale ? 16 : 8);
|
||||
const int zp_expert_stride =
|
||||
@@ -405,7 +376,7 @@ __global__ void Marlin(
|
||||
|
||||
int iters = div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
||||
|
||||
if constexpr (!has_act_order && group_blocks != -1) {
|
||||
if constexpr (group_blocks != -1) {
|
||||
if (group_blocks >= thread_k_blocks) {
|
||||
// Ensure that the number of tiles in each stripe is a multiple of the
|
||||
// groupsize; this avoids an annoying special case where a stripe starts
|
||||
@@ -551,9 +522,6 @@ __global__ void Marlin(
|
||||
if constexpr (has_zp) {
|
||||
zp_ptr += (expert_id - old_expert_id) * zp_expert_stride;
|
||||
}
|
||||
if constexpr (has_act_order) {
|
||||
g_idx += (expert_id - old_expert_id) * prob_k;
|
||||
}
|
||||
if (has_bias) {
|
||||
b_bias_ptr += (expert_id - old_expert_id) * b_bias_expert_stride;
|
||||
}
|
||||
@@ -694,27 +662,17 @@ __global__ void Marlin(
|
||||
b_sh_stride * thread_k_blocks / (is_a_8bit ? 2 : 1);
|
||||
constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
|
||||
|
||||
// Scale sizes/strides without act_order
|
||||
// Scale sizes/strides
|
||||
int s_gl_stride = prob_n / (is_8bit_scale ? 16 : 8);
|
||||
constexpr int s_sh_stride = 16 * thread_n_blocks / (is_8bit_scale ? 16 : 8);
|
||||
constexpr int s_tb_groups =
|
||||
!has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
|
||||
group_blocks != -1 && group_blocks < thread_k_blocks
|
||||
? thread_k_blocks / group_blocks
|
||||
: 1;
|
||||
constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
|
||||
int s_gl_rd_delta = s_gl_stride;
|
||||
|
||||
// Scale size/strides with act_order
|
||||
constexpr int tb_k = 16 * thread_k_blocks;
|
||||
constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
|
||||
// constexpr int act_s_row_stride = 1;
|
||||
// int act_s_col_stride = act_s_row_stride * num_groups;
|
||||
constexpr int act_s_max_num_groups = 32;
|
||||
int act_s_col_stride = 1;
|
||||
int act_s_col_warp_stride = act_s_col_stride * 8;
|
||||
|
||||
constexpr int tb_n_warps = thread_n_blocks / (is_a_8bit ? 2 : 4);
|
||||
int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
|
||||
|
||||
// Zero-points sizes/strides
|
||||
int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4;
|
||||
@@ -750,25 +708,16 @@ __global__ void Marlin(
|
||||
auto b_sh_rd = threadIdx.x * b_thread_vecs;
|
||||
b_sh_rd += b_sh_rd / b_sh_stride * (b_sh_stride * (b_sh_wr_iters - 1));
|
||||
|
||||
// For act_order
|
||||
int slice_k_start = tb_k * slice_row;
|
||||
int slice_k_finish = slice_k_start + tb_k * slice_iters;
|
||||
int slice_k_start_shared_fetch = slice_k_start;
|
||||
int slice_n_offset = act_s_col_tb_stride * slice_col;
|
||||
|
||||
// No act_order
|
||||
int s_gl_rd;
|
||||
if constexpr (!has_act_order) {
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
}
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
}
|
||||
auto s_sh_wr = threadIdx.x;
|
||||
bool s_sh_wr_pred = threadIdx.x < s_sh_stage;
|
||||
@@ -885,10 +834,8 @@ __global__ void Marlin(
|
||||
: (sh_size_b_red_min + sh_bias_size);
|
||||
|
||||
int4* sh_bias = sh_new + sh_size_b_red_min;
|
||||
int4* sh_g_idx = sh_new + sh_b_red_bias_size;
|
||||
int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
|
||||
constexpr int sh_s_size = has_act_order ? (act_s_max_num_groups * s_sh_stride)
|
||||
: (stages * s_sh_stage);
|
||||
int4* sh_zp = sh_new + sh_b_red_bias_size;
|
||||
constexpr int sh_s_size = stages * s_sh_stage;
|
||||
int4* sh_s = sh_zp + (stages * zp_sh_stage);
|
||||
int4* sh_a = sh_s + sh_s_size;
|
||||
|
||||
@@ -897,9 +844,8 @@ __global__ void Marlin(
|
||||
I4 frag_b_quant[2][b_thread_vecs];
|
||||
FragC frag_c[thread_m_blocks][is_a_8bit ? 2 : 4][2];
|
||||
FragC frag_c_tmp[thread_m_blocks][is_a_8bit ? 2 : 4][2];
|
||||
FragS frag_s[2][4]; // No act-order
|
||||
FragS frag_s[2][4];
|
||||
FragS frag_bias[2][4];
|
||||
FragS act_frag_s[2][4][4]; // For act-order
|
||||
int frag_qzp[2][num_ints_per_thread]; // Zero-points
|
||||
FragZP frag_zp; // Zero-points in fp16
|
||||
FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ
|
||||
@@ -929,42 +875,6 @@ __global__ void Marlin(
|
||||
reinterpret_cast<float*>(frag_c)[i] = 0;
|
||||
};
|
||||
|
||||
int sh_first_group_id = -1;
|
||||
int sh_num_groups = -1;
|
||||
|
||||
auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id,
|
||||
int last_group_id) {
|
||||
sh_first_group_id = first_group_id;
|
||||
sh_num_groups = last_group_id - first_group_id + 1;
|
||||
|
||||
if (sh_num_groups > act_s_max_num_groups) {
|
||||
sh_num_groups = act_s_max_num_groups;
|
||||
}
|
||||
|
||||
if (sh_first_group_id + sh_num_groups > num_groups) {
|
||||
sh_num_groups = num_groups - sh_first_group_id;
|
||||
}
|
||||
|
||||
int row_offset = first_group_id * s_gl_stride;
|
||||
|
||||
if (is_async) {
|
||||
for (int i = 0; i < sh_num_groups; i++) {
|
||||
if (threadIdx.x < s_sh_stride) {
|
||||
cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
|
||||
&scales_ptr[row_offset + (i * s_gl_stride) +
|
||||
slice_n_offset + threadIdx.x]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < sh_num_groups; i++) {
|
||||
if (threadIdx.x < s_sh_stride) {
|
||||
sh_s[(i * s_sh_stride) + threadIdx.x] =
|
||||
scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
|
||||
threadIdx.x];
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// Asynchronously fetch the next A, B and s tile from global to the next
|
||||
// shared memory pipeline location.
|
||||
auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
|
||||
@@ -995,44 +905,27 @@ __global__ void Marlin(
|
||||
|
||||
b_gl_rd += b_gl_rd_delta_o;
|
||||
|
||||
if constexpr (has_act_order) {
|
||||
// Fetch g_idx thread-block portion
|
||||
int full_pipe = a_off;
|
||||
int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
|
||||
if (cur_k < prob_k && cur_k < slice_k_finish) {
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
if constexpr (group_blocks != -1) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
int4 const* cur_g_idx_stage_ptr =
|
||||
reinterpret_cast<int4 const*>(&g_idx[cur_k]);
|
||||
|
||||
if (threadIdx.x < g_idx_stage) {
|
||||
cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
|
||||
&cur_g_idx_stage_ptr[threadIdx.x]);
|
||||
// Only fetch scales if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (s_sh_wr_pred) {
|
||||
cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
|
||||
}
|
||||
s_gl_rd += s_gl_rd_delta * s_tb_groups;
|
||||
}
|
||||
} else {
|
||||
if constexpr (group_blocks != -1) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
}
|
||||
|
||||
// Only fetch scales if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (s_sh_wr_pred) {
|
||||
cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
|
||||
}
|
||||
s_gl_rd += s_gl_rd_delta * s_tb_groups;
|
||||
}
|
||||
}
|
||||
if constexpr (has_zp && group_blocks != -1) {
|
||||
int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
|
||||
|
||||
if constexpr (has_zp && group_blocks != -1) {
|
||||
int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
|
||||
|
||||
// Only fetch zero points if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (zp_sh_wr_pred) {
|
||||
cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
|
||||
}
|
||||
zp_gl_rd += zp_gl_rd_delta * zp_tb_groups;
|
||||
// Only fetch zero points if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (zp_sh_wr_pred) {
|
||||
cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
|
||||
}
|
||||
zp_gl_rd += zp_gl_rd_delta * zp_tb_groups;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1080,150 +973,57 @@ __global__ void Marlin(
|
||||
}
|
||||
};
|
||||
|
||||
bool is_same_group[stages];
|
||||
int same_group_id[stages];
|
||||
|
||||
auto init_same_group = [&](int pipe) {
|
||||
if constexpr (!has_act_order) {
|
||||
return;
|
||||
}
|
||||
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
|
||||
|
||||
int group_id_1 = sh_g_idx_int_ptr[0];
|
||||
int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
|
||||
|
||||
is_same_group[pipe] = group_id_1 == group_id_2;
|
||||
same_group_id[pipe] = group_id_1;
|
||||
};
|
||||
|
||||
auto fetch_scales_to_registers = [&](int k, int full_pipe) {
|
||||
int pipe = full_pipe % stages;
|
||||
using IT1 = typename std::conditional_t<is_a_8bit, int2, int4>;
|
||||
using IT0 = typename std::conditional_t<is_a_8bit, int, int2>;
|
||||
constexpr int group_blocks2 = div_ceil(group_blocks, is_a_8bit ? 2 : 1);
|
||||
|
||||
if constexpr (!has_act_order) {
|
||||
// No act-order case
|
||||
if constexpr (group_blocks == -1) {
|
||||
// load only when starting a new slice
|
||||
if (k == 0 && full_pipe == 0 && dequant_skip_flop) {
|
||||
reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd];
|
||||
reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
|
||||
}
|
||||
} else if constexpr (group_blocks != -1) {
|
||||
if constexpr (group_blocks >= thread_k_blocks) {
|
||||
constexpr int g = group_blocks / thread_k_blocks;
|
||||
if (pipe % g == 0) {
|
||||
if (k % b_sh_wr_iters == 0) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * (g * (pipe / g));
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
|
||||
} else {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
}
|
||||
}
|
||||
} else if (group_blocks2 < b_sh_wr_iters || k % b_sh_wr_iters == 0) {
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
|
||||
int k_blocks = b_sh_wr_iters * warp_row + k % b_sh_wr_iters;
|
||||
int cur_group_id = k_blocks / group_blocks2;
|
||||
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
|
||||
sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[k % 2])[0] =
|
||||
reinterpret_cast<int2*>(
|
||||
sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)];
|
||||
}
|
||||
} else if (group_blocks >= b_sh_wr_iters) {
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int2*>(&frag_s[0])[0];
|
||||
}
|
||||
if constexpr (group_blocks == -1) {
|
||||
// Load only when starting a new slice.
|
||||
if (k == 0 && full_pipe == 0 && dequant_skip_flop) {
|
||||
reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd];
|
||||
reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
|
||||
}
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
constexpr int g = group_blocks / thread_k_blocks;
|
||||
if (pipe % g == 0) {
|
||||
if (k % b_sh_wr_iters == 0) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * (g * (pipe / g));
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
|
||||
} else {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
}
|
||||
}
|
||||
} else if (group_blocks2 < b_sh_wr_iters || k % b_sh_wr_iters == 0) {
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
|
||||
return;
|
||||
}
|
||||
int k_blocks = b_sh_wr_iters * warp_row + k % b_sh_wr_iters;
|
||||
int cur_group_id = k_blocks / group_blocks2;
|
||||
|
||||
// Act-order case
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
// Determine K of the "current" thread-block
|
||||
int cur_k = slice_k_start + tb_k * full_pipe;
|
||||
if (cur_k >= prob_k || cur_k >= slice_k_finish) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset (to current thread-block) since we read g_idx portion from the
|
||||
// shared memory
|
||||
cur_k = 0;
|
||||
|
||||
// Progress to current iteration
|
||||
cur_k += k % b_sh_wr_iters;
|
||||
|
||||
// Determine "position" inside the thread-block (based on warp and
|
||||
// thread-id)
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
int warp_col = warp_id % tb_n_warps;
|
||||
|
||||
cur_k += warp_row * 16 * b_sh_wr_iters;
|
||||
|
||||
auto th_id = threadIdx.x % 32;
|
||||
cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
|
||||
|
||||
int s_col_shift =
|
||||
/*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
|
||||
(th_id / 4) * act_s_col_stride;
|
||||
|
||||
if (is_same_group[pipe]) {
|
||||
if (k % 2 == 0) {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
|
||||
sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
|
||||
s_col_shift];
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
|
||||
sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
|
||||
} else {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
|
||||
reinterpret_cast<int2*>(&frag_s[k % 2])[0] = reinterpret_cast<int2*>(
|
||||
sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)];
|
||||
}
|
||||
|
||||
for (int i = 1; i < 4; i++) {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
|
||||
} else if (group_blocks >= b_sh_wr_iters) {
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int2*>(&frag_s[0])[0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
|
||||
|
||||
constexpr int k_frag_offsets[4] = {0, 1, 8,
|
||||
9}; // Tensor core offsets per thread
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int actual_k = cur_k + k_frag_offsets[i];
|
||||
|
||||
int group_id = sh_g_idx_int_ptr[actual_k];
|
||||
int rel_group_id = group_id - sh_first_group_id;
|
||||
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
|
||||
sh_s[rel_group_id * s_sh_stride + s_col_shift];
|
||||
}
|
||||
};
|
||||
|
||||
auto fetch_zp_to_registers = [&](int k, int full_pipe) {
|
||||
// This code does not handle group_blocks == 0,
|
||||
// which signifies act_order.
|
||||
// has_zp implies AWQ, which doesn't have act_order,
|
||||
static_assert(!has_zp || group_blocks != 0);
|
||||
|
||||
if constexpr (has_zp && !is_zp_float) {
|
||||
@@ -1313,8 +1113,7 @@ __global__ void Marlin(
|
||||
constexpr int g =
|
||||
group_blocks > 0 ? div_ceil(group_blocks, thread_k_blocks) : 1;
|
||||
const bool is_new_zp =
|
||||
(group_blocks == 0) ||
|
||||
((group_blocks > 0) && (group_blocks < b_sh_wr_iters || k == 0)) &&
|
||||
(group_blocks > 0 && (group_blocks < b_sh_wr_iters || k == 0)) &&
|
||||
(pipe % g == 0) ||
|
||||
(group_blocks == -1 && is_first_matmul_in_slice);
|
||||
if constexpr (has_zp && !is_zp_float) {
|
||||
@@ -1382,15 +1181,8 @@ __global__ void Marlin(
|
||||
sub_zp<a_type_id>(frag_b1, frag_zp[j], 1);
|
||||
}
|
||||
|
||||
// Apply scale to frag_b0
|
||||
if constexpr (has_act_order && !is_a_8bit) {
|
||||
static_assert(group_blocks != -1);
|
||||
scale4<a_type_id>(frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j],
|
||||
act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0);
|
||||
scale4<a_type_id>(frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j],
|
||||
act_frag_s[k2][2][j], act_frag_s[k2][3][j], 1);
|
||||
} else if constexpr (!dequant_skip_flop && has_zp && !is_zp_float &&
|
||||
group_blocks == -1 && !is_a_8bit) {
|
||||
if constexpr (!dequant_skip_flop && has_zp && !is_zp_float &&
|
||||
group_blocks == -1 && !is_a_8bit) {
|
||||
int idx = (threadIdx.x / 4) % 2;
|
||||
scalar_t2 s2 = Adtype::nums2num2(
|
||||
reinterpret_cast<scalar_t*>(&frag_s[j / 2][j % 2 * 2 + 0])[idx],
|
||||
@@ -1805,7 +1597,7 @@ __global__ void Marlin(
|
||||
|
||||
// For per-column quantization we finally apply the scale here (only for
|
||||
// 4-bit)
|
||||
if constexpr (!has_act_order && group_blocks == -1 && !is_a_8bit &&
|
||||
if constexpr (group_blocks == -1 && !is_a_8bit &&
|
||||
b_type.size_bits() == 4 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
c_scalar_t2 tmp_scale = s[0];
|
||||
@@ -1911,15 +1703,6 @@ __global__ void Marlin(
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < stages - 1; i++) {
|
||||
if (has_act_order && i == 0) {
|
||||
int last_g_idx = slice_k_start + stages * tb_k * 2;
|
||||
if (last_g_idx >= prob_k) {
|
||||
last_g_idx = prob_k - 1;
|
||||
}
|
||||
fetch_act_order_scales_to_shared(true, g_idx[slice_k_start],
|
||||
g_idx[last_g_idx]);
|
||||
}
|
||||
|
||||
if constexpr (has_zp && !is_zp_float && group_blocks == -1) {
|
||||
if (i == 0) {
|
||||
fetch_col_zp_to_shared();
|
||||
@@ -1933,14 +1716,10 @@ __global__ void Marlin(
|
||||
|
||||
zero_accums();
|
||||
wait_for_stage();
|
||||
init_same_group(0);
|
||||
fetch_to_registers(0, 0);
|
||||
fetch_scales_to_registers(0, 0);
|
||||
fetch_zp_to_registers(0, 0);
|
||||
a_gl_rd_col += a_gl_rd_delta_o * (stages - 1);
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start_shared_fetch += tb_k * (stages - 1);
|
||||
}
|
||||
};
|
||||
if (slice_iters) {
|
||||
start_pipes();
|
||||
@@ -1965,7 +1744,6 @@ __global__ void Marlin(
|
||||
slice_iters >= stages);
|
||||
pipe++;
|
||||
wait_for_stage();
|
||||
init_same_group(pipe % stages);
|
||||
}
|
||||
|
||||
if constexpr (!is_a_8bit) {
|
||||
@@ -1983,25 +1761,6 @@ __global__ void Marlin(
|
||||
|
||||
a_gl_rd_col += a_gl_rd_delta_o * stages;
|
||||
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start += tb_k * stages;
|
||||
|
||||
if (slice_k_start < prob_k) {
|
||||
slice_k_start_shared_fetch += tb_k * stages;
|
||||
int first_group_id = g_idx[slice_k_start];
|
||||
int last_g_idx = slice_k_start + stages * tb_k * 2;
|
||||
if (last_g_idx >= prob_k) {
|
||||
last_g_idx = prob_k - 1;
|
||||
}
|
||||
int last_group_id = g_idx[last_g_idx];
|
||||
if (last_group_id >= sh_first_group_id + sh_num_groups) {
|
||||
fetch_act_order_scales_to_shared(false, first_group_id,
|
||||
last_group_id);
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process results and, if necessary, proceed to the next column slice.
|
||||
// While this pattern may not be the most readable, other ways of writing
|
||||
// the loop seemed to noticeably worse performance after compilation.
|
||||
@@ -2059,7 +1818,7 @@ __global__ void Marlin(
|
||||
bool last = slice_idx == slice_count - 1;
|
||||
// For per-column scales, we only fetch them here in the final step before
|
||||
// write-out
|
||||
if constexpr (!has_act_order && group_blocks == -1 &&
|
||||
if constexpr (group_blocks == -1 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
if (b_type.size_bits() == 8 || (last || use_atomic_add) || is_a_8bit) {
|
||||
if (s_sh_wr_pred) {
|
||||
@@ -2078,7 +1837,7 @@ __global__ void Marlin(
|
||||
cp_async_fence();
|
||||
}
|
||||
|
||||
if constexpr (!has_act_order && group_blocks == -1 &&
|
||||
if constexpr (group_blocks == -1 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp || is_a_8bit)) {
|
||||
if constexpr (is_a_8bit) {
|
||||
cp_async_wait<0>();
|
||||
@@ -2109,7 +1868,7 @@ __global__ void Marlin(
|
||||
// For 8-bit channelwise, we apply the scale before the global reduction
|
||||
// that converts the fp32 results to fp16 (so that we avoid possible
|
||||
// overflow in fp16)
|
||||
if constexpr (!has_act_order && group_blocks == -1 && is_a_8bit) {
|
||||
if constexpr (group_blocks == -1 && is_a_8bit) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
float2 aa[2];
|
||||
@@ -2131,8 +1890,7 @@ __global__ void Marlin(
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!has_act_order && group_blocks == -1 &&
|
||||
b_type.size_bits() == 8 &&
|
||||
} else if (group_blocks == -1 && b_type.size_bits() == 8 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
if (threadIdx.x / 32 < tb_n_warps) {
|
||||
#pragma unroll
|
||||
@@ -2202,33 +1960,25 @@ __global__ void Marlin(
|
||||
b_gl_rd += b_sh_stride * slice_col + b_gl_rd_delta_o * slice_row;
|
||||
|
||||
bias_gl_rd = (thread_n_blocks * 16 / 8) * slice_col + threadIdx.x;
|
||||
// Update slice k/n for scales loading
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start = tb_k * slice_row;
|
||||
slice_k_finish = slice_k_start + tb_k * slice_iters;
|
||||
slice_k_start_shared_fetch = slice_k_start;
|
||||
slice_n_offset = act_s_col_tb_stride * slice_col;
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / zp_sh_stride) +
|
||||
zp_sh_stride * slice_col + threadIdx.x % zp_sh_stride;
|
||||
}
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / zp_sh_stride) +
|
||||
zp_sh_stride * slice_col + threadIdx.x % zp_sh_stride;
|
||||
}
|
||||
start_pipes();
|
||||
}
|
||||
|
||||
@@ -45,84 +45,6 @@ __global__ void MarlinDefault(MARLIN_KERNEL_PARAMS){};
|
||||
|
||||
using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS);
|
||||
|
||||
// For a given "a" of size [M,K] performs a permutation of the K columns based
|
||||
// on the given "perm" indices.
|
||||
template <int moe_block_size>
|
||||
__global__ void permute_cols_kernel(
|
||||
int4 const* __restrict__ a_int4_ptr, int const* __restrict__ perm_int_ptr,
|
||||
int4* __restrict__ out_int4_ptr,
|
||||
const int32_t* __restrict__ sorted_token_ids_ptr,
|
||||
const int32_t* __restrict__ expert_ids_ptr,
|
||||
const int32_t* __restrict__ num_tokens_past_padded_ptr, int size_m,
|
||||
int size_k, int top_k) {
|
||||
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
int num_moe_blocks = div_ceil(num_tokens_past_padded, moe_block_size);
|
||||
int32_t block_sorted_ids[moe_block_size];
|
||||
int block_num_valid_tokens = 0;
|
||||
int64_t old_expert_id = 0;
|
||||
int64_t expert_id = 0;
|
||||
int row_stride = size_k * sizeof(half) / 16;
|
||||
|
||||
auto read_moe_block_data = [&](int block_id) {
|
||||
block_num_valid_tokens = moe_block_size;
|
||||
int4* tmp_block_sorted_ids = reinterpret_cast<int4*>(block_sorted_ids);
|
||||
for (int i = 0; i < moe_block_size / 4; i++) {
|
||||
tmp_block_sorted_ids[i] =
|
||||
((int4*)sorted_token_ids_ptr)[block_id * moe_block_size / 4 + i];
|
||||
}
|
||||
for (int i = 0; i < moe_block_size; i++) {
|
||||
if (block_sorted_ids[i] >= size_m * top_k) {
|
||||
block_num_valid_tokens = i;
|
||||
break;
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
auto permute_row = [&](int row) {
|
||||
int iters = size_k / default_threads;
|
||||
int rest = size_k % default_threads;
|
||||
|
||||
int in_offset = (row / top_k) * row_stride;
|
||||
int out_offset = row * row_stride;
|
||||
|
||||
half const* a_row_half =
|
||||
reinterpret_cast<half const*>(a_int4_ptr + in_offset);
|
||||
half* out_half = reinterpret_cast<half*>(out_int4_ptr + out_offset);
|
||||
|
||||
int base_k = 0;
|
||||
|
||||
for (int i = 0; i < iters; i++) {
|
||||
auto cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
|
||||
base_k += default_threads;
|
||||
}
|
||||
|
||||
if (rest) {
|
||||
if (threadIdx.x < rest) {
|
||||
auto cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (int index = blockIdx.x; index < num_moe_blocks; index += gridDim.x) {
|
||||
old_expert_id = expert_id;
|
||||
int tmp_expert_id = expert_ids_ptr[index];
|
||||
if (tmp_expert_id == -1) continue;
|
||||
expert_id = tmp_expert_id;
|
||||
perm_int_ptr += (expert_id - old_expert_id) * size_k;
|
||||
read_moe_block_data(index);
|
||||
|
||||
for (int i = 0; i < block_num_valid_tokens; i++)
|
||||
permute_row(block_sorted_ids[i]);
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int thread_k;
|
||||
int thread_n;
|
||||
@@ -152,9 +74,7 @@ typedef struct {
|
||||
|
||||
int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
|
||||
int prob_n, int prob_k, int num_bits, int group_size,
|
||||
bool has_act_order, bool is_k_full, int stages) {
|
||||
bool cache_scales_chunk = has_act_order && !is_k_full;
|
||||
|
||||
int stages) {
|
||||
int tb_n = th_config.thread_n;
|
||||
int tb_k = th_config.thread_k;
|
||||
|
||||
@@ -162,28 +82,17 @@ int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
|
||||
int tb_groups;
|
||||
if (group_size == -1) {
|
||||
tb_groups = 1;
|
||||
} else if (group_size == 0) {
|
||||
tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
|
||||
} else {
|
||||
tb_groups = div_ceil(tb_k, group_size);
|
||||
}
|
||||
|
||||
if (cache_scales_chunk) {
|
||||
int load_groups =
|
||||
tb_groups * stages * 2; // Chunk size is 2x pipeline over dim K
|
||||
load_groups = max(load_groups, 32); // We load at least 32 scale groups
|
||||
return load_groups * tb_n * 2;
|
||||
} else {
|
||||
int tb_scales = tb_groups * tb_n * 2;
|
||||
|
||||
return tb_scales * stages;
|
||||
}
|
||||
int tb_scales = tb_groups * tb_n * 2;
|
||||
return tb_scales * stages;
|
||||
}
|
||||
|
||||
int get_kernel_cache_size(thread_config_t const& th_config, bool m_block_size_8,
|
||||
int thread_m_blocks, int prob_m, int prob_n,
|
||||
int prob_k, int num_bits, int group_size,
|
||||
bool has_act_order, bool is_k_full, int has_zp,
|
||||
int prob_k, int num_bits, int group_size, int has_zp,
|
||||
int is_zp_float, bool is_a_8bit, int stages) {
|
||||
int pack_factor = 32 / num_bits;
|
||||
|
||||
@@ -203,10 +112,8 @@ int get_kernel_cache_size(thread_config_t const& th_config, bool m_block_size_8,
|
||||
(sh_b_size > sh_red_size ? sh_red_size : sh_b_size) + sh_bias_size;
|
||||
tmp_size = max(max(sh_b_size, sh_red_size), tmp_size);
|
||||
|
||||
int sh_s_size =
|
||||
get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
|
||||
group_size, has_act_order, is_k_full, stages);
|
||||
int sh_g_idx_size = has_act_order && !is_k_full ? stages * tb_k / 4 : 0;
|
||||
int sh_s_size = get_scales_cache_size(th_config, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, stages);
|
||||
int sh_zp_size = 0;
|
||||
if (has_zp) {
|
||||
if (is_zp_float)
|
||||
@@ -217,16 +124,15 @@ int get_kernel_cache_size(thread_config_t const& th_config, bool m_block_size_8,
|
||||
sh_zp_size = sh_s_size / 2;
|
||||
}
|
||||
|
||||
int total_size = tmp_size + sh_a_size + sh_s_size + sh_zp_size +
|
||||
sh_g_idx_size + sh_block_meta_size;
|
||||
int total_size =
|
||||
tmp_size + sh_a_size + sh_s_size + sh_zp_size + sh_block_meta_size;
|
||||
|
||||
return total_size;
|
||||
}
|
||||
|
||||
bool is_valid_config(thread_config_t const& th_config, bool m_block_size_8,
|
||||
int thread_m_blocks, int prob_m, int prob_n, int prob_k,
|
||||
int num_bits, int group_size, bool has_act_order,
|
||||
bool is_k_full, int has_zp, int is_zp_float,
|
||||
int num_bits, int group_size, int has_zp, int is_zp_float,
|
||||
bool is_a_8bit, int stages, int max_shared_mem) {
|
||||
// Sanity
|
||||
if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
|
||||
@@ -250,19 +156,20 @@ bool is_valid_config(thread_config_t const& th_config, bool m_block_size_8,
|
||||
}
|
||||
|
||||
// Check that pipeline fits into cache
|
||||
int cache_size =
|
||||
get_kernel_cache_size(th_config, m_block_size_8, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
int cache_size = get_kernel_cache_size(
|
||||
th_config, m_block_size_8, thread_m_blocks, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
return cache_size <= max_shared_mem;
|
||||
}
|
||||
|
||||
MarlinFuncPtr get_marlin_kernel(
|
||||
const vllm::ScalarType a_type, const vllm::ScalarType b_type,
|
||||
const vllm::ScalarType c_type, const vllm::ScalarType s_type,
|
||||
int thread_m_blocks, int thread_n_blocks, int thread_k_blocks,
|
||||
bool m_block_size_8, bool has_act_order, bool has_zp, int group_blocks,
|
||||
int threads, bool is_zp_float, int stages) {
|
||||
MarlinFuncPtr get_marlin_kernel(const vllm::ScalarType a_type,
|
||||
const vllm::ScalarType b_type,
|
||||
const vllm::ScalarType c_type,
|
||||
const vllm::ScalarType s_type,
|
||||
int thread_m_blocks, int thread_n_blocks,
|
||||
int thread_k_blocks, bool m_block_size_8,
|
||||
bool has_zp, int group_blocks, int threads,
|
||||
bool is_zp_float, int stages) {
|
||||
int num_bits = b_type.size_bits();
|
||||
auto kernel = MarlinDefault;
|
||||
|
||||
@@ -275,9 +182,8 @@ exec_config_t determine_exec_config(
|
||||
const vllm::ScalarType& a_type, const vllm::ScalarType& b_type,
|
||||
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
|
||||
int prob_n, int prob_k, int num_experts, int top_k, int thread_m_blocks,
|
||||
bool m_block_size_8, int num_bits, int group_size, bool has_act_order,
|
||||
bool is_k_full, bool has_zp, bool is_zp_float, bool is_a_8bit, int stages,
|
||||
int max_shared_mem, int sms) {
|
||||
bool m_block_size_8, int num_bits, int group_size, bool has_zp,
|
||||
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms) {
|
||||
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
||||
thread_config_t* thread_configs = thread_m_blocks > 1
|
||||
? large_batch_thread_configs
|
||||
@@ -293,27 +199,22 @@ exec_config_t determine_exec_config(
|
||||
thread_config_t th_config = thread_configs[i];
|
||||
|
||||
if (!is_valid_config(th_config, m_block_size_8, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages,
|
||||
prob_n, prob_k, num_bits, group_size, has_zp,
|
||||
is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem - 512)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int cache_size = get_kernel_cache_size(
|
||||
th_config, m_block_size_8, thread_m_blocks, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, has_act_order, is_k_full, has_zp, is_zp_float,
|
||||
is_a_8bit, stages);
|
||||
num_bits, group_size, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
|
||||
int group_blocks = 0;
|
||||
if (!has_act_order) {
|
||||
group_blocks = group_size == -1 ? -1 : (group_size / 16);
|
||||
}
|
||||
int group_blocks = group_size == -1 ? -1 : (group_size / 16);
|
||||
|
||||
auto kernel =
|
||||
get_marlin_kernel(a_type, b_type, c_type, s_type, thread_m_blocks,
|
||||
th_config.thread_n / 16, th_config.thread_k / 16,
|
||||
m_block_size_8, has_act_order, has_zp, group_blocks,
|
||||
th_config.num_threads, is_zp_float, stages);
|
||||
auto kernel = get_marlin_kernel(
|
||||
a_type, b_type, c_type, s_type, thread_m_blocks,
|
||||
th_config.thread_n / 16, th_config.thread_k / 16, m_block_size_8,
|
||||
has_zp, group_blocks, th_config.num_threads, is_zp_float, stages);
|
||||
|
||||
if (kernel == MarlinDefault) continue;
|
||||
|
||||
@@ -342,15 +243,14 @@ exec_config_t determine_exec_config(
|
||||
}
|
||||
|
||||
void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
void* a_s, void* b_s, void* g_s, void* zp, void* g_idx,
|
||||
void* perm, void* a_tmp, void* sorted_token_ids,
|
||||
void* expert_ids, void* num_tokens_past_padded,
|
||||
void* topk_weights, int moe_block_size, int num_experts,
|
||||
int top_k, bool mul_topk_weights, int prob_m, int prob_n,
|
||||
int prob_k, void* workspace, vllm::ScalarType const& a_type,
|
||||
void* a_s, void* b_s, void* g_s, void* zp,
|
||||
void* sorted_token_ids, void* expert_ids,
|
||||
void* num_tokens_past_padded, void* topk_weights,
|
||||
int moe_block_size, int num_experts, int top_k,
|
||||
bool mul_topk_weights, int prob_m, int prob_n, int prob_k,
|
||||
void* workspace, vllm::ScalarType const& a_type,
|
||||
vllm::ScalarType const& b_type, vllm::ScalarType const& c_type,
|
||||
vllm::ScalarType const& s_type, bool has_bias,
|
||||
bool has_act_order, bool is_k_full, bool has_zp, int num_groups,
|
||||
vllm::ScalarType const& s_type, bool has_bias, bool has_zp,
|
||||
int group_size, int dev, cudaStream_t stream, int thread_k,
|
||||
int thread_n, int sms, int blocks_per_sm, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float) {
|
||||
@@ -361,25 +261,13 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [",
|
||||
prob_m, ", ", prob_n, ", ", prob_k, "]");
|
||||
|
||||
int group_blocks = 0;
|
||||
if (has_act_order) {
|
||||
if (is_k_full) {
|
||||
STD_TORCH_CHECK(group_size != -1);
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
} else {
|
||||
STD_TORCH_CHECK(group_size == 0);
|
||||
group_blocks = 0;
|
||||
}
|
||||
int group_blocks;
|
||||
if (group_size == -1) {
|
||||
group_blocks = -1;
|
||||
} else {
|
||||
if (group_size == -1) {
|
||||
group_blocks = -1;
|
||||
} else {
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
}
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
}
|
||||
|
||||
int num_bits = b_type.size_bits();
|
||||
@@ -392,9 +280,6 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
const int4* b_s_ptr = (const int4*)b_s;
|
||||
const float* g_s_ptr = (const float*)g_s;
|
||||
const int4* zp_ptr = (const int4*)zp;
|
||||
const int* g_idx_ptr = (const int*)g_idx;
|
||||
const int* perm_ptr = (const int*)perm;
|
||||
int4* a_tmp_ptr = (int4*)a_tmp;
|
||||
const int32_t* sorted_token_ids_ptr = (const int32_t*)sorted_token_ids;
|
||||
const int32_t* expert_ids_ptr = (const int32_t*)expert_ids;
|
||||
const int32_t* num_tokens_past_padded_ptr =
|
||||
@@ -402,37 +287,6 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
const float* topk_weights_ptr = (const float*)topk_weights;
|
||||
int* locks = (int*)workspace;
|
||||
|
||||
if (has_act_order) {
|
||||
// Permute A columns
|
||||
auto kernel = permute_cols_kernel<8>;
|
||||
if (moe_block_size == 8) {
|
||||
} else if (moe_block_size == 16)
|
||||
kernel = permute_cols_kernel<16>;
|
||||
else if (moe_block_size == 32)
|
||||
kernel = permute_cols_kernel<32>;
|
||||
else if (moe_block_size == 48)
|
||||
kernel = permute_cols_kernel<48>;
|
||||
else if (moe_block_size == 64)
|
||||
kernel = permute_cols_kernel<64>;
|
||||
else
|
||||
STD_TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size);
|
||||
|
||||
// avoid ">>>" being formatted to "> > >"
|
||||
// clang-format off
|
||||
kernel<<<sms, default_threads, 0, stream>>>(
|
||||
A_ptr, perm_ptr, a_tmp_ptr, sorted_token_ids_ptr, expert_ids_ptr,
|
||||
num_tokens_past_padded_ptr, prob_m, prob_k, top_k);
|
||||
// clang-format on
|
||||
A_ptr = a_tmp_ptr;
|
||||
prob_m = prob_m * top_k;
|
||||
top_k = 1;
|
||||
|
||||
// If we have a full K, then we can run the non-act-order version of Marlin
|
||||
// (since the weight rows are reordered by increasing group ids, and by
|
||||
// having a full K, we have full original groups)
|
||||
if (is_k_full) has_act_order = false;
|
||||
}
|
||||
|
||||
int max_shared_mem = 0;
|
||||
cudaDeviceGetAttribute(&max_shared_mem,
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
|
||||
@@ -476,9 +330,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
// Auto config
|
||||
exec_cfg = determine_exec_config(
|
||||
a_type, b_type, c_type, s_type, prob_m, prob_n, prob_k, num_experts,
|
||||
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size,
|
||||
has_act_order, is_k_full, has_zp, is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem, sms);
|
||||
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
|
||||
is_zp_float, is_a_8bit, stages, max_shared_mem, sms);
|
||||
thread_tfg = exec_cfg.tb_cfg;
|
||||
}
|
||||
|
||||
@@ -494,36 +347,32 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
is_valid_config(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem),
|
||||
prob_n, prob_k, num_bits, group_size, has_zp, is_zp_float,
|
||||
is_a_8bit, stages, max_shared_mem),
|
||||
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_k = ", thread_tfg.thread_k,
|
||||
", thread_n = ", thread_tfg.thread_n,
|
||||
", num_threads = ", thread_tfg.num_threads, " for MKN = [", prob_m, ", ",
|
||||
prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
|
||||
", group_size = ", group_size, ", has_act_order = ", has_act_order,
|
||||
", is_k_full = ", is_k_full, ", has_zp = ", has_zp,
|
||||
", group_size = ", group_size, ", has_zp = ", has_zp,
|
||||
", is_zp_float = ", is_zp_float, ", max_shared_mem = ", max_shared_mem);
|
||||
|
||||
int sh_cache_size =
|
||||
get_kernel_cache_size(thread_tfg, m_block_size_8, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
int sh_cache_size = get_kernel_cache_size(
|
||||
thread_tfg, m_block_size_8, thread_m_blocks, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
|
||||
auto kernel = get_marlin_kernel(
|
||||
a_type, b_type, c_type, s_type, thread_m_blocks, thread_n_blocks,
|
||||
thread_k_blocks, m_block_size_8, has_act_order, has_zp, group_blocks,
|
||||
num_threads, is_zp_float, stages);
|
||||
auto kernel =
|
||||
get_marlin_kernel(a_type, b_type, c_type, s_type, thread_m_blocks,
|
||||
thread_n_blocks, thread_k_blocks, m_block_size_8,
|
||||
has_zp, group_blocks, num_threads, is_zp_float, stages);
|
||||
|
||||
if (kernel == MarlinDefault) {
|
||||
STD_TORCH_CHECK(
|
||||
false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ",
|
||||
prob_k, "]", ", has_act_order = ", has_act_order,
|
||||
", num_groups = ", num_groups, ", group_size = ", group_size,
|
||||
", thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_n_blocks = ", thread_n_blocks,
|
||||
", thread_k_blocks = ", thread_k_blocks, ", num_bits = ", num_bits);
|
||||
STD_TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
|
||||
", ", prob_k, "]", ", group_size = ", group_size,
|
||||
", thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_n_blocks = ", thread_n_blocks,
|
||||
", thread_k_blocks = ", thread_k_blocks,
|
||||
", num_bits = ", num_bits);
|
||||
}
|
||||
|
||||
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
@@ -531,10 +380,10 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
// avoid ">>>" being formatted to "> > >"
|
||||
// clang-format off
|
||||
kernel<<<blocks, num_threads, max_shared_mem, stream>>>(
|
||||
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr, g_idx_ptr,
|
||||
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
|
||||
sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr,
|
||||
topk_weights_ptr, top_k, mul_topk_weights, num_groups, prob_m,
|
||||
prob_n, prob_k, locks, has_bias, use_atomic_add, use_fp32_reduce);
|
||||
topk_weights_ptr, top_k, mul_topk_weights, prob_m, prob_n, prob_k, locks,
|
||||
has_bias, use_atomic_add, use_fp32_reduce);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
@@ -548,15 +397,13 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
std::optional<torch::stable::Tensor> const& a_scales_or_none,
|
||||
std::optional<torch::stable::Tensor> const& global_scale_or_none,
|
||||
std::optional<torch::stable::Tensor> const& b_zeros_or_none,
|
||||
std::optional<torch::stable::Tensor> const& g_idx_or_none,
|
||||
std::optional<torch::stable::Tensor> const& perm_or_none,
|
||||
torch::stable::Tensor& workspace, torch::stable::Tensor& sorted_token_ids,
|
||||
torch::stable::Tensor& expert_ids,
|
||||
torch::stable::Tensor& num_tokens_past_padded,
|
||||
torch::stable::Tensor& topk_weights, int64_t moe_block_size, int64_t top_k,
|
||||
bool mul_topk_weights, vllm::ScalarTypeId const& b_type_id, int64_t size_m,
|
||||
int64_t size_n, int64_t size_k, bool is_k_full, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float, int64_t thread_k, int64_t thread_n,
|
||||
int64_t size_n, int64_t size_k, bool use_atomic_add, bool use_fp32_reduce,
|
||||
bool is_zp_float, int64_t thread_k, int64_t thread_n,
|
||||
int64_t blocks_per_sm) {
|
||||
vllm::ScalarTypeId a_type_id, c_type_id, s_type_id;
|
||||
|
||||
@@ -717,7 +564,7 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
c_tmp = torch::stable::new_empty(a, {0}, kFloat);
|
||||
}
|
||||
|
||||
// Detect groupsize and act_order
|
||||
// Detect group size.
|
||||
int num_groups = -1;
|
||||
int group_size = -1;
|
||||
|
||||
@@ -728,50 +575,13 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
" is not size_n = ", size_n);
|
||||
num_groups = b_scales.size(1);
|
||||
|
||||
torch::stable::Tensor g_idx, perm, a_tmp;
|
||||
if (g_idx_or_none.has_value() && perm_or_none.has_value()) {
|
||||
g_idx = g_idx_or_none.value();
|
||||
perm = perm_or_none.value();
|
||||
|
||||
STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
|
||||
STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
|
||||
STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
|
||||
STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
|
||||
|
||||
// Verify g_idx and perm
|
||||
STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
|
||||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
|
||||
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
|
||||
" and perm.size(-1) = ", perm.size(-1),
|
||||
", where size_k = ", size_k);
|
||||
if (num_groups > 1) {
|
||||
STD_TORCH_CHECK(
|
||||
size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by b_scales.size(1) = ", b_scales.size(1));
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
g_idx = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
perm = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
}
|
||||
bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0;
|
||||
|
||||
if (has_act_order) {
|
||||
a_tmp = torch::stable::new_empty(a, {size_m * top_k, size_k}, c_dtype);
|
||||
if (is_k_full) {
|
||||
STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
|
||||
STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by num_groups = ", num_groups);
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
group_size = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
a_tmp = torch::stable::new_empty(a, {0}, c_dtype);
|
||||
if (num_groups > 1) {
|
||||
STD_TORCH_CHECK(
|
||||
size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by b_scales.size(1) = ", b_scales.size(1));
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
group_size = -1;
|
||||
}
|
||||
group_size = -1;
|
||||
}
|
||||
|
||||
torch::stable::Tensor global_scale;
|
||||
@@ -880,15 +690,13 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(),
|
||||
a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(),
|
||||
global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(),
|
||||
g_idx.mutable_data_ptr(), perm.mutable_data_ptr(),
|
||||
a_tmp.mutable_data_ptr(), sorted_token_ids.mutable_data_ptr(),
|
||||
expert_ids.mutable_data_ptr(), num_tokens_past_padded.mutable_data_ptr(),
|
||||
sorted_token_ids.mutable_data_ptr(), expert_ids.mutable_data_ptr(),
|
||||
num_tokens_past_padded.mutable_data_ptr(),
|
||||
topk_weights.mutable_data_ptr(), moe_block_size, num_experts, top_k,
|
||||
mul_topk_weights, size_m, size_n, size_k, workspace.mutable_data_ptr(),
|
||||
a_type, b_type, c_type, s_type, has_bias, has_act_order, is_k_full,
|
||||
has_zp, num_groups, group_size, dev, get_current_cuda_stream(dev),
|
||||
thread_k, thread_n, sms, blocks_per_sm, use_atomic_add, use_fp32_reduce,
|
||||
is_zp_float);
|
||||
a_type, b_type, c_type, s_type, has_bias, has_zp, group_size, dev,
|
||||
get_current_cuda_stream(dev), thread_k, thread_n, sms, blocks_per_sm,
|
||||
use_atomic_add, use_fp32_reduce, is_zp_float);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
@@ -79,13 +79,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
"Tensor! b_q_weight, Tensor? b_bias_or_none,"
|
||||
"Tensor! b_scales, Tensor? a_scales, Tensor? global_scale, Tensor? "
|
||||
"b_zeros_or_none,"
|
||||
"Tensor? g_idx_or_none, Tensor? perm_or_none, Tensor! workspace,"
|
||||
"Tensor! workspace,"
|
||||
"Tensor sorted_token_ids,"
|
||||
"Tensor! expert_ids, Tensor! num_tokens_past_padded,"
|
||||
"Tensor! topk_weights, int moe_block_size, int top_k, "
|
||||
"bool mul_topk_weights, int b_type_id,"
|
||||
"int size_m, int size_n, int size_k,"
|
||||
"bool is_full_k, bool use_atomic_add,"
|
||||
"int size_m, int size_n, int size_k, bool use_atomic_add,"
|
||||
"bool use_fp32_reduce, bool is_zp_float,"
|
||||
"int thread_k, int thread_n, int blocks_per_sm) -> Tensor");
|
||||
|
||||
|
||||
@@ -43,9 +43,6 @@ void per_token_group_quant_int8(const torch::stable::Tensor& input,
|
||||
int64_t group_size, double eps, double int8_min,
|
||||
double int8_max);
|
||||
|
||||
torch::stable::Tensor permute_cols(torch::stable::Tensor const& A,
|
||||
torch::stable::Tensor const& perm);
|
||||
|
||||
#ifndef USE_ROCM
|
||||
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
|
||||
bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability);
|
||||
@@ -587,11 +584,10 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
torch::stable::Tensor b_q_weight,
|
||||
torch::stable::Tensor b_gptq_qzeros,
|
||||
torch::stable::Tensor b_gptq_scales,
|
||||
torch::stable::Tensor b_g_idx, bool use_exllama,
|
||||
bool use_v2_format, int64_t bit);
|
||||
bool use_exllama, bool use_v2_format,
|
||||
int64_t bit);
|
||||
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
|
||||
int64_t bit);
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, int64_t bit);
|
||||
|
||||
// Cache ops (shared CUDA/ROCm)
|
||||
void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst,
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
static constexpr int default_threads = 256;
|
||||
static constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }
|
||||
|
||||
// For a given "a" of size [M,K] performs a permutation of the K columns based
|
||||
// on the given "perm" indices.
|
||||
// Currently only supports 16bit types (since we permute half types)
|
||||
__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
|
||||
int const* __restrict__ perm_int_ptr,
|
||||
int4* __restrict__ out_int4_ptr, int size_m,
|
||||
int size_k, int block_rows) {
|
||||
int start_row = block_rows * blockIdx.x;
|
||||
int finish_row = start_row + block_rows;
|
||||
if (finish_row > size_m) {
|
||||
finish_row = size_m;
|
||||
}
|
||||
int cur_block_rows = std::max(finish_row - start_row, 0);
|
||||
|
||||
int row_stride = size_k * sizeof(half) / 16;
|
||||
|
||||
auto permute_row = [&](int row) {
|
||||
int iters = size_k / default_threads;
|
||||
int rest = size_k % default_threads;
|
||||
|
||||
int offset = row * row_stride;
|
||||
|
||||
half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
|
||||
half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
|
||||
|
||||
int base_k = 0;
|
||||
|
||||
for (int i = 0; i < iters; i++) {
|
||||
int cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
|
||||
base_k += default_threads;
|
||||
}
|
||||
|
||||
if (rest) {
|
||||
if (threadIdx.x < rest) {
|
||||
int cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (int i = 0; i < cur_block_rows; i++) {
|
||||
int cur_row = start_row + i;
|
||||
if (cur_row < size_m) {
|
||||
permute_row(cur_row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// More efficient version of A[..., perm]
|
||||
// taken from gptq_marlin.cu
|
||||
torch::stable::Tensor permute_cols(torch::stable::Tensor const& A,
|
||||
torch::stable::Tensor const& perm) {
|
||||
const int32_t dev = A.get_device_index();
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(dev);
|
||||
const auto stream = get_current_cuda_stream(dev);
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
A.scalar_type() == torch::headeronly::ScalarType::Half ||
|
||||
A.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"Currently only 16bit types are supported");
|
||||
STD_TORCH_CHECK(A.is_contiguous(), "A must be contiguous");
|
||||
STD_TORCH_CHECK(A.size(-1) % 8 == 0,
|
||||
"A columns must be a multiple of 8 (128bits)");
|
||||
auto A_2d = torch::stable::view(A, {-1, A.size(-1)});
|
||||
|
||||
torch::stable::Tensor D = torch::stable::empty_like(A);
|
||||
int sms;
|
||||
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
|
||||
int block_rows = div_ceil(A_2d.size(0), sms);
|
||||
permute_cols_kernel<<<sms, default_threads, 0, stream>>>(
|
||||
reinterpret_cast<int4 const*>(A_2d.const_data_ptr()),
|
||||
perm.const_data_ptr<int>(), reinterpret_cast<int4*>(D.mutable_data_ptr()),
|
||||
A_2d.size(0), A_2d.size(1), block_rows);
|
||||
return D;
|
||||
}
|
||||
@@ -121,7 +121,7 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,
|
||||
const uint32_t*, const half*,
|
||||
half*, const int, const int,
|
||||
const int, const int,
|
||||
const bool, const int*);
|
||||
const bool);
|
||||
|
||||
template <bool first_block, int m_count>
|
||||
__global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
@@ -129,7 +129,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -157,12 +157,7 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,7 +258,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -291,12 +286,7 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,7 +370,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -408,12 +398,7 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -504,7 +489,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, half* __restrict__ c,
|
||||
const int size_m, const int size_n, const int size_k, const int groups,
|
||||
const bool use_v2_format, const int* __restrict__ b_q_perm) {
|
||||
const bool use_v2_format) {
|
||||
MatrixView_half a_(a, size_m, size_k);
|
||||
MatrixView_half_rw c_(c, size_m, size_n);
|
||||
MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
|
||||
@@ -532,12 +517,7 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(
|
||||
const half* a_ptr = a_.item_ptr(offset_m + m, 0);
|
||||
half* block_a_ptr = block_a[m];
|
||||
|
||||
half a0;
|
||||
if (b_q_perm)
|
||||
a0 = a_ptr[b_q_perm[offset_k + t]];
|
||||
else
|
||||
a0 = a_ptr[offset_k + t];
|
||||
block_a_ptr[t] = a0;
|
||||
block_a_ptr[t] = a_ptr[offset_k + t];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -656,10 +636,9 @@ fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(
|
||||
|
||||
void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_q_perm,
|
||||
half* c, int size_m, int size_n, int size_k,
|
||||
int m_count, int groups, bool use_v2_format,
|
||||
int bit) {
|
||||
const half* b_gptq_scales, half* c, int size_m,
|
||||
int size_n, int size_k, int m_count, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -672,13 +651,13 @@ void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,
|
||||
pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k,
|
||||
groups, use_v2_format, b_q_perm);
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,
|
||||
b_gptq_scales, c, size_m, size_n,
|
||||
size_k, groups, use_v2_format);
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_8bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -694,14 +673,8 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -722,8 +695,6 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -752,25 +723,13 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,
|
||||
zeros[3] + zero_offset);
|
||||
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -778,7 +737,7 @@ __global__ void reconstruct_exllama_8bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_4bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -794,14 +753,8 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -828,8 +781,6 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);
|
||||
dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -860,25 +811,13 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
false);
|
||||
|
||||
b_ptr += size_n;
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 4; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -886,7 +825,7 @@ __global__ void reconstruct_exllama_4bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_3bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -902,14 +841,8 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -930,8 +863,6 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -962,24 +893,13 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],
|
||||
size_n, zeros[3] + zero_offset);
|
||||
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 16; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -987,7 +907,7 @@ __global__ void reconstruct_exllama_3bit_kernel(
|
||||
}
|
||||
|
||||
__global__ void reconstruct_exllama_2bit_kernel(
|
||||
const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_gptq_qzeros,
|
||||
const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,
|
||||
const int groups, const bool use_v2_format, half* __restrict__ b) {
|
||||
@@ -1003,14 +923,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
|
||||
int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
|
||||
|
||||
// Preload remapping table
|
||||
__shared__ int perm[BLOCK_KN_SIZE];
|
||||
auto t = threadIdx.x;
|
||||
|
||||
if (b_q_perm) {
|
||||
if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t];
|
||||
}
|
||||
|
||||
// Column
|
||||
int n = offset_n + t * 4;
|
||||
if (n >= size_n) return;
|
||||
@@ -1031,8 +945,6 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
b_gptq_qzeros_.item4(zeros, group, n);
|
||||
b_gptq_scales_.item4_h2(scales, group, n);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
int k = offset_k;
|
||||
int lk = 0;
|
||||
|
||||
@@ -1055,25 +967,13 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + zero_offset);
|
||||
|
||||
b_ptr += size_n;
|
||||
// half* dqh = (half*)dq;
|
||||
if (b_q_perm) {
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]),
|
||||
__high2half(dq[2][j]), __high2half(dq[3][j]));
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]),
|
||||
__low2half(dq[1][j]), __low2half(dq[2][j]),
|
||||
__low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
for (int j = 0; j < 8; j++) {
|
||||
for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
|
||||
b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]),
|
||||
__low2half(dq[2][j]), __low2half(dq[3][j]));
|
||||
b_.set4(offset_k + lk++, n, __high2half(dq[0][j]),
|
||||
__high2half(dq[1][j]), __high2half(dq[2][j]),
|
||||
__high2half(dq[3][j]));
|
||||
}
|
||||
}
|
||||
k += 32;
|
||||
@@ -1082,9 +982,8 @@ __global__ void reconstruct_exllama_2bit_kernel(
|
||||
|
||||
void reconstruct_exllama(const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_q_perm,
|
||||
half* out, int height, int width, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
const half* b_gptq_scales, half* out, int height,
|
||||
int width, int groups, bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -1102,15 +1001,15 @@ void reconstruct_exllama(const uint32_t* b_q_weight,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,
|
||||
b_q_weight, b_gptq_qzeros, b_gptq_scales, height, width, groups,
|
||||
use_v2_format, out);
|
||||
}
|
||||
|
||||
__global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
|
||||
half* __restrict__ mul, const half* __restrict__ scales,
|
||||
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
|
||||
int batch, int height, int width, bool use_v2_format) {
|
||||
const uint32_t* __restrict__ zeros, int batch, int height, int width,
|
||||
int groups, bool use_v2_format) {
|
||||
int zero_width = width / 8;
|
||||
int vec_height = height * 4;
|
||||
const int blockwidth2 = BLOCK_KN_SIZE / 2;
|
||||
@@ -1144,6 +1043,7 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
|
||||
int i = width * h + w;
|
||||
int g_h = h * 8;
|
||||
int group_size = vec_height * 2 / groups;
|
||||
int k = 0;
|
||||
int z_w = w / 8;
|
||||
int z_mod = (w % 8) * 4;
|
||||
@@ -1156,8 +1056,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
half2 scales_tmp[4];
|
||||
half2 zeros_tmp[4];
|
||||
for (int tmp_k = 0; tmp_k < 4; tmp_k++) {
|
||||
int g = g_idx[g_h + (k + tmp_k) * 2];
|
||||
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
|
||||
int g = (g_h + (k + tmp_k) * 2) / group_size;
|
||||
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
|
||||
half scale_f = scales[g * width + w];
|
||||
half scale_f2 = scales[g2 * width + w];
|
||||
half2 scale = __halves2half2(scale_f, scale_f2);
|
||||
@@ -1209,8 +1109,8 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(
|
||||
__global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
const half2* __restrict__ vec, const uint32_t* __restrict__ mat,
|
||||
half* __restrict__ mul, const half* __restrict__ scales,
|
||||
const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,
|
||||
int batch, int height, int width, bool use_v2_format) {
|
||||
const uint32_t* __restrict__ zeros, int batch, int height, int width,
|
||||
int groups, bool use_v2_format) {
|
||||
int zero_width = width / 4;
|
||||
int vec_height = height * 2;
|
||||
const int blockwidth2 = BLOCK_KN_SIZE / 2;
|
||||
@@ -1236,6 +1136,7 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
|
||||
int i = width * h + w;
|
||||
int g_h = h * 4;
|
||||
int group_size = vec_height * 2 / groups;
|
||||
int k = 0;
|
||||
int z_w = w / 4;
|
||||
int z_mod = (w % 4) * 8;
|
||||
@@ -1248,8 +1149,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
half2 scales_tmp[2];
|
||||
half2 zeros_tmp[2];
|
||||
for (int tmp_k = 0; tmp_k < 2; tmp_k++) {
|
||||
int g = g_idx[g_h + (k + tmp_k) * 2];
|
||||
int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1];
|
||||
int g = (g_h + (k + tmp_k) * 2) / group_size;
|
||||
int g2 = (g_h + (k + tmp_k) * 2 + 1) / group_size;
|
||||
half scale_f = scales[g * width + w];
|
||||
half scale_f2 = scales[g2 * width + w];
|
||||
half2 scale = __halves2half2(scale_f, scale_f2);
|
||||
@@ -1296,8 +1197,8 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(
|
||||
|
||||
void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx,
|
||||
half* c, int size_m, int size_n, int size_k,
|
||||
const half* b_gptq_scales, half* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
@@ -1314,16 +1215,15 @@ void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(
|
||||
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,
|
||||
size_m, size_k / 32 * bit, size_n, use_v2_format);
|
||||
(const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, size_m,
|
||||
size_k / 32 * bit, size_n, groups, use_v2_format);
|
||||
}
|
||||
|
||||
template <class T, int bit>
|
||||
__global__ void reconstruct_gptq_kernel(
|
||||
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
|
||||
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
|
||||
const int height, const int width, const int group,
|
||||
const bool use_v2_format, half* __restrict__ out) {
|
||||
const uint32_t* __restrict__ w_zeros, const int height, const int width,
|
||||
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
|
||||
// Start of block
|
||||
|
||||
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
|
||||
@@ -1333,8 +1233,8 @@ __global__ void reconstruct_gptq_kernel(
|
||||
// Views
|
||||
|
||||
MatrixView_half_rw out_(out, height, width);
|
||||
MatrixView_half w_scales_(w_scales, group, width);
|
||||
T w_zeros_(w_zeros, group, width);
|
||||
MatrixView_half w_scales_(w_scales, num_groups, width);
|
||||
T w_zeros_(w_zeros, num_groups, width);
|
||||
|
||||
// GPTQv2 and GPTQv1 handles zero points differently
|
||||
int zero_offset = use_v2_format ? 0 : 1;
|
||||
@@ -1344,7 +1244,7 @@ __global__ void reconstruct_gptq_kernel(
|
||||
|
||||
#pragma unroll
|
||||
for (int s = 0; s < 32; s += bit) {
|
||||
int group = g_idx[row + s / bit];
|
||||
int group = (row + s / bit) / (height / num_groups);
|
||||
half w_scale = w_scales_.item(group, column);
|
||||
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
|
||||
half w_item =
|
||||
@@ -1357,9 +1257,8 @@ __global__ void reconstruct_gptq_kernel(
|
||||
|
||||
__global__ void reconstruct_gptq_3bit_kernel(
|
||||
const uint32_t* __restrict__ w, const half* __restrict__ w_scales,
|
||||
const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,
|
||||
const int height, const int width, const int group,
|
||||
const bool use_v2_format, half* __restrict__ out) {
|
||||
const uint32_t* __restrict__ w_zeros, const int height, const int width,
|
||||
const int num_groups, const bool use_v2_format, half* __restrict__ out) {
|
||||
// Start of block
|
||||
auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;
|
||||
auto row = blockIdx.y * 32;
|
||||
@@ -1368,8 +1267,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
// Views
|
||||
|
||||
MatrixView_half_rw out_(out, height, width);
|
||||
MatrixView_half w_scales_(w_scales, group, width);
|
||||
MatrixView_q3_row w_zeros_(w_zeros, group, width);
|
||||
MatrixView_half w_scales_(w_scales, num_groups, width);
|
||||
MatrixView_q3_row w_zeros_(w_zeros, num_groups, width);
|
||||
|
||||
// GPTQv2 and GPTQv1 handles zero points differently
|
||||
int zero_offset = use_v2_format ? 0 : 1;
|
||||
@@ -1381,7 +1280,7 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; i += 1) {
|
||||
int group = g_idx[row + i];
|
||||
int group = (row + i) / (height / num_groups);
|
||||
half w_scale = w_scales_.item(group, column);
|
||||
uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;
|
||||
int w_item;
|
||||
@@ -1402,9 +1301,8 @@ __global__ void reconstruct_gptq_3bit_kernel(
|
||||
}
|
||||
|
||||
void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx, half* out,
|
||||
int height, int width, int groups, bool use_v2_format,
|
||||
int bit) {
|
||||
const half* b_gptq_scales, half* out, int height,
|
||||
int width, int groups, bool use_v2_format, int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = BLOCK_KN_SIZE;
|
||||
blockDim.y = 1;
|
||||
@@ -1423,17 +1321,16 @@ void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,
|
||||
b_gptq_qzeros, b_g_idx, height,
|
||||
width, groups, use_v2_format, out);
|
||||
b_gptq_qzeros, height, width, groups,
|
||||
use_v2_format, out);
|
||||
}
|
||||
|
||||
void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
const uint32_t* b_q_weight,
|
||||
const uint32_t* b_gptq_qzeros,
|
||||
const half* b_gptq_scales, const int* b_g_idx,
|
||||
half* c, half* temp_dq, int size_m, int size_n,
|
||||
int size_k, int groups, bool use_exllama,
|
||||
bool use_v2_format, int bit) {
|
||||
const half* b_gptq_scales, half* c, half* temp_dq,
|
||||
int size_m, int size_n, int size_k, int groups,
|
||||
bool use_exllama, bool use_v2_format, int bit) {
|
||||
bool use_reconstruct;
|
||||
if (use_exllama) {
|
||||
use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||
|
||||
@@ -1446,11 +1343,11 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
if (use_reconstruct) {
|
||||
// Reconstruct FP16 matrix, then cuBLAS
|
||||
if (use_exllama) {
|
||||
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
temp_dq, size_k, size_n, groups, use_v2_format, bit);
|
||||
reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
|
||||
size_k, size_n, groups, use_v2_format, bit);
|
||||
} else {
|
||||
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
temp_dq, size_k, size_n, groups, use_v2_format, bit);
|
||||
reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, temp_dq,
|
||||
size_k, size_n, groups, use_v2_format, bit);
|
||||
}
|
||||
|
||||
const half alpha = __float2half(1.0f);
|
||||
@@ -1464,20 +1361,20 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,
|
||||
int last_chunk_size = size_m - last_chunk;
|
||||
|
||||
if (max_chunks) {
|
||||
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
|
||||
b_g_idx, c, last_chunk, size_n, size_k,
|
||||
BLOCK_M_SIZE_MAX, groups, use_v2_format, bit);
|
||||
gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c,
|
||||
last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX,
|
||||
groups, use_v2_format, bit);
|
||||
}
|
||||
|
||||
if (last_chunk_size) {
|
||||
gemm_half_q_half_cuda_part(
|
||||
a + last_chunk * size_k, b_q_weight, b_gptq_qzeros, b_gptq_scales,
|
||||
b_g_idx, c + last_chunk * size_n, last_chunk_size, size_n, size_k,
|
||||
c + last_chunk * size_n, last_chunk_size, size_n, size_k,
|
||||
last_chunk_size, groups, use_v2_format, bit);
|
||||
}
|
||||
} else {
|
||||
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,
|
||||
c, size_m, size_n, size_k, use_v2_format, bit);
|
||||
gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m,
|
||||
size_n, size_k, groups, use_v2_format, bit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1533,214 +1430,8 @@ __global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight,
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void make_sequential_4bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 3;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 3;
|
||||
int w2_subrow = source_row & 0x07;
|
||||
int w2_row_shift = w2_subrow << 2;
|
||||
int wnew2_row_shift = i << 2;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x0000000f0000000f;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
__global__ void make_sequential_2bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 4;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 4;
|
||||
int w2_subrow = source_row & 0x0f;
|
||||
int w2_row_shift = w2_subrow << 1;
|
||||
int wnew2_row_shift = i << 1;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x0000000300000003;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
__global__ void make_sequential_3bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
auto w_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w_column >= w_width) return;
|
||||
auto w_new_row = blockIdx.y * 3;
|
||||
auto q_perm_idx = blockIdx.y << 5;
|
||||
uint32_t dst[3] = {0, 0, 0};
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
int z_w = (source_row / 32) * 3;
|
||||
int z_mod = source_row % 32;
|
||||
int z_bit;
|
||||
|
||||
if (z_mod != 10) {
|
||||
if (z_mod != 21) {
|
||||
z_bit = z_mod;
|
||||
if (z_bit > 21) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 64;
|
||||
z_w += 2;
|
||||
} else if (z_bit > 10) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 32;
|
||||
z_w += 1;
|
||||
} else {
|
||||
z_bit *= 3;
|
||||
}
|
||||
} else {
|
||||
z_w += 1;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t src;
|
||||
if (z_mod == 10) {
|
||||
src = (w[z_w * w_width + w_column] >> 30) |
|
||||
((w[(z_w + 1) * w_width + w_column] << 2) & 0x4);
|
||||
} else if (z_mod == 21) {
|
||||
src = (w[z_w * w_width + w_column] >> 31) |
|
||||
((w[(z_w + 1) * w_width + w_column] << 1) & 0x6);
|
||||
} else {
|
||||
src = w[z_w * w_width + w_column];
|
||||
src >>= z_bit;
|
||||
src &= 0x07;
|
||||
}
|
||||
|
||||
z_w = 0;
|
||||
if (i != 10) {
|
||||
if (i != 21) {
|
||||
z_bit = i;
|
||||
if (z_bit > 21) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 64;
|
||||
z_w += 2;
|
||||
} else if (z_bit > 10) {
|
||||
z_bit *= 3;
|
||||
z_bit -= 32;
|
||||
z_w += 1;
|
||||
} else {
|
||||
z_bit *= 3;
|
||||
}
|
||||
} else {
|
||||
z_w += 1;
|
||||
}
|
||||
}
|
||||
if (i == 10) {
|
||||
dst[z_w] |= (src & 0x03) << 30;
|
||||
dst[z_w + 1] |= ((src & 0x4) >> 2);
|
||||
} else if (i == 21) {
|
||||
dst[z_w] |= (src & 0x01) << 31;
|
||||
dst[z_w + 1] |= ((src & 0x6) >> 1);
|
||||
} else {
|
||||
dst[z_w] |= (src << z_bit);
|
||||
}
|
||||
}
|
||||
w_new[w_new_row * w_width + w_column] = dst[0];
|
||||
w_new[(w_new_row + 1) * w_width + w_column] = dst[1];
|
||||
w_new[(w_new_row + 2) * w_width + w_column] = dst[2];
|
||||
}
|
||||
|
||||
__global__ void make_sequential_8bit_kernel(const uint32_t* __restrict__ w,
|
||||
uint32_t* __restrict__ w_new,
|
||||
const int* __restrict__ q_perm,
|
||||
const int w_width) {
|
||||
const uint64_t* w2 = (uint64_t*)w;
|
||||
uint64_t* w_new2 = (uint64_t*)w_new;
|
||||
int w2_stride = w_width >> 1;
|
||||
auto w2_column = THREADS_X * blockIdx.x + threadIdx.x;
|
||||
if (w2_column >= w2_stride) return;
|
||||
auto w_new2_row = blockIdx.y;
|
||||
int q_perm_idx = w_new2_row << 2;
|
||||
uint64_t dst = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int source_row = q_perm[q_perm_idx++];
|
||||
|
||||
int w2_row = source_row >> 2;
|
||||
int w2_subrow = source_row & 0x03;
|
||||
int w2_row_shift = w2_subrow << 3;
|
||||
int wnew2_row_shift = i << 3;
|
||||
|
||||
uint64_t src = w2[w2_row * w2_stride + w2_column];
|
||||
src >>= w2_row_shift;
|
||||
src &= 0x000000ff000000ff;
|
||||
src <<= wnew2_row_shift;
|
||||
dst |= src;
|
||||
}
|
||||
w_new2[w_new2_row * w2_stride + w2_column] = dst;
|
||||
}
|
||||
|
||||
void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,
|
||||
int width, int bit) {
|
||||
if (q_perm) {
|
||||
uint32_t* new_qweight = NULL;
|
||||
cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t));
|
||||
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = THREADS_X;
|
||||
blockDim.y = 1;
|
||||
gridDim.x = DIVIDE(width, THREADS_X);
|
||||
gridDim.y = height / 32 * bit;
|
||||
|
||||
auto kernel = make_sequential_4bit_kernel;
|
||||
if (bit == 2) {
|
||||
kernel = make_sequential_2bit_kernel;
|
||||
} else if (bit == 3) {
|
||||
kernel = make_sequential_3bit_kernel;
|
||||
gridDim.y = height / 32;
|
||||
} else if (bit == 8) {
|
||||
kernel = make_sequential_8bit_kernel;
|
||||
}
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, new_qweight, q_perm,
|
||||
width);
|
||||
// Replace qweights
|
||||
cudaMemcpyAsync(q_weight, new_qweight,
|
||||
height / 32 * bit * width * sizeof(uint32_t),
|
||||
cudaMemcpyDeviceToDevice);
|
||||
// Cleanup
|
||||
cudaDeviceSynchronize();
|
||||
cudaFree(new_qweight);
|
||||
}
|
||||
void shuffle_exllama_weight(uint32_t* q_weight, int height, int width,
|
||||
int bit) {
|
||||
dim3 blockDim, gridDim;
|
||||
blockDim.x = THREADS_X;
|
||||
blockDim.y = 1;
|
||||
@@ -1765,10 +1456,17 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
torch::stable::Tensor b_q_weight,
|
||||
torch::stable::Tensor b_gptq_qzeros,
|
||||
torch::stable::Tensor b_gptq_scales,
|
||||
torch::stable::Tensor b_g_idx, bool use_exllama,
|
||||
bool use_v2_format, int64_t bit) {
|
||||
bool use_exllama, bool use_v2_format,
|
||||
int64_t bit) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a.get_device_index());
|
||||
const int64_t num_groups = b_gptq_qzeros.size(0);
|
||||
STD_TORCH_CHECK(num_groups > 0, "GPTQ requires at least one scale group");
|
||||
STD_TORCH_CHECK(b_gptq_scales.size(0) == num_groups,
|
||||
"GPTQ scales and zero-points must have the same number of "
|
||||
"groups");
|
||||
STD_TORCH_CHECK(a.size(1) % num_groups == 0, "Input size K = ", a.size(1),
|
||||
" is not divisible by the number of groups = ", num_groups);
|
||||
auto c = torch::stable::new_zeros(a, {a.size(0), b_q_weight.size(1)});
|
||||
auto temp_dq =
|
||||
torch::stable::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)},
|
||||
@@ -1778,28 +1476,19 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
get_current_cuda_blas_handle(), (const half*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_gptq_qzeros.data_ptr(),
|
||||
(const half*)b_gptq_scales.data_ptr(),
|
||||
b_g_idx.device().type() == torch::stable::DeviceType::Meta
|
||||
? NULL
|
||||
: (const int*)b_g_idx.data_ptr(),
|
||||
(half*)c.data_ptr(), (half*)temp_dq.data_ptr(),
|
||||
c.size(0), // m
|
||||
c.size(1), // n
|
||||
a.size(1), // k
|
||||
b_gptq_qzeros.size(0), // group number
|
||||
use_exllama, use_v2_format, bit);
|
||||
(const half*)b_gptq_scales.data_ptr(), (half*)c.data_ptr(),
|
||||
(half*)temp_dq.data_ptr(),
|
||||
c.size(0), // m
|
||||
c.size(1), // n
|
||||
a.size(1), // k
|
||||
num_groups, use_exllama, use_v2_format, bit);
|
||||
return c;
|
||||
}
|
||||
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
|
||||
int64_t bit) {
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, int64_t bit) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
q_weight.get_device_index());
|
||||
vllm::gptq::shuffle_exllama_weight(
|
||||
(uint32_t*)q_weight.data_ptr(),
|
||||
q_perm.device().type() == torch::stable::DeviceType::Meta ||
|
||||
q_perm.numel() == 0
|
||||
? NULL
|
||||
: (int*)q_perm.data_ptr(),
|
||||
q_weight.size(0) * 32 / bit, q_weight.size(1), bit);
|
||||
vllm::gptq::shuffle_exllama_weight((uint32_t*)q_weight.data_ptr(),
|
||||
q_weight.size(0) * 32 / bit,
|
||||
q_weight.size(1), bit);
|
||||
}
|
||||
|
||||
@@ -75,14 +75,14 @@ QUANT_CONFIGS = [
|
||||
"b_type": "kU4B8",
|
||||
"thread_configs": THREAD_CONFIGS,
|
||||
"thread_m_blocks": THREAD_M_BLOCKS,
|
||||
"group_blocks": [-1, 0, 2, 4, 8],
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": "kU8B128",
|
||||
"thread_configs": THREAD_CONFIGS,
|
||||
"thread_m_blocks": THREAD_M_BLOCKS,
|
||||
"group_blocks": [-1, 0, 2, 4, 8],
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
{
|
||||
|
||||
@@ -11,11 +11,9 @@
|
||||
|
||||
namespace marlin {
|
||||
|
||||
template <int const num_threads, int const num_bits, bool const has_perm,
|
||||
bool is_a_8bit>
|
||||
template <int const num_threads, int const num_bits, bool is_a_8bit>
|
||||
__global__ void gptq_marlin_repack_kernel(
|
||||
uint32_t const* __restrict__ b_q_weight_ptr,
|
||||
uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
|
||||
uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
|
||||
int size_k, int size_n) {
|
||||
constexpr int pack_factor = 32 / num_bits;
|
||||
|
||||
@@ -44,31 +42,14 @@ __global__ void gptq_marlin_repack_kernel(
|
||||
|
||||
extern __shared__ int4 sh[];
|
||||
|
||||
constexpr int perm_size = target_tile_k_size / 4;
|
||||
|
||||
int4* sh_perm_ptr = sh;
|
||||
int4* sh_pipe_ptr = sh_perm_ptr;
|
||||
if constexpr (has_perm) {
|
||||
sh_pipe_ptr += perm_size;
|
||||
}
|
||||
int4* sh_pipe_ptr = sh;
|
||||
|
||||
constexpr int tile_ints = target_tile_k_size / pack_factor;
|
||||
|
||||
constexpr int stage_n_threads = target_tile_n_size / 4;
|
||||
constexpr int stage_k_threads = has_perm ? target_tile_k_size : tile_ints;
|
||||
constexpr int stage_k_threads = tile_ints;
|
||||
constexpr int stage_size = stage_k_threads * stage_n_threads;
|
||||
|
||||
auto load_perm_to_shared = [&](int k_tile_id) {
|
||||
int first_k_int4 = (k_tile_id * target_tile_k_size) / 4;
|
||||
|
||||
int4 const* perm_int4_ptr = reinterpret_cast<int4 const*>(perm_ptr);
|
||||
|
||||
if (threadIdx.x < perm_size) {
|
||||
sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x];
|
||||
}
|
||||
__syncthreads();
|
||||
};
|
||||
|
||||
auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) {
|
||||
if (n_tile_id >= n_tiles) {
|
||||
cp_async_fence();
|
||||
@@ -79,36 +60,17 @@ __global__ void gptq_marlin_repack_kernel(
|
||||
|
||||
int4* sh_ptr = sh_pipe_ptr + stage_size * pipe;
|
||||
|
||||
if constexpr (has_perm) {
|
||||
if (threadIdx.x < stage_size) {
|
||||
auto k_id = threadIdx.x / stage_n_threads;
|
||||
auto n_id = threadIdx.x % stage_n_threads;
|
||||
if (threadIdx.x < stage_size) {
|
||||
auto k_id = threadIdx.x / stage_n_threads;
|
||||
auto n_id = threadIdx.x % stage_n_threads;
|
||||
|
||||
uint32_t const* sh_perm_int_ptr =
|
||||
reinterpret_cast<uint32_t const*>(sh_perm_ptr);
|
||||
int first_k = k_tile_id * target_tile_k_size;
|
||||
int first_k_packed = first_k / pack_factor;
|
||||
|
||||
int src_k = sh_perm_int_ptr[k_id];
|
||||
int src_k_packed = src_k / pack_factor;
|
||||
|
||||
cp_async4(
|
||||
&sh_ptr[k_id * stage_n_threads + n_id],
|
||||
reinterpret_cast<int4 const*>(&(
|
||||
b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)])));
|
||||
}
|
||||
|
||||
} else {
|
||||
if (threadIdx.x < stage_size) {
|
||||
auto k_id = threadIdx.x / stage_n_threads;
|
||||
auto n_id = threadIdx.x % stage_n_threads;
|
||||
|
||||
int first_k = k_tile_id * target_tile_k_size;
|
||||
int first_k_packed = first_k / pack_factor;
|
||||
|
||||
cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
|
||||
reinterpret_cast<int4 const*>(
|
||||
&(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
|
||||
first_n + (n_id * 4)])));
|
||||
}
|
||||
cp_async4(&sh_ptr[k_id * stage_n_threads + n_id],
|
||||
reinterpret_cast<int4 const*>(
|
||||
&(b_q_weight_ptr[(first_k_packed + k_id) * size_n +
|
||||
first_n + (n_id * 4)])));
|
||||
}
|
||||
|
||||
cp_async_fence();
|
||||
@@ -139,56 +101,34 @@ __global__ void gptq_marlin_repack_kernel(
|
||||
int4* sh_stage_ptr = sh_pipe_ptr + stage_size * pipe;
|
||||
uint32_t* sh_stage_int_ptr = reinterpret_cast<uint32_t*>(sh_stage_ptr);
|
||||
|
||||
uint32_t* sh_perm_int_ptr = reinterpret_cast<uint32_t*>(sh_perm_ptr);
|
||||
|
||||
uint32_t vals[8];
|
||||
|
||||
if constexpr (has_perm) {
|
||||
static_assert(!is_a_8bit);
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int k_idx = tc_row + tc_offsets[i];
|
||||
|
||||
uint32_t src_k = sh_perm_int_ptr[k_idx];
|
||||
uint32_t src_k_pos = src_k % pack_factor;
|
||||
|
||||
uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n];
|
||||
uint32_t b1_cur_val = (b1_val >> (src_k_pos * num_bits)) & mask;
|
||||
|
||||
uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8];
|
||||
uint32_t b2_cur_val = (b2_val >> (src_k_pos * num_bits)) & mask;
|
||||
|
||||
vals[i] = b1_cur_val;
|
||||
vals[4 + i] = b2_cur_val;
|
||||
}
|
||||
|
||||
} else {
|
||||
uint32_t b1_vals[tile_ints];
|
||||
uint32_t b2_vals[tile_ints];
|
||||
uint32_t b1_vals[tile_ints];
|
||||
uint32_t b2_vals[tile_ints];
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_ints; i++) {
|
||||
if constexpr (is_a_8bit) {
|
||||
b1_vals[i] =
|
||||
sh_stage_int_ptr[cur_n + sh_stride * i + (warp_id % 2) * 8];
|
||||
} else {
|
||||
b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
|
||||
b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
|
||||
}
|
||||
for (int i = 0; i < tile_ints; i++) {
|
||||
if constexpr (is_a_8bit) {
|
||||
b1_vals[i] =
|
||||
sh_stage_int_ptr[cur_n + sh_stride * i + (warp_id % 2) * 8];
|
||||
} else {
|
||||
b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
|
||||
b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int cur_elem = tc_row + (is_a_8bit ? i : tc_offsets[i]);
|
||||
int cur_int = cur_elem / pack_factor;
|
||||
int cur_pos = cur_elem % pack_factor;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int cur_elem = tc_row + (is_a_8bit ? i : tc_offsets[i]);
|
||||
int cur_int = cur_elem / pack_factor;
|
||||
int cur_pos = cur_elem % pack_factor;
|
||||
|
||||
vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask;
|
||||
if constexpr (is_a_8bit)
|
||||
vals[4 + i] =
|
||||
(b1_vals[cur_int + tile_ints / 2] >> (cur_pos * num_bits)) & mask;
|
||||
else
|
||||
vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask;
|
||||
}
|
||||
vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask;
|
||||
if constexpr (is_a_8bit)
|
||||
vals[4 + i] =
|
||||
(b1_vals[cur_int + tile_ints / 2] >> (cur_pos * num_bits)) & mask;
|
||||
else
|
||||
vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask;
|
||||
}
|
||||
|
||||
constexpr int tile_size =
|
||||
@@ -248,10 +188,6 @@ __global__ void gptq_marlin_repack_kernel(
|
||||
for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
|
||||
int n_tile_id = 0;
|
||||
|
||||
if constexpr (has_perm) {
|
||||
load_perm_to_shared(k_tile_id);
|
||||
}
|
||||
|
||||
start_pipes(k_tile_id, n_tile_id);
|
||||
|
||||
while (n_tile_id < n_tiles) {
|
||||
@@ -269,21 +205,19 @@ __global__ void gptq_marlin_repack_kernel(
|
||||
|
||||
} // namespace marlin
|
||||
|
||||
#define CALL_IF(NUM_BITS, HAS_PERM, IS_A_8BIT) \
|
||||
else if (num_bits == NUM_BITS && has_perm == HAS_PERM && \
|
||||
is_a_8bit == IS_A_8BIT) { \
|
||||
#define CALL_IF(NUM_BITS, IS_A_8BIT) \
|
||||
else if (num_bits == NUM_BITS && is_a_8bit == IS_A_8BIT) { \
|
||||
cudaFuncSetAttribute( \
|
||||
marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
|
||||
HAS_PERM, IS_A_8BIT>, \
|
||||
IS_A_8BIT>, \
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
|
||||
marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
|
||||
HAS_PERM, IS_A_8BIT> \
|
||||
IS_A_8BIT> \
|
||||
<<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
|
||||
b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
|
||||
b_q_weight_ptr, out_ptr, size_k, size_n); \
|
||||
}
|
||||
|
||||
torch::stable::Tensor gptq_marlin_repack(torch::stable::Tensor& b_q_weight,
|
||||
torch::stable::Tensor& perm,
|
||||
int64_t size_k, int64_t size_n,
|
||||
int64_t num_bits, bool is_a_8bit) {
|
||||
// Verify compatibility with marlin tile of 16x64
|
||||
@@ -311,11 +245,6 @@ torch::stable::Tensor gptq_marlin_repack(torch::stable::Tensor& b_q_weight,
|
||||
b_q_weight.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"b_q_weight type is not kInt");
|
||||
|
||||
STD_TORCH_CHECK(perm.is_cuda(), "perm is not on GPU");
|
||||
STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
|
||||
STD_TORCH_CHECK(perm.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"perm type is not at::kInt");
|
||||
|
||||
const int32_t device_index = b_q_weight.get_device_index();
|
||||
torch::stable::accelerator::DeviceGuard device_guard(device_index);
|
||||
const cudaStream_t stream = get_current_cuda_stream(device_index);
|
||||
@@ -325,14 +254,9 @@ torch::stable::Tensor gptq_marlin_repack(torch::stable::Tensor& b_q_weight,
|
||||
{size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
|
||||
b_q_weight.scalar_type(), std::nullopt, b_q_weight.device());
|
||||
|
||||
// Detect if there is act_order
|
||||
bool has_perm = perm.size(0) != 0;
|
||||
|
||||
// Get ptrs
|
||||
uint32_t const* b_q_weight_ptr =
|
||||
reinterpret_cast<uint32_t const*>(b_q_weight.const_data_ptr());
|
||||
uint32_t const* perm_ptr =
|
||||
reinterpret_cast<uint32_t const*>(perm.const_data_ptr());
|
||||
uint32_t* out_ptr = reinterpret_cast<uint32_t*>(out.mutable_data_ptr());
|
||||
|
||||
int blocks;
|
||||
@@ -345,17 +269,15 @@ torch::stable::Tensor gptq_marlin_repack(torch::stable::Tensor& b_q_weight,
|
||||
|
||||
if (false) {
|
||||
}
|
||||
CALL_IF(4, false, false)
|
||||
CALL_IF(4, true, false)
|
||||
CALL_IF(8, false, false)
|
||||
CALL_IF(8, true, false)
|
||||
CALL_IF(4, false)
|
||||
CALL_IF(8, false)
|
||||
|
||||
CALL_IF(4, false, true)
|
||||
CALL_IF(8, false, true)
|
||||
CALL_IF(4, true)
|
||||
CALL_IF(8, true)
|
||||
|
||||
else {
|
||||
STD_TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits,
|
||||
", has_perm = ", has_perm, ", is_a_8bit = ", is_a_8bit);
|
||||
", is_a_8bit = ", is_a_8bit);
|
||||
}
|
||||
|
||||
return out;
|
||||
|
||||
@@ -7,17 +7,16 @@
|
||||
#include "marlin_dtypes.cuh"
|
||||
#include "core/scalar_type.hpp"
|
||||
|
||||
#define MARLIN_KERNEL_PARAMS \
|
||||
const int4 *__restrict__ A, const int4 *__restrict__ B, \
|
||||
int4 *__restrict__ C, int4 *__restrict__ C_tmp, \
|
||||
const int4 *__restrict__ b_bias_ptr, \
|
||||
const float *__restrict__ a_scales_ptr, \
|
||||
const int4 *__restrict__ scales_ptr, \
|
||||
const float *__restrict__ global_scale_ptr, \
|
||||
const int4 *__restrict__ zp_ptr, const int *__restrict__ g_idx, \
|
||||
int num_groups, int prob_m, int prob_n, int prob_k, int lda, int *locks, \
|
||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce, \
|
||||
int max_shared_mem
|
||||
#define MARLIN_KERNEL_PARAMS \
|
||||
const int4 *__restrict__ A, const int4 *__restrict__ B, \
|
||||
int4 *__restrict__ C, int4 *__restrict__ C_tmp, \
|
||||
const int4 *__restrict__ b_bias_ptr, \
|
||||
const float *__restrict__ a_scales_ptr, \
|
||||
const int4 *__restrict__ scales_ptr, \
|
||||
const float *__restrict__ global_scale_ptr, \
|
||||
const int4 *__restrict__ zp_ptr, int prob_m, int prob_n, int prob_k, \
|
||||
int lda, int *locks, bool has_bias, bool use_atomic_add, \
|
||||
bool use_fp32_reduce, int max_shared_mem
|
||||
|
||||
namespace MARLIN_NAMESPACE_NAME {
|
||||
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
|
||||
|
||||
@@ -47,11 +47,6 @@ using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS);
|
||||
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 750
|
||||
|
||||
__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
|
||||
int const* __restrict__ perm_int_ptr,
|
||||
int4* __restrict__ out_int4_ptr, int size_m,
|
||||
int size_k, int lda, int block_rows) {}
|
||||
|
||||
} // namespace marlin
|
||||
|
||||
torch::stable::Tensor marlin_gemm(
|
||||
@@ -62,11 +57,9 @@ torch::stable::Tensor marlin_gemm(
|
||||
std::optional<torch::stable::Tensor> const& a_scales_or_none,
|
||||
std::optional<torch::stable::Tensor> const& global_scale_or_none,
|
||||
std::optional<torch::stable::Tensor> const& b_zeros_or_none,
|
||||
std::optional<torch::stable::Tensor> const& g_idx_or_none,
|
||||
std::optional<torch::stable::Tensor> const& perm_or_none,
|
||||
torch::stable::Tensor& workspace, vllm::ScalarTypeId const& b_type_id,
|
||||
int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
|
||||
bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) {
|
||||
int64_t size_m, int64_t size_n, int64_t size_k, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float) {
|
||||
STD_TORCH_CHECK_NOT_IMPLEMENTED(false,
|
||||
"marlin_gemm(..) requires CUDA_ARCH >= 7.5");
|
||||
return torch::stable::empty({1, 1});
|
||||
@@ -74,62 +67,6 @@ torch::stable::Tensor marlin_gemm(
|
||||
|
||||
#else
|
||||
|
||||
// For a given "a" of size [M,K] performs a permutation of the K columns based
|
||||
// on the given "perm" indices.
|
||||
__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
|
||||
int const* __restrict__ perm_int_ptr,
|
||||
int4* __restrict__ out_int4_ptr, int size_m,
|
||||
int size_k, int lda, int block_rows) {
|
||||
auto start_row = block_rows * blockIdx.x;
|
||||
int finish_row = start_row + block_rows;
|
||||
if (finish_row > size_m) {
|
||||
finish_row = size_m;
|
||||
}
|
||||
int cur_block_rows = finish_row - start_row;
|
||||
|
||||
int input_row_stride = lda * sizeof(half) / 16;
|
||||
int output_row_stride = size_k * sizeof(half) / 16;
|
||||
|
||||
auto permute_row = [&](int row) {
|
||||
int iters = size_k / default_threads;
|
||||
int rest = size_k % default_threads;
|
||||
|
||||
int input_offset = row * input_row_stride;
|
||||
int output_offset = row * output_row_stride;
|
||||
|
||||
half const* a_row_half =
|
||||
reinterpret_cast<half const*>(a_int4_ptr + input_offset);
|
||||
half* out_half = reinterpret_cast<half*>(out_int4_ptr + output_offset);
|
||||
|
||||
int base_k = 0;
|
||||
|
||||
for (int i = 0; i < iters; i++) {
|
||||
auto cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
|
||||
base_k += default_threads;
|
||||
}
|
||||
|
||||
if (rest) {
|
||||
if (threadIdx.x < rest) {
|
||||
auto cur_k = base_k + threadIdx.x;
|
||||
int src_pos = perm_int_ptr[cur_k];
|
||||
|
||||
out_half[cur_k] = a_row_half[src_pos];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (int i = 0; i < cur_block_rows; i++) {
|
||||
int cur_row = start_row + i;
|
||||
if (cur_row < size_m) {
|
||||
permute_row(cur_row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int thread_k;
|
||||
int thread_n;
|
||||
@@ -159,9 +96,7 @@ typedef struct {
|
||||
|
||||
int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
|
||||
int prob_n, int prob_k, int num_bits, int group_size,
|
||||
bool has_act_order, bool is_k_full, int stages) {
|
||||
bool cache_scales_chunk = has_act_order && !is_k_full;
|
||||
|
||||
int stages) {
|
||||
int tb_n = th_config.thread_n;
|
||||
int tb_k = th_config.thread_k;
|
||||
|
||||
@@ -169,29 +104,18 @@ int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
|
||||
int tb_groups;
|
||||
if (group_size == -1) {
|
||||
tb_groups = 1;
|
||||
} else if (group_size == 0) {
|
||||
tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size
|
||||
} else {
|
||||
tb_groups = div_ceil(tb_k, group_size);
|
||||
}
|
||||
|
||||
if (cache_scales_chunk) {
|
||||
int load_groups =
|
||||
tb_groups * stages * 2; // Chunk size is 2x pipeline over dim K
|
||||
load_groups = max(load_groups, 32); // We load at least 32 scale groups
|
||||
return load_groups * tb_n * 2;
|
||||
} else {
|
||||
int tb_scales = tb_groups * tb_n * 2;
|
||||
|
||||
return tb_scales * stages;
|
||||
}
|
||||
int tb_scales = tb_groups * tb_n * 2;
|
||||
return tb_scales * stages;
|
||||
}
|
||||
|
||||
int get_kernel_cache_size(thread_config_t const& th_config, int thread_m_blocks,
|
||||
int prob_m, int prob_n, int prob_k, int num_bits,
|
||||
int group_size, bool has_act_order, bool is_k_full,
|
||||
int has_zp, bool is_zp_float, bool is_a_8bit,
|
||||
int stages) {
|
||||
int group_size, int has_zp, bool is_zp_float,
|
||||
bool is_a_8bit, int stages) {
|
||||
int pack_factor = 32 / num_bits;
|
||||
|
||||
// Get B size
|
||||
@@ -206,10 +130,8 @@ int get_kernel_cache_size(thread_config_t const& th_config, int thread_m_blocks,
|
||||
(sh_b_size > sh_red_size ? sh_red_size : sh_b_size) + sh_bias_size;
|
||||
tmp_size = max(max(sh_b_size, sh_red_size), tmp_size);
|
||||
|
||||
int sh_s_size =
|
||||
get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits,
|
||||
group_size, has_act_order, is_k_full, stages);
|
||||
int sh_g_idx_size = has_act_order && !is_k_full ? stages * tb_k / 4 : 0;
|
||||
int sh_s_size = get_scales_cache_size(th_config, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, stages);
|
||||
int sh_zp_size = 0;
|
||||
if (has_zp) {
|
||||
if (is_zp_float)
|
||||
@@ -220,17 +142,15 @@ int get_kernel_cache_size(thread_config_t const& th_config, int thread_m_blocks,
|
||||
sh_zp_size = sh_s_size / 2;
|
||||
}
|
||||
|
||||
int total_size =
|
||||
tmp_size + sh_a_size + sh_s_size + sh_zp_size + sh_g_idx_size;
|
||||
int total_size = tmp_size + sh_a_size + sh_s_size + sh_zp_size;
|
||||
|
||||
return total_size;
|
||||
}
|
||||
|
||||
bool is_valid_config(thread_config_t const& th_config, int thread_m_blocks,
|
||||
int prob_m, int prob_n, int prob_k, int num_bits,
|
||||
int group_size, bool has_act_order, bool is_k_full,
|
||||
int has_zp, bool is_zp_float, bool is_a_8bit, int stages,
|
||||
int max_shared_mem) {
|
||||
int group_size, int has_zp, bool is_zp_float,
|
||||
bool is_a_8bit, int stages, int max_shared_mem) {
|
||||
// Sanity
|
||||
if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
|
||||
th_config.num_threads == -1) {
|
||||
@@ -255,16 +175,18 @@ bool is_valid_config(thread_config_t const& th_config, int thread_m_blocks,
|
||||
// Check that pipeline fits into cache
|
||||
int cache_size = get_kernel_cache_size(
|
||||
th_config, thread_m_blocks, prob_m, prob_n, prob_k, num_bits, group_size,
|
||||
has_act_order, is_k_full, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
has_zp, is_zp_float, is_a_8bit, stages);
|
||||
return cache_size <= max_shared_mem;
|
||||
}
|
||||
|
||||
MarlinFuncPtr get_marlin_kernel(
|
||||
const vllm::ScalarType a_type, const vllm::ScalarType b_type,
|
||||
const vllm::ScalarType c_type, const vllm::ScalarType s_type,
|
||||
int thread_m_blocks, int thread_n_blocks, int thread_k_blocks,
|
||||
bool m_block_size_8, bool has_act_order, bool has_zp, int group_blocks,
|
||||
int threads, bool is_zp_float, int stages) {
|
||||
MarlinFuncPtr get_marlin_kernel(const vllm::ScalarType a_type,
|
||||
const vllm::ScalarType b_type,
|
||||
const vllm::ScalarType c_type,
|
||||
const vllm::ScalarType s_type,
|
||||
int thread_m_blocks, int thread_n_blocks,
|
||||
int thread_k_blocks, bool m_block_size_8,
|
||||
bool has_zp, int group_blocks, int threads,
|
||||
bool is_zp_float, int stages) {
|
||||
int num_bits = b_type.size_bits();
|
||||
auto kernel = MarlinDefault;
|
||||
|
||||
@@ -277,9 +199,8 @@ exec_config_t determine_exec_config(
|
||||
const vllm::ScalarType& a_type, const vllm::ScalarType& b_type,
|
||||
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
|
||||
int prob_n, int prob_k, int thread_m_blocks, bool m_block_size_8,
|
||||
int num_bits, int group_size, bool has_act_order, bool is_k_full,
|
||||
bool has_zp, bool is_zp_float, int is_a_8bit, int stages,
|
||||
int max_shared_mem, int sms) {
|
||||
int num_bits, int group_size, bool has_zp, bool is_zp_float, int is_a_8bit,
|
||||
int stages, int max_shared_mem, int sms) {
|
||||
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
||||
thread_config_t* thread_configs = thread_m_blocks > 1
|
||||
? large_batch_thread_configs
|
||||
@@ -293,27 +214,21 @@ exec_config_t determine_exec_config(
|
||||
thread_config_t th_config = thread_configs[i];
|
||||
|
||||
if (!is_valid_config(th_config, thread_m_blocks, prob_m, prob_n, prob_k,
|
||||
num_bits, group_size, has_act_order, is_k_full, has_zp,
|
||||
is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem - 512)) {
|
||||
num_bits, group_size, has_zp, is_zp_float, is_a_8bit,
|
||||
stages, max_shared_mem - 512)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int cache_size = get_kernel_cache_size(th_config, thread_m_blocks, prob_m,
|
||||
prob_n, prob_k, num_bits, group_size,
|
||||
has_act_order, is_k_full, has_zp,
|
||||
is_zp_float, is_a_8bit, stages);
|
||||
int cache_size = get_kernel_cache_size(
|
||||
th_config, thread_m_blocks, prob_m, prob_n, prob_k, num_bits,
|
||||
group_size, has_zp, is_zp_float, is_a_8bit, stages);
|
||||
|
||||
int group_blocks = 0;
|
||||
if (!has_act_order) {
|
||||
group_blocks = group_size == -1 ? -1 : group_size / 16;
|
||||
}
|
||||
int group_blocks = group_size == -1 ? -1 : group_size / 16;
|
||||
|
||||
auto kernel =
|
||||
get_marlin_kernel(a_type, b_type, c_type, s_type, thread_m_blocks,
|
||||
th_config.thread_n / 16, th_config.thread_k / 16,
|
||||
m_block_size_8, has_act_order, has_zp, group_blocks,
|
||||
th_config.num_threads, is_zp_float, stages);
|
||||
auto kernel = get_marlin_kernel(
|
||||
a_type, b_type, c_type, s_type, thread_m_blocks,
|
||||
th_config.thread_n / 16, th_config.thread_k / 16, m_block_size_8,
|
||||
has_zp, group_blocks, th_config.num_threads, is_zp_float, stages);
|
||||
|
||||
if (kernel == MarlinDefault) continue;
|
||||
|
||||
@@ -324,38 +239,25 @@ exec_config_t determine_exec_config(
|
||||
}
|
||||
|
||||
void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
void* a_s, void* b_s, void* g_s, void* zp, void* g_idx,
|
||||
void* perm, void* a_tmp, int prob_m, int prob_n, int prob_k,
|
||||
int lda, void* workspace, vllm::ScalarType const& a_type,
|
||||
vllm::ScalarType const& b_type, vllm::ScalarType const& c_type,
|
||||
vllm::ScalarType const& s_type, bool has_bias,
|
||||
bool has_act_order, bool is_k_full, bool has_zp, int num_groups,
|
||||
int group_size, int dev, cudaStream_t stream, int thread_k_init,
|
||||
void* a_s, void* b_s, void* g_s, void* zp, int prob_m,
|
||||
int prob_n, int prob_k, int lda, void* workspace,
|
||||
vllm::ScalarType const& a_type, vllm::ScalarType const& b_type,
|
||||
vllm::ScalarType const& c_type, vllm::ScalarType const& s_type,
|
||||
bool has_bias, bool has_zp, int num_groups, int group_size,
|
||||
int dev, cudaStream_t stream, int thread_k_init,
|
||||
int thread_n_init, int sms, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float) {
|
||||
bool is_a_8bit = a_type.size_bits() == 8;
|
||||
STD_TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [",
|
||||
prob_m, ", ", prob_n, ", ", prob_k, "]");
|
||||
|
||||
int group_blocks = 0;
|
||||
if (has_act_order) {
|
||||
if (is_k_full) {
|
||||
STD_TORCH_CHECK(group_size != -1);
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
} else {
|
||||
STD_TORCH_CHECK(group_size == 0);
|
||||
group_blocks = 0;
|
||||
}
|
||||
int group_blocks;
|
||||
if (group_size == -1) {
|
||||
group_blocks = -1;
|
||||
} else {
|
||||
if (group_size == -1) {
|
||||
group_blocks = -1;
|
||||
} else {
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
}
|
||||
group_blocks = group_size / 16;
|
||||
STD_TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
|
||||
" is not divisible by group_blocks = ", group_blocks);
|
||||
}
|
||||
|
||||
int num_bits = b_type.size_bits();
|
||||
@@ -370,28 +272,8 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
const float* g_s_ptr = (const float*)g_s;
|
||||
|
||||
const int4* zp_ptr = (const int4*)zp;
|
||||
const int* g_idx_ptr = (const int*)g_idx;
|
||||
const int* perm_ptr = (const int*)perm;
|
||||
int4* a_tmp_ptr = (int4*)a_tmp;
|
||||
int* locks = (int*)workspace;
|
||||
|
||||
if (has_act_order) {
|
||||
// Permute A columns
|
||||
int block_rows = div_ceil(prob_m, sms);
|
||||
// avoid ">>>" being formatted to "> > >"
|
||||
// clang-format off
|
||||
permute_cols_kernel<<<sms, default_threads, 0, stream>>>(
|
||||
A_ptr, perm_ptr, a_tmp_ptr, prob_m, prob_k, lda, block_rows);
|
||||
// clang-format on
|
||||
A_ptr = a_tmp_ptr;
|
||||
lda = prob_k;
|
||||
|
||||
// If we have a full K, then we can run the non-act-order version of Marlin
|
||||
// (since the weight rows are reordered by increasing group ids, and by
|
||||
// having a full K, we have full original groups)
|
||||
if (is_k_full) has_act_order = false;
|
||||
}
|
||||
|
||||
int max_shared_mem = 0;
|
||||
cudaDeviceGetAttribute(&max_shared_mem,
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
|
||||
@@ -451,18 +333,17 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
// Auto config
|
||||
exec_cfg = determine_exec_config(
|
||||
a_type, b_type, c_type, s_type, prob_m_split, prob_n, prob_k,
|
||||
thread_m_blocks, m_block_size_8, num_bits, group_size, has_act_order,
|
||||
is_k_full, has_zp, is_zp_float, is_a_8bit, stages, max_shared_mem,
|
||||
sms);
|
||||
thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
|
||||
is_zp_float, is_a_8bit, stages, max_shared_mem, sms);
|
||||
thread_tfg = exec_cfg.tb_cfg;
|
||||
if (thread_tfg.thread_n != -1) {
|
||||
if (prob_n / thread_tfg.thread_n *
|
||||
div_ceil(prob_m_split, thread_m_blocks * 16) * 4 <=
|
||||
sms) {
|
||||
if (is_valid_config({128, 64, 128}, thread_m_blocks, prob_m_split,
|
||||
prob_n, prob_k, num_bits, group_size,
|
||||
has_act_order, is_k_full, has_zp, is_zp_float,
|
||||
is_a_8bit, stages, max_shared_mem_new)) {
|
||||
prob_n, prob_k, num_bits, group_size, has_zp,
|
||||
is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem_new)) {
|
||||
thread_tfg = {128, 64, 128};
|
||||
exec_cfg = {1, thread_tfg};
|
||||
}
|
||||
@@ -487,30 +368,27 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
is_valid_config(thread_tfg, thread_m_blocks, prob_m_split, prob_n,
|
||||
prob_k, num_bits, group_size, has_act_order, is_k_full,
|
||||
has_zp, is_zp_float, is_a_8bit, stages,
|
||||
max_shared_mem_new),
|
||||
prob_k, num_bits, group_size, has_zp, is_zp_float,
|
||||
is_a_8bit, stages, max_shared_mem_new),
|
||||
"Invalid thread config: thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_k = ", thread_tfg.thread_k,
|
||||
", thread_n = ", thread_tfg.thread_n,
|
||||
", num_threads = ", thread_tfg.num_threads, " for MKN = [", prob_m,
|
||||
", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
|
||||
", prob_m_split = ", prob_m_split, ", group_size = ", group_size,
|
||||
", has_act_order = ", has_act_order, ", is_k_full = ", is_k_full,
|
||||
", has_zp = ", has_zp, ", is_zp_float = ", is_zp_float,
|
||||
", stages = ", stages, ", max_shared_mem_new = ", max_shared_mem_new);
|
||||
|
||||
auto kernel = get_marlin_kernel(
|
||||
a_type, b_type, c_type, s_type, thread_m_blocks, thread_n_blocks,
|
||||
thread_k_blocks, m_block_size_8, has_act_order, has_zp, group_blocks,
|
||||
num_threads, is_zp_float, stages);
|
||||
thread_k_blocks, m_block_size_8, has_zp, group_blocks, num_threads,
|
||||
is_zp_float, stages);
|
||||
|
||||
if (kernel == MarlinDefault) {
|
||||
STD_TORCH_CHECK(
|
||||
false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n, ", ",
|
||||
prob_k, "]", ", has_act_order = ", has_act_order,
|
||||
", num_groups = ", num_groups, ", group_size = ", group_size,
|
||||
", prob_m_split = ", prob_m_split,
|
||||
prob_k, "]", ", num_groups = ", num_groups,
|
||||
", group_size = ", group_size, ", prob_m_split = ", prob_m_split,
|
||||
", thread_m_blocks = ", thread_m_blocks,
|
||||
", thread_n_blocks = ", thread_n_blocks,
|
||||
", thread_k_blocks = ", thread_k_blocks,
|
||||
@@ -527,7 +405,6 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
// clang-format off
|
||||
kernel<<<blocks, num_threads, max_shared_mem_new, stream>>>(
|
||||
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
|
||||
g_idx_ptr, num_groups,
|
||||
prob_m_split, prob_n, prob_k, lda, locks, has_bias, part_use_atomic_add,
|
||||
use_fp32_reduce, max_shared_mem_new);
|
||||
// clang-format on
|
||||
@@ -550,11 +427,9 @@ torch::stable::Tensor marlin_gemm(
|
||||
std::optional<torch::stable::Tensor> const& a_scales_or_none,
|
||||
std::optional<torch::stable::Tensor> const& global_scale_or_none,
|
||||
std::optional<torch::stable::Tensor> const& b_zeros_or_none,
|
||||
std::optional<torch::stable::Tensor> const& g_idx_or_none,
|
||||
std::optional<torch::stable::Tensor> const& perm_or_none,
|
||||
torch::stable::Tensor& workspace, vllm::ScalarTypeId const& b_type_id,
|
||||
int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
|
||||
bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) {
|
||||
int64_t size_m, int64_t size_n, int64_t size_k, bool use_atomic_add,
|
||||
bool use_fp32_reduce, bool is_zp_float) {
|
||||
vllm::ScalarTypeId a_type_id, c_type_id, s_type_id;
|
||||
|
||||
auto c_scalar_type = a.scalar_type();
|
||||
@@ -722,7 +597,7 @@ torch::stable::Tensor marlin_gemm(
|
||||
std::nullopt, device);
|
||||
}
|
||||
|
||||
// Detect groupsize and act_order
|
||||
// Detect group size.
|
||||
int num_groups = -1;
|
||||
int group_size = -1;
|
||||
|
||||
@@ -733,51 +608,13 @@ torch::stable::Tensor marlin_gemm(
|
||||
" is not size_n = ", size_n);
|
||||
num_groups = b_scales.size(0);
|
||||
|
||||
torch::stable::Tensor g_idx, perm, a_tmp;
|
||||
if (g_idx_or_none.has_value() && perm_or_none.has_value()) {
|
||||
g_idx = g_idx_or_none.value();
|
||||
perm = perm_or_none.value();
|
||||
|
||||
STD_TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU");
|
||||
STD_TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous");
|
||||
STD_TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU");
|
||||
STD_TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous");
|
||||
|
||||
// Verify g_idx and perm
|
||||
STD_TORCH_CHECK((g_idx.size(-1) == 0 && perm.size(-1) == 0) ||
|
||||
(g_idx.size(-1) == size_k && perm.size(-1) == size_k),
|
||||
"Unexpected g_idx.size(-1) = ", g_idx.size(-1),
|
||||
" and perm.size(-1) = ", perm.size(-1),
|
||||
", where size_k = ", size_k);
|
||||
if (num_groups > 1) {
|
||||
STD_TORCH_CHECK(
|
||||
size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by b_scales.size(0) = ", b_scales.size(0));
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
g_idx = torch::stable::empty({0}, c_scalar_type, std::nullopt, device);
|
||||
perm = torch::stable::empty({0}, c_scalar_type, std::nullopt, device);
|
||||
a_tmp = torch::stable::empty({0}, c_scalar_type, std::nullopt, device);
|
||||
}
|
||||
bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0;
|
||||
|
||||
if (has_act_order) {
|
||||
a_tmp = torch::stable::empty({size_m, size_k}, c_scalar_type, std::nullopt,
|
||||
device);
|
||||
if (is_k_full) {
|
||||
STD_TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
|
||||
STD_TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by num_groups = ", num_groups);
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
group_size = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
a_tmp = torch::stable::empty({0}, c_scalar_type, std::nullopt, device);
|
||||
if (num_groups > 1) {
|
||||
STD_TORCH_CHECK(
|
||||
size_k % num_groups == 0, "size_k = ", size_k,
|
||||
", is not divisible by b_scales.size(0) = ", b_scales.size(0));
|
||||
group_size = size_k / num_groups;
|
||||
} else {
|
||||
group_size = -1;
|
||||
}
|
||||
group_size = -1;
|
||||
}
|
||||
|
||||
torch::stable::Tensor global_scale;
|
||||
@@ -882,11 +719,9 @@ torch::stable::Tensor marlin_gemm(
|
||||
a.const_data_ptr(), b_q_weight.const_data_ptr(), c.mutable_data_ptr(),
|
||||
c_tmp.mutable_data_ptr(), b_bias.mutable_data_ptr(),
|
||||
a_scales.mutable_data_ptr(), b_scales.mutable_data_ptr(),
|
||||
global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(),
|
||||
g_idx.mutable_data_ptr(), perm.mutable_data_ptr(),
|
||||
a_tmp.mutable_data_ptr(), size_m, size_n, size_k, a.stride(0),
|
||||
workspace.mutable_data_ptr(), a_type, b_type, c_type, s_type, has_bias,
|
||||
has_act_order, is_k_full, has_zp, num_groups, group_size, device_index,
|
||||
global_scale.mutable_data_ptr(), b_zeros.mutable_data_ptr(), size_m,
|
||||
size_n, size_k, a.stride(0), workspace.mutable_data_ptr(), a_type, b_type,
|
||||
c_type, s_type, has_bias, has_zp, num_groups, group_size, device_index,
|
||||
get_current_cuda_stream(device_index), thread_k, thread_n, sms,
|
||||
use_atomic_add, use_fp32_reduce, is_zp_float);
|
||||
|
||||
|
||||
@@ -51,10 +51,9 @@ template <typename scalar_t, // compute dtype, half or nv_float16
|
||||
// only works when thread_m_blocks == 1
|
||||
const int stages, // number of stages for the async global->shared
|
||||
// fetch pipeline
|
||||
const bool has_act_order, // whether act_order is enabled
|
||||
const int group_blocks, // number of consecutive 16x16 blocks
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float // is zero point of float16 type?
|
||||
const int group_blocks, // number of consecutive 16x16 blocks
|
||||
// with a separate quantization scale
|
||||
const bool is_zp_float // is zero point of float16 type?
|
||||
>
|
||||
__global__ void Marlin(
|
||||
const int4* __restrict__ A, // fp16 input matrix of shape mxk
|
||||
@@ -63,11 +62,9 @@ __global__ void Marlin(
|
||||
int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce)
|
||||
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
|
||||
// (k/groupsize)xn
|
||||
const int* __restrict__ g_idx, // int32 group indices of shape k
|
||||
int num_groups, // number of scale groups per output channel
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
int* locks, // extra global storage for barrier synchronization
|
||||
bool use_fp32_reduce // whether to use fp32 global reduce
|
||||
) {}
|
||||
@@ -140,29 +137,6 @@ __device__ inline void sub_zp(
|
||||
frag_b[1] = __hsub2(frag_b[1], zp);
|
||||
}
|
||||
|
||||
// Same as above, but for act_order (each K is multiplied individually)
|
||||
template <vllm::ScalarTypeId type_id>
|
||||
__device__ inline void scale4(
|
||||
typename MarlinScalarType<type_id>::FragB& frag_b,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_1,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_2,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_3,
|
||||
typename MarlinScalarType<type_id>::FragS& frag_s_4, int i) {
|
||||
using scalar_t = typename MarlinScalarType<type_id>::scalar_t;
|
||||
using scalar_t2 = typename MarlinScalarType<type_id>::scalar_t2;
|
||||
|
||||
scalar_t2 s_val_1_2;
|
||||
s_val_1_2.x = reinterpret_cast<scalar_t*>(&frag_s_1)[i];
|
||||
s_val_1_2.y = reinterpret_cast<scalar_t*>(&frag_s_2)[i];
|
||||
|
||||
scalar_t2 s_val_3_4;
|
||||
s_val_3_4.x = reinterpret_cast<scalar_t*>(&frag_s_3)[i];
|
||||
s_val_3_4.y = reinterpret_cast<scalar_t*>(&frag_s_4)[i];
|
||||
|
||||
frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
|
||||
frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
|
||||
}
|
||||
|
||||
// Given 2 floats multiply by 2 scales (halves)
|
||||
template <vllm::ScalarTypeId type_id>
|
||||
__device__ inline void scale_float(
|
||||
@@ -256,14 +230,11 @@ __global__ void Marlin(
|
||||
// 4bit packed zero-points of shape
|
||||
// (k/groupsize, n/pack_factor)
|
||||
const int4* __restrict__ zp_ptr,
|
||||
// int32 group indices of shape k
|
||||
const int* __restrict__ g_idx,
|
||||
int num_groups, // number of scale groups per output channel
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
int lda, // A.stride(0), equal to prob_k is A is contiguous
|
||||
int* locks, // extra global storage for barrier synchronization
|
||||
int prob_m, // batch dimension m
|
||||
int prob_n, // output dimension n
|
||||
int prob_k, // reduction dimension k
|
||||
int lda, // A.stride(0), equal to prob_k is A is contiguous
|
||||
int* locks, // extra global storage for barrier synchronization
|
||||
bool has_bias,
|
||||
bool use_atomic_add, // whether to use atomic add to reduce
|
||||
bool use_fp32_reduce, // whether to use fp32 global reduce
|
||||
@@ -358,7 +329,8 @@ __global__ void Marlin(
|
||||
global_scale_f32 = global_scale_ptr[0];
|
||||
}
|
||||
|
||||
constexpr bool has_act_order = group_blocks == 0;
|
||||
static_assert(group_blocks != 0,
|
||||
"runtime activation ordering is not supported");
|
||||
constexpr int m_block_size = m_block_size_8 ? 8 : (16 * thread_m_blocks);
|
||||
|
||||
extern __shared__ int4 sh[];
|
||||
@@ -391,7 +363,7 @@ __global__ void Marlin(
|
||||
|
||||
int iters = div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
||||
|
||||
if constexpr (!has_act_order && group_blocks != -1) {
|
||||
if constexpr (group_blocks != -1) {
|
||||
if (group_blocks >= thread_k_blocks) {
|
||||
// Ensure that the number of tiles in each stripe is a multiple of the
|
||||
// groupsize; this avoids an annoying special case where a stripe starts
|
||||
@@ -558,27 +530,17 @@ __global__ void Marlin(
|
||||
b_sh_stride * thread_k_blocks / (is_a_8bit ? 2 : 1);
|
||||
constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
|
||||
|
||||
// Scale sizes/strides without act_order
|
||||
// Scale sizes/strides
|
||||
int s_gl_stride = prob_n / (is_8bit_scale ? 16 : 8);
|
||||
constexpr int s_sh_stride = 16 * thread_n_blocks / (is_8bit_scale ? 16 : 8);
|
||||
constexpr int s_tb_groups =
|
||||
!has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
|
||||
group_blocks != -1 && group_blocks < thread_k_blocks
|
||||
? thread_k_blocks / group_blocks
|
||||
: 1;
|
||||
constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
|
||||
int s_gl_rd_delta = s_gl_stride;
|
||||
|
||||
// Scale size/strides with act_order
|
||||
constexpr int tb_k = 16 * thread_k_blocks;
|
||||
constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
|
||||
// constexpr int act_s_row_stride = 1;
|
||||
// int act_s_col_stride = act_s_row_stride * num_groups;
|
||||
constexpr int act_s_max_num_groups = 32;
|
||||
int act_s_col_stride = 1;
|
||||
int act_s_col_warp_stride = act_s_col_stride * 8;
|
||||
|
||||
constexpr int tb_n_warps = thread_n_blocks / (is_a_8bit ? 2 : 4);
|
||||
int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
|
||||
|
||||
// Zero-points sizes/strides
|
||||
int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4;
|
||||
@@ -615,25 +577,16 @@ __global__ void Marlin(
|
||||
auto b_sh_rd = threadIdx.x * b_thread_vecs;
|
||||
b_sh_rd += b_sh_rd / b_sh_stride * (b_sh_stride * (b_sh_wr_iters - 1));
|
||||
|
||||
// For act_order
|
||||
int slice_k_start = tb_k * slice_row;
|
||||
int slice_k_finish = slice_k_start + tb_k * slice_iters;
|
||||
int slice_k_start_shared_fetch = slice_k_start;
|
||||
int slice_n_offset = act_s_col_tb_stride * slice_col;
|
||||
|
||||
// No act_order
|
||||
int s_gl_rd;
|
||||
if constexpr (!has_act_order) {
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
}
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
}
|
||||
auto s_sh_wr = threadIdx.x;
|
||||
bool s_sh_wr_pred = threadIdx.x < s_sh_stage;
|
||||
@@ -757,10 +710,8 @@ __global__ void Marlin(
|
||||
: (sh_size_b_red_min + sh_bias_size);
|
||||
|
||||
int4* sh_bias = sh_new + sh_size_b_red_min;
|
||||
int4* sh_g_idx = sh_new + sh_b_red_bias_size;
|
||||
int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
|
||||
constexpr int sh_s_size = has_act_order ? (act_s_max_num_groups * s_sh_stride)
|
||||
: (stages * s_sh_stage);
|
||||
int4* sh_zp = sh_new + sh_b_red_bias_size;
|
||||
constexpr int sh_s_size = stages * s_sh_stage;
|
||||
int4* sh_s = sh_zp + (stages * zp_sh_stage);
|
||||
int4* sh_a = sh_s + sh_s_size;
|
||||
|
||||
@@ -769,9 +720,8 @@ __global__ void Marlin(
|
||||
I4 frag_b_quant[2][b_thread_vecs];
|
||||
FragC frag_c[thread_m_blocks][is_a_8bit ? 2 : 4][2];
|
||||
FragC frag_c_tmp[thread_m_blocks][is_a_8bit ? 2 : 4][2];
|
||||
FragS frag_s[2][4]; // No act-order
|
||||
FragS frag_s[2][4];
|
||||
FragS frag_bias[2][4];
|
||||
FragS act_frag_s[2][4][4]; // For act-order
|
||||
int frag_qzp[2][num_ints_per_thread]; // Zero-points
|
||||
FragZP frag_zp; // Zero-points in fp16
|
||||
FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ
|
||||
@@ -801,42 +751,6 @@ __global__ void Marlin(
|
||||
reinterpret_cast<float*>(frag_c)[i] = 0;
|
||||
};
|
||||
|
||||
int sh_first_group_id = -1;
|
||||
int sh_num_groups = -1;
|
||||
|
||||
auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id,
|
||||
int last_group_id) {
|
||||
sh_first_group_id = first_group_id;
|
||||
sh_num_groups = last_group_id - first_group_id + 1;
|
||||
|
||||
if (sh_num_groups > act_s_max_num_groups) {
|
||||
sh_num_groups = act_s_max_num_groups;
|
||||
}
|
||||
|
||||
if (sh_first_group_id + sh_num_groups > num_groups) {
|
||||
sh_num_groups = num_groups - sh_first_group_id;
|
||||
}
|
||||
|
||||
int row_offset = first_group_id * s_gl_stride;
|
||||
|
||||
if (is_async) {
|
||||
for (int i = 0; i < sh_num_groups; i++) {
|
||||
if (threadIdx.x < s_sh_stride) {
|
||||
cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
|
||||
&scales_ptr[row_offset + (i * s_gl_stride) +
|
||||
slice_n_offset + threadIdx.x]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < sh_num_groups; i++) {
|
||||
if (threadIdx.x < s_sh_stride) {
|
||||
sh_s[(i * s_sh_stride) + threadIdx.x] =
|
||||
scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
|
||||
threadIdx.x];
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// Asynchronously fetch the next A, B and s tile from global to the next
|
||||
// shared memory pipeline location.
|
||||
auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
|
||||
@@ -862,44 +776,27 @@ __global__ void Marlin(
|
||||
|
||||
b_gl_rd += b_gl_rd_delta_o;
|
||||
|
||||
if constexpr (has_act_order) {
|
||||
// Fetch g_idx thread-block portion
|
||||
int full_pipe = a_off;
|
||||
int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
|
||||
if (cur_k < prob_k && cur_k < slice_k_finish) {
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
if constexpr (group_blocks != -1) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
int4 const* cur_g_idx_stage_ptr =
|
||||
reinterpret_cast<int4 const*>(&g_idx[cur_k]);
|
||||
|
||||
if (threadIdx.x < g_idx_stage) {
|
||||
cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
|
||||
&cur_g_idx_stage_ptr[threadIdx.x]);
|
||||
// Only fetch scales if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (s_sh_wr_pred) {
|
||||
cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
|
||||
}
|
||||
s_gl_rd += s_gl_rd_delta * s_tb_groups;
|
||||
}
|
||||
} else {
|
||||
if constexpr (group_blocks != -1) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
}
|
||||
|
||||
// Only fetch scales if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (s_sh_wr_pred) {
|
||||
cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
|
||||
}
|
||||
s_gl_rd += s_gl_rd_delta * s_tb_groups;
|
||||
}
|
||||
}
|
||||
if constexpr (has_zp && group_blocks != -1) {
|
||||
int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
|
||||
|
||||
if constexpr (has_zp && group_blocks != -1) {
|
||||
int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
|
||||
|
||||
// Only fetch zero points if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (zp_sh_wr_pred) {
|
||||
cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
|
||||
}
|
||||
zp_gl_rd += zp_gl_rd_delta * zp_tb_groups;
|
||||
// Only fetch zero points if this tile starts a new group
|
||||
if (pipe % div_ceil(group_blocks, thread_k_blocks) == 0) {
|
||||
if (zp_sh_wr_pred) {
|
||||
cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]);
|
||||
}
|
||||
zp_gl_rd += zp_gl_rd_delta * zp_tb_groups;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -947,150 +844,57 @@ __global__ void Marlin(
|
||||
}
|
||||
};
|
||||
|
||||
bool is_same_group[stages];
|
||||
int same_group_id[stages];
|
||||
|
||||
auto init_same_group = [&](int pipe) {
|
||||
if constexpr (!has_act_order) {
|
||||
return;
|
||||
}
|
||||
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
|
||||
|
||||
int group_id_1 = sh_g_idx_int_ptr[0];
|
||||
int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
|
||||
|
||||
is_same_group[pipe] = group_id_1 == group_id_2;
|
||||
same_group_id[pipe] = group_id_1;
|
||||
};
|
||||
|
||||
auto fetch_scales_to_registers = [&](int k, int full_pipe) {
|
||||
int pipe = full_pipe % stages;
|
||||
using IT1 = typename std::conditional_t<is_a_8bit, int2, int4>;
|
||||
using IT0 = typename std::conditional_t<is_a_8bit, int, int2>;
|
||||
constexpr int group_blocks2 = div_ceil(group_blocks, is_a_8bit ? 2 : 1);
|
||||
|
||||
if constexpr (!has_act_order) {
|
||||
// No act-order case
|
||||
if constexpr (group_blocks == -1) {
|
||||
// load only when starting a new slice
|
||||
if (k == 0 && full_pipe == 0 && dequant_skip_flop) {
|
||||
reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd];
|
||||
reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
|
||||
}
|
||||
} else if constexpr (group_blocks != -1) {
|
||||
if constexpr (group_blocks >= thread_k_blocks) {
|
||||
constexpr int g = group_blocks / thread_k_blocks;
|
||||
if (pipe % g == 0) {
|
||||
if (k % b_sh_wr_iters == 0) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * (g * (pipe / g));
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
|
||||
} else {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
}
|
||||
}
|
||||
} else if (group_blocks2 < b_sh_wr_iters || k % b_sh_wr_iters == 0) {
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
|
||||
int k_blocks = b_sh_wr_iters * warp_row + k % b_sh_wr_iters;
|
||||
int cur_group_id = k_blocks / group_blocks2;
|
||||
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
|
||||
sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[k % 2])[0] =
|
||||
reinterpret_cast<int2*>(
|
||||
sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)];
|
||||
}
|
||||
} else if (group_blocks >= b_sh_wr_iters) {
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int2*>(&frag_s[0])[0];
|
||||
}
|
||||
if constexpr (group_blocks == -1) {
|
||||
// Load only when starting a new slice.
|
||||
if (k == 0 && full_pipe == 0 && dequant_skip_flop) {
|
||||
reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd];
|
||||
reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
|
||||
}
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
constexpr int g = group_blocks / thread_k_blocks;
|
||||
if (pipe % g == 0) {
|
||||
if (k % b_sh_wr_iters == 0) {
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * (g * (pipe / g));
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
|
||||
} else {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
}
|
||||
}
|
||||
} else if (group_blocks2 < b_sh_wr_iters || k % b_sh_wr_iters == 0) {
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
|
||||
return;
|
||||
}
|
||||
int k_blocks = b_sh_wr_iters * warp_row + k % b_sh_wr_iters;
|
||||
int cur_group_id = k_blocks / group_blocks2;
|
||||
|
||||
// Act-order case
|
||||
int4* sh_s_stage = sh_s + s_sh_stage * pipe;
|
||||
|
||||
// Determine K of the "current" thread-block
|
||||
int cur_k = slice_k_start + tb_k * full_pipe;
|
||||
if (cur_k >= prob_k || cur_k >= slice_k_finish) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset (to current thread-block) since we read g_idx portion from the
|
||||
// shared memory
|
||||
cur_k = 0;
|
||||
|
||||
// Progress to current iteration
|
||||
cur_k += k % b_sh_wr_iters;
|
||||
|
||||
// Determine "position" inside the thread-block (based on warp and
|
||||
// thread-id)
|
||||
auto warp_id = threadIdx.x / 32;
|
||||
int warp_row = warp_id / tb_n_warps;
|
||||
int warp_col = warp_id % tb_n_warps;
|
||||
|
||||
cur_k += warp_row * 16 * b_sh_wr_iters;
|
||||
|
||||
auto th_id = threadIdx.x % 32;
|
||||
cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
|
||||
|
||||
int s_col_shift =
|
||||
/*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
|
||||
(th_id / 4) * act_s_col_stride;
|
||||
|
||||
if (is_same_group[pipe]) {
|
||||
if (k % 2 == 0) {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
|
||||
sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
|
||||
s_col_shift];
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
|
||||
sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
|
||||
} else {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
|
||||
reinterpret_cast<int2*>(&frag_s[k % 2])[0] = reinterpret_cast<int2*>(
|
||||
sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)];
|
||||
}
|
||||
|
||||
for (int i = 1; i < 4; i++) {
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
|
||||
} else if (group_blocks >= b_sh_wr_iters) {
|
||||
if constexpr (!is_8bit_scale) {
|
||||
reinterpret_cast<int4*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int4*>(&frag_s[0])[0];
|
||||
} else {
|
||||
reinterpret_cast<int2*>(&frag_s[1])[0] =
|
||||
reinterpret_cast<int2*>(&frag_s[0])[0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
|
||||
int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
|
||||
|
||||
constexpr int k_frag_offsets[4] = {0, 1, 8,
|
||||
9}; // Tensor core offsets per thread
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
int actual_k = cur_k + k_frag_offsets[i];
|
||||
|
||||
int group_id = sh_g_idx_int_ptr[actual_k];
|
||||
int rel_group_id = group_id - sh_first_group_id;
|
||||
|
||||
*(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
|
||||
sh_s[rel_group_id * s_sh_stride + s_col_shift];
|
||||
}
|
||||
};
|
||||
|
||||
auto fetch_zp_to_registers = [&](int k, int full_pipe) {
|
||||
// This code does not handle group_blocks == 0,
|
||||
// which signifies act_order.
|
||||
// has_zp implies AWQ, which doesn't have act_order,
|
||||
static_assert(!has_zp || group_blocks != 0);
|
||||
|
||||
if constexpr (has_zp && !is_zp_float) {
|
||||
@@ -1180,8 +984,7 @@ __global__ void Marlin(
|
||||
constexpr int g =
|
||||
group_blocks > 0 ? div_ceil(group_blocks, thread_k_blocks) : 1;
|
||||
const bool is_new_zp =
|
||||
(group_blocks == 0) ||
|
||||
((group_blocks > 0) && (group_blocks < b_sh_wr_iters || k == 0)) &&
|
||||
(group_blocks > 0 && (group_blocks < b_sh_wr_iters || k == 0)) &&
|
||||
(pipe % g == 0) ||
|
||||
(group_blocks == -1 && is_first_matmul_in_slice);
|
||||
if constexpr (has_zp && !is_zp_float) {
|
||||
@@ -1249,15 +1052,8 @@ __global__ void Marlin(
|
||||
sub_zp<a_type_id>(frag_b1, frag_zp[j], 1);
|
||||
}
|
||||
|
||||
// Apply scale to frag_b0
|
||||
if constexpr (has_act_order && !is_a_8bit) {
|
||||
static_assert(group_blocks != -1);
|
||||
scale4<a_type_id>(frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j],
|
||||
act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0);
|
||||
scale4<a_type_id>(frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j],
|
||||
act_frag_s[k2][2][j], act_frag_s[k2][3][j], 1);
|
||||
} else if constexpr (!dequant_skip_flop && has_zp && !is_zp_float &&
|
||||
group_blocks == -1 && !is_a_8bit) {
|
||||
if constexpr (!dequant_skip_flop && has_zp && !is_zp_float &&
|
||||
group_blocks == -1 && !is_a_8bit) {
|
||||
int idx = (threadIdx.x / 4) % 2;
|
||||
scalar_t2 s2 = Adtype::nums2num2(
|
||||
reinterpret_cast<scalar_t*>(&frag_s[j / 2][j % 2 * 2 + 0])[idx],
|
||||
@@ -1661,7 +1457,7 @@ __global__ void Marlin(
|
||||
|
||||
// For per-column quantization we finally apply the scale here (only for
|
||||
// 4-bit)
|
||||
if constexpr (!has_act_order && group_blocks == -1 && !is_a_8bit &&
|
||||
if constexpr (group_blocks == -1 && !is_a_8bit &&
|
||||
b_type.size_bits() == 4 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
c_scalar_t2 tmp_scale = s[0];
|
||||
@@ -1750,15 +1546,6 @@ __global__ void Marlin(
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < stages - 1; i++) {
|
||||
if (has_act_order && i == 0) {
|
||||
int last_g_idx = slice_k_start + stages * tb_k * 2;
|
||||
if (last_g_idx >= prob_k) {
|
||||
last_g_idx = prob_k - 1;
|
||||
}
|
||||
fetch_act_order_scales_to_shared(true, g_idx[slice_k_start],
|
||||
g_idx[last_g_idx]);
|
||||
}
|
||||
|
||||
if constexpr (has_zp && !is_zp_float && group_blocks == -1) {
|
||||
if (i == 0) {
|
||||
fetch_col_zp_to_shared();
|
||||
@@ -1772,14 +1559,10 @@ __global__ void Marlin(
|
||||
|
||||
zero_accums();
|
||||
wait_for_stage();
|
||||
init_same_group(0);
|
||||
fetch_to_registers(0, 0);
|
||||
fetch_scales_to_registers(0, 0);
|
||||
fetch_zp_to_registers(0, 0);
|
||||
a_gl_rd += a_gl_rd_delta_o * (stages - 1);
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start_shared_fetch += tb_k * (stages - 1);
|
||||
}
|
||||
};
|
||||
if (slice_iters) {
|
||||
start_pipes();
|
||||
@@ -1804,7 +1587,6 @@ __global__ void Marlin(
|
||||
slice_iters >= stages);
|
||||
pipe++;
|
||||
wait_for_stage();
|
||||
init_same_group(pipe % stages);
|
||||
}
|
||||
|
||||
if constexpr (!is_a_8bit) {
|
||||
@@ -1822,25 +1604,6 @@ __global__ void Marlin(
|
||||
|
||||
a_gl_rd += a_gl_rd_delta_o * stages;
|
||||
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start += tb_k * stages;
|
||||
|
||||
if (slice_k_start < prob_k) {
|
||||
slice_k_start_shared_fetch += tb_k * stages;
|
||||
int first_group_id = g_idx[slice_k_start];
|
||||
int last_g_idx = slice_k_start + stages * tb_k * 2;
|
||||
if (last_g_idx >= prob_k) {
|
||||
last_g_idx = prob_k - 1;
|
||||
}
|
||||
int last_group_id = g_idx[last_g_idx];
|
||||
if (last_group_id >= sh_first_group_id + sh_num_groups) {
|
||||
fetch_act_order_scales_to_shared(false, first_group_id,
|
||||
last_group_id);
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process results and, if necessary, proceed to the next column slice.
|
||||
// While this pattern may not be the most readable, other ways of writing
|
||||
// the loop seemed to noticeably worse performance after compilation.
|
||||
@@ -1898,7 +1661,7 @@ __global__ void Marlin(
|
||||
bool last = slice_idx == slice_count - 1;
|
||||
// For per-column scales, we only fetch them here in the final step before
|
||||
// write-out
|
||||
if constexpr (!has_act_order && group_blocks == -1 &&
|
||||
if constexpr (group_blocks == -1 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
if (b_type.size_bits() == 8 || (last || use_atomic_add) || is_a_8bit) {
|
||||
if (s_sh_wr_pred) {
|
||||
@@ -1917,7 +1680,7 @@ __global__ void Marlin(
|
||||
cp_async_fence();
|
||||
}
|
||||
|
||||
if constexpr (!has_act_order && group_blocks == -1 &&
|
||||
if constexpr (group_blocks == -1 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp || is_a_8bit)) {
|
||||
if constexpr (is_a_8bit) {
|
||||
cp_async_wait<0>();
|
||||
@@ -1948,7 +1711,7 @@ __global__ void Marlin(
|
||||
// For 8-bit channelwise, we apply the scale before the global reduction
|
||||
// that converts the fp32 results to fp16 (so that we avoid possible
|
||||
// overflow in fp16)
|
||||
if constexpr (!has_act_order && group_blocks == -1 && is_a_8bit) {
|
||||
if constexpr (group_blocks == -1 && is_a_8bit) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; j++) {
|
||||
float2 aa[2];
|
||||
@@ -1970,8 +1733,7 @@ __global__ void Marlin(
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!has_act_order && group_blocks == -1 &&
|
||||
b_type.size_bits() == 8 &&
|
||||
} else if (group_blocks == -1 && b_type.size_bits() == 8 &&
|
||||
(has_zp && dequant_skip_flop || !has_zp)) {
|
||||
if (threadIdx.x / 32 < tb_n_warps) {
|
||||
#pragma unroll
|
||||
@@ -2042,33 +1804,25 @@ __global__ void Marlin(
|
||||
b_gl_rd += b_sh_stride * slice_col + b_gl_rd_delta_o * slice_row;
|
||||
|
||||
bias_gl_rd = (thread_n_blocks * 16 / 8) * slice_col + threadIdx.x;
|
||||
// Update slice k/n for scales loading
|
||||
if constexpr (has_act_order) {
|
||||
slice_k_start = tb_k * slice_row;
|
||||
slice_k_finish = slice_k_start + tb_k * slice_iters;
|
||||
slice_k_start_shared_fetch = slice_k_start;
|
||||
slice_n_offset = act_s_col_tb_stride * slice_col;
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
if constexpr (group_blocks == -1) {
|
||||
s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else if constexpr (group_blocks >= thread_k_blocks) {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
s_sh_stride * slice_col + threadIdx.x;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
|
||||
zp_sh_stride * slice_col + threadIdx.x;
|
||||
} else {
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / zp_sh_stride) +
|
||||
zp_sh_stride * slice_col + threadIdx.x % zp_sh_stride;
|
||||
}
|
||||
s_gl_rd =
|
||||
s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / s_sh_stride) +
|
||||
s_sh_stride * slice_col + threadIdx.x % s_sh_stride;
|
||||
zp_gl_rd =
|
||||
zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks +
|
||||
threadIdx.x / zp_sh_stride) +
|
||||
zp_sh_stride * slice_col + threadIdx.x % zp_sh_stride;
|
||||
}
|
||||
start_pipes();
|
||||
}
|
||||
|
||||
@@ -27,8 +27,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"per_token_group_quant_int8(Tensor input, Tensor! output_q, Tensor! "
|
||||
"output_s, int group_size, float eps, float int8_min, float int8_max) -> "
|
||||
"()");
|
||||
ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
|
||||
|
||||
ops.def("get_cuda_view_from_cpu_tensor(Tensor cpu_tensor) -> Tensor");
|
||||
|
||||
#ifndef USE_ROCM
|
||||
@@ -85,15 +83,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"marlin_gemm(Tensor a, Tensor? c_or_none, Tensor b_q_weight, "
|
||||
"Tensor? b_bias_or_none,Tensor b_scales, "
|
||||
"Tensor? a_scales, Tensor? global_scale, Tensor? b_zeros_or_none, "
|
||||
"Tensor? "
|
||||
"g_idx_or_none, Tensor? perm_or_none, Tensor workspace, int b_type_id, "
|
||||
"SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, "
|
||||
"Tensor workspace, int b_type_id, "
|
||||
"SymInt size_m, SymInt size_n, SymInt size_k, "
|
||||
"bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) -> Tensor");
|
||||
// conditionally compiled so impl registrations are in source file
|
||||
|
||||
// gptq_marlin repack from GPTQ.
|
||||
ops.def(
|
||||
"gptq_marlin_repack(Tensor b_q_weight, Tensor perm, "
|
||||
"gptq_marlin_repack(Tensor b_q_weight, "
|
||||
"SymInt size_k, SymInt size_n, int num_bits, bool is_a_8bit) -> Tensor");
|
||||
// conditionally compiled so impl registrations are in source file
|
||||
|
||||
@@ -679,12 +676,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// to prevent the meta function registry.
|
||||
ops.def(
|
||||
"gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, "
|
||||
"Tensor b_gptq_scales, Tensor b_g_idx, bool use_exllama, bool "
|
||||
"Tensor b_gptq_scales, bool use_exllama, bool "
|
||||
"use_v2_format, int bit) "
|
||||
"-> Tensor");
|
||||
|
||||
// Post processing for GPTQ.
|
||||
ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
|
||||
ops.def("gptq_shuffle(Tensor! q_weight, int bit) -> ()");
|
||||
|
||||
// Mamba selective scan kernel
|
||||
ops.def(
|
||||
@@ -735,8 +732,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("per_token_group_quant_int8",
|
||||
TORCH_BOX(&per_token_group_quant_int8));
|
||||
|
||||
ops.impl("permute_cols", TORCH_BOX(&permute_cols));
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// CUTLASS scaled_mm ops
|
||||
ops.impl("cutlass_scaled_mm", TORCH_BOX(&cutlass_scaled_mm));
|
||||
|
||||
+2
-3
@@ -26,12 +26,11 @@ void wvSplitKQ(const at::Tensor& in_a, const at::Tensor& in_b,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros, torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
bool use_v2_format);
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
torch::Tensor b_scales, bool use_v2_format);
|
||||
|
||||
void moe_gptq_gemm_rdna3(torch::Tensor a, torch::Tensor c,
|
||||
torch::Tensor b_q_weight, torch::Tensor b_scales,
|
||||
|
||||
+40
-63
@@ -253,11 +253,13 @@ __forceinline__ __device__ void load4_scales(const T* scales_row, int n,
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <typename T, int M_COUNT>
|
||||
__global__ void gemm_q4_kernel_rdna3(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
__global__ void gemm_q4_kernel_rdna3(const T* __restrict__ a,
|
||||
const uint32_t* __restrict__ b_q_weight,
|
||||
const uint32_t* __restrict__ b_qzeros,
|
||||
const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m,
|
||||
const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset) {
|
||||
const int t = threadIdx.x;
|
||||
const int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
|
||||
const int offset_m = blockIdx.y * M_COUNT;
|
||||
@@ -274,8 +276,8 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
constexpr int LDS_PAD = 8;
|
||||
__shared__ T block_a[M_COUNT][BLOCK_KN_SIZE + LDS_PAD];
|
||||
|
||||
// Stage A: each thread loads 1 K element per M row into LDS (with optional
|
||||
// act-order permutation). THREADS_X == BLOCK_KN_SIZE so this is a 1:1 map.
|
||||
// Stage A: each thread loads 1 K element per M row into LDS.
|
||||
// THREADS_X == BLOCK_KN_SIZE so this is a 1:1 map.
|
||||
// For M_COUNT > 1 with size_m not a multiple of M_COUNT, slots past size_m
|
||||
// are zero-padded so the dot product contribution is 0 (we then skip the
|
||||
// atomic write for those rows below).
|
||||
@@ -298,10 +300,7 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
T av;
|
||||
if (offset_m + m < size_m) {
|
||||
const T* a_row = a + (offset_m + m) * size_k;
|
||||
if (b_q_perm)
|
||||
av = a_row[b_q_perm[offset_k + t]];
|
||||
else
|
||||
av = a_row[offset_k + t];
|
||||
av = a_row[offset_k + t];
|
||||
} else {
|
||||
av = tzero<T>(); // zero-pad invalid M rows
|
||||
}
|
||||
@@ -314,14 +313,6 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
// the LDS load above — but here all THREADS_X (=256) threads always do,
|
||||
// regardless of whether their `n` is in bounds.
|
||||
__syncthreads();
|
||||
} else if (b_q_perm) {
|
||||
// bf16 M=1 fast path skips LDS, but its global read below is sequential
|
||||
// and cannot apply act-order. When a permutation is present, stage the
|
||||
// single A row through LDS (as fp16 / M>1 do) so the read picks it up.
|
||||
// b_q_perm is block-uniform, so the __syncthreads is non-divergent.
|
||||
if (offset_k + t < end_k)
|
||||
block_a[0][t] = a[offset_m * size_k + b_q_perm[offset_k + t]];
|
||||
__syncthreads();
|
||||
}
|
||||
if (n >= size_n) return;
|
||||
|
||||
@@ -452,14 +443,11 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
// fp32-aliased union. Storing as uint32 keeps the IR-level type
|
||||
// opaque so the inner v_dot2 cannot be folded to fp32 widening.
|
||||
//
|
||||
// A is read direct from global (no LDS staging — see USE_LDS_A above),
|
||||
// except under act-order, where it comes from the permuted LDS copy.
|
||||
// A is read direct from global (no LDS staging — see USE_LDS_A above).
|
||||
pack4 a_pack;
|
||||
{
|
||||
const uint32_t* a_words =
|
||||
b_q_perm
|
||||
? reinterpret_cast<const uint32_t*>(&block_a[0][a_off])
|
||||
: reinterpret_cast<const uint32_t*>(a + offset_k + a_off);
|
||||
reinterpret_cast<const uint32_t*>(a + offset_k + a_off);
|
||||
a_pack.u[0] = a_words[0];
|
||||
a_pack.u[1] = a_words[1];
|
||||
a_pack.u[2] = a_words[2];
|
||||
@@ -623,8 +611,7 @@ __global__ void gemm_q4_kernel_rdna3(
|
||||
template <typename T, int M_COUNT>
|
||||
__global__ void gemm_q4_kernel_rdna3(const T*, const uint32_t*, const uint32_t*,
|
||||
const T*, T*, const int, const int,
|
||||
const int, const int, const int,
|
||||
const int*) {}
|
||||
const int, const int, const int) {}
|
||||
|
||||
#endif // __HIP__RDNA3__ || !__HIP_DEVICE_COMPILE__
|
||||
|
||||
@@ -635,17 +622,17 @@ __global__ void gemm_q4_kernel_rdna3(const T*, const uint32_t*, const uint32_t*,
|
||||
template <typename T, int M_COUNT>
|
||||
void launch_gemm_q4_for_mcount(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
dim3 block(THREADS_X);
|
||||
dim3 grid((size_n + BLOCK_KN_SIZE * 4 - 1) / (BLOCK_KN_SIZE * 4),
|
||||
(size_m + M_COUNT - 1) / M_COUNT,
|
||||
(size_k + BLOCK_KN_SIZE - 1) / BLOCK_KN_SIZE);
|
||||
|
||||
gemm_q4_kernel_rdna3<T, M_COUNT><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_kernel_rdna3<T, M_COUNT>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
// Dispatch to the largest M_COUNT template that doesn't waste more than
|
||||
@@ -661,31 +648,30 @@ void launch_gemm_q4_for_mcount(const T* a, const uint32_t* b_q_weight,
|
||||
// only burn instructions on the last block, never affect correctness.
|
||||
template <typename T>
|
||||
void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m, int size_n,
|
||||
int size_k, int groups, bool use_v2_format,
|
||||
cudaStream_t stream) {
|
||||
const uint32_t* b_qzeros, const T* b_scales, T* c,
|
||||
int size_m, int size_n, int size_k, int groups,
|
||||
bool use_v2_format, cudaStream_t stream) {
|
||||
const int zero_offset = use_v2_format ? 0 : 1;
|
||||
|
||||
if (size_m == 1) {
|
||||
launch_gemm_q4_for_mcount<T, 1>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 1>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else if (size_m <= 3) {
|
||||
launch_gemm_q4_for_mcount<T, 2>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 2>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else if (size_m <= 7) {
|
||||
launch_gemm_q4_for_mcount<T, 4>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 4>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else {
|
||||
// M_COUNT=8 covers M up to 15 here; M >= 16 should ideally take the
|
||||
// WMMA path, but if it falls through we still produce correct output —
|
||||
// just leaving 3-5× of throughput on the table for prefill workloads.
|
||||
launch_gemm_q4_for_mcount<T, 8>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_for_mcount<T, 8>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -701,7 +687,6 @@ void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
// b_q_weight[K/8, N] uint32 (already shuffled via gptq_shuffle)
|
||||
// b_qzeros [groups, N/8] uint32 (packed 4-bit zeros)
|
||||
// b_scales [groups, N] half or bfloat16
|
||||
// b_g_idx [K] or empty int32 (act-order permutation; empty=identity)
|
||||
// use_v2_format bool (true = GPTQv2, no +1 zero offset)
|
||||
//
|
||||
// Output:
|
||||
@@ -709,17 +694,16 @@ void launch_gemm_q4(const T* a, const uint32_t* b_q_weight,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format);
|
||||
torch::Tensor b_scales, bool use_v2_format);
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros, torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format) {
|
||||
bool use_v2_format) {
|
||||
if (a.dim() == 2 && b_q_weight.dim() == 2 && a.size(1) % 16 == 0 &&
|
||||
b_q_weight.size(1) % 16 == 0 &&
|
||||
((a.scalar_type() == torch::kBFloat16 && a.size(0) >= 16) ||
|
||||
(a.scalar_type() == torch::kHalf && a.size(0) >= 64))) {
|
||||
return gptq_gemm_rdna3_wmma(a, b_q_weight, b_qzeros, b_scales, b_g_idx,
|
||||
return gptq_gemm_rdna3_wmma(a, b_q_weight, b_qzeros, b_scales,
|
||||
use_v2_format);
|
||||
}
|
||||
|
||||
@@ -753,25 +737,18 @@ torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
auto opts = torch::TensorOptions().dtype(a.dtype()).device(a.device());
|
||||
at::Tensor c = torch::zeros({size_m, size_n}, opts);
|
||||
|
||||
const int* g_idx_ptr = nullptr;
|
||||
if (!b_g_idx.device().is_meta() && b_g_idx.numel() > 0) {
|
||||
TORCH_CHECK(b_g_idx.scalar_type() == torch::kInt32,
|
||||
"b_g_idx must be int32");
|
||||
g_idx_ptr = (const int*)b_g_idx.data_ptr();
|
||||
}
|
||||
|
||||
if (a.scalar_type() == torch::kHalf) {
|
||||
vllm::gptq_rdna3::launch_gemm_q4<half>(
|
||||
(const half*)a.data_ptr(), (const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(), (const half*)b_scales.data_ptr(),
|
||||
g_idx_ptr, (half*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
use_v2_format, stream);
|
||||
(half*)c.data_ptr(), size_m, size_n, size_k, groups, use_v2_format,
|
||||
stream);
|
||||
} else {
|
||||
vllm::gptq_rdna3::launch_gemm_q4<vllm::gptq_rdna3::bf16_t>(
|
||||
(const vllm::gptq_rdna3::bf16_t*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(),
|
||||
(const vllm::gptq_rdna3::bf16_t*)b_scales.data_ptr(), g_idx_ptr,
|
||||
(const vllm::gptq_rdna3::bf16_t*)b_scales.data_ptr(),
|
||||
(vllm::gptq_rdna3::bf16_t*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
use_v2_format, stream);
|
||||
}
|
||||
|
||||
+83
-142
@@ -373,7 +373,7 @@ __global__ void gemm_q4_wmma_kernel_16x16_1w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -471,25 +471,16 @@ __global__ void gemm_q4_wmma_kernel_16x16_1w(
|
||||
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
// Permuted (act-order): scattered global reads, no vectorization.
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
// Sequential A reads: replace 16 single-element global_load_b16 with
|
||||
// a bulk 32-byte copy. The AMDGPU backend lowers a memcpy of this
|
||||
// size + alignment to two `global_load_b128` instructions. size_k is
|
||||
// a multiple of 16 (TORCH_CHECK above) and k_tile increments by 16,
|
||||
// so k_tile + 16 is always within bounds — no tail handling needed.
|
||||
// Note: we memcpy into the whole vector (`&a_frag`) rather than
|
||||
// `&a_frag[0]`; ext_vector_type element addresses aren't reliably
|
||||
// valid C pointers across compiler versions.
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes (16 × 2)");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
// Sequential A reads: replace 16 single-element global_load_b16 with
|
||||
// a bulk 32-byte copy. The AMDGPU backend lowers a memcpy of this
|
||||
// size + alignment to two `global_load_b128` instructions. size_k is
|
||||
// a multiple of 16 (TORCH_CHECK above) and k_tile increments by 16,
|
||||
// so k_tile + 16 is always within bounds — no tail handling needed.
|
||||
// Note: we memcpy into the whole vector (`&a_frag`) rather than
|
||||
// `&a_frag[0]`; ext_vector_type element addresses aren't reliably
|
||||
// valid C pointers across compiler versions.
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes (16 × 2)");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -597,25 +588,24 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_16x16_1w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_16x16_1w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// 1 wave per block (32 lanes), 16x16 C tile per block. gridDim.z splits
|
||||
// K so that more blocks (and therefore more waves) are in flight; with
|
||||
// K_SPLIT > 1 the kernel switches to atomic write-back at the epilogue.
|
||||
const int k_split = compute_wmma_k_split(size_k);
|
||||
dim3 block(32);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 15) / 16, k_split);
|
||||
gemm_q4_wmma_kernel_16x16_1w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_16x16_1w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -657,7 +647,7 @@ __global__ void gemm_q4_wmma_kernel_32x16_2w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -757,16 +747,8 @@ __global__ void gemm_q4_wmma_kernel_32x16_2w(
|
||||
V16 a_frag, b_frag;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -844,16 +826,15 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_32x16_2w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fallback to v1 for size_m < 32. With M-tile=32 the v2 block has 2 waves
|
||||
// working on rows [0..15] and [16..31]; at M < 32 the second wave processes
|
||||
// out-of-range M rows (zero-padded a_frag → wmma produces nothing useful)
|
||||
@@ -862,9 +843,9 @@ void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
// max-num-seqs=32 lands at M≈32 steady-state; the M=16 sliver is edge),
|
||||
// but the fallback costs nothing and is the right shape.
|
||||
if (size_m < 32) {
|
||||
launch_gemm_q4_wmma_16x16_1w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_16x16_1w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -875,9 +856,9 @@ void launch_gemm_q4_wmma_32x16_2w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split(size_k);
|
||||
dim3 block(64);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 31) / 32, k_split);
|
||||
gemm_q4_wmma_kernel_32x16_2w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_32x16_2w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -917,7 +898,7 @@ __global__ void gemm_q4_wmma_kernel_64x16_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1019,16 +1000,8 @@ __global__ void gemm_q4_wmma_kernel_64x16_4w(
|
||||
V16 a_frag, b_frag;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1100,21 +1073,20 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x16_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x16_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v2 for M < 64 (would waste 1+ waves on out-of-range rows).
|
||||
if (size_m < 64) {
|
||||
launch_gemm_q4_wmma_32x16_2w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_32x16_2w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1122,9 +1094,9 @@ void launch_gemm_q4_wmma_64x16_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 16);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 15) / 16, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x16_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x16_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -1158,7 +1130,7 @@ __global__ void gemm_q4_wmma_kernel_64x32_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1264,16 +1236,8 @@ __global__ void gemm_q4_wmma_kernel_64x32_4w(
|
||||
V16 a_frag, b_frag0, b_frag1;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1352,23 +1316,22 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x32_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x32_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v3 when M < 64 (small-M decode/prefill stays on the
|
||||
// narrower 64M × 16N path) or when N < 32 (tile would waste a wave on
|
||||
// out-of-range cols).
|
||||
if (size_m < 64 || size_n < 32) {
|
||||
launch_gemm_q4_wmma_64x16_4w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_64x16_4w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1376,9 +1339,9 @@ void launch_gemm_q4_wmma_64x32_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 32);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 31) / 32, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x32_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x32_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
#if defined(__HIP__RDNA3__) || !defined(__HIP_DEVICE_COMPILE__)
|
||||
@@ -1412,7 +1375,7 @@ __global__ void gemm_q4_wmma_kernel_64x64_4w(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1510,16 +1473,8 @@ __global__ void gemm_q4_wmma_kernel_64x64_4w(
|
||||
V16 a_frag, b_frag0, b_frag1, b_frag2, b_frag3;
|
||||
if (m_row < size_m) {
|
||||
const T* a_row = a + m_row * size_k;
|
||||
if (b_q_perm) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) {
|
||||
T v = a_row[b_q_perm[k_tile + i]];
|
||||
a_frag[i] = bitcast_elem<T, E>(v);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
}
|
||||
static_assert(sizeof(a_frag) == 32, "V16 must be 32 bytes");
|
||||
__builtin_memcpy(&a_frag, a_row + k_tile, sizeof(a_frag));
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 16; i++) a_frag[i] = (E)0;
|
||||
@@ -1611,7 +1566,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k16(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -1712,8 +1667,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k16(
|
||||
dequant_into(next_buf, k_next);
|
||||
}
|
||||
|
||||
// A-load: vectorized 256-bit. b_q_perm branch removed from V7 hot path
|
||||
// to eliminate ~450 ISA instructions of dead code (6× icache bloat).
|
||||
// A-load: vectorized 256-bit.
|
||||
V16 a_frag, b_frag0, b_frag1, b_frag2, b_frag3;
|
||||
if (a_row_ptr) {
|
||||
__builtin_memcpy(&a_frag, a_row_ptr + k_tile, sizeof(a_frag));
|
||||
@@ -1810,7 +1764,7 @@ __global__ void gemm_q4_wmma_kernel_128x64_k32(
|
||||
const T* __restrict__ a, const uint32_t* __restrict__ b_q,
|
||||
const uint32_t* __restrict__ b_qzeros, const T* __restrict__ b_scales,
|
||||
T* __restrict__ c, const int size_m, const int size_n, const int size_k,
|
||||
const int groups, const int zero_offset, const int* __restrict__ b_q_perm) {
|
||||
const int groups, const int zero_offset) {
|
||||
using E = typename WmmaNative<T>::elem;
|
||||
using V16 = typename WmmaNative<T>::v16;
|
||||
|
||||
@@ -2006,40 +1960,36 @@ template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_64x64_4w(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int, const int*) {
|
||||
}
|
||||
const int, const int) {}
|
||||
template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_128x64_k16(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int,
|
||||
const int*) {}
|
||||
const int, const int) {}
|
||||
template <typename T>
|
||||
__global__ void gemm_q4_wmma_kernel_128x64_k32(const T*, const uint32_t*,
|
||||
const uint32_t*, const T*, T*,
|
||||
const int, const int, const int,
|
||||
const int, const int,
|
||||
const int*) {}
|
||||
const int, const int) {}
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const uint32_t* b_qzeros, const T* b_scales,
|
||||
const int* b_q_perm, T* c, int size_m,
|
||||
int size_n, int size_k, int groups,
|
||||
int zero_offset, cudaStream_t stream) {
|
||||
T* c, int size_m, int size_n, int size_k,
|
||||
int groups, int zero_offset,
|
||||
cudaStream_t stream) {
|
||||
// Fall back to v4 when N < 64 (would waste 1+ waves on out-of-range cols).
|
||||
if (size_m < 64 || size_n < 64) {
|
||||
launch_gemm_q4_wmma_64x32_4w<T>(a, b_q_weight, b_qzeros, b_scales, b_q_perm,
|
||||
c, size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
launch_gemm_q4_wmma_64x32_4w<T>(a, b_q_weight, b_qzeros, b_scales, c,
|
||||
size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
return;
|
||||
}
|
||||
|
||||
// V8 (128M × 64N, K=32/iter, 8-wave dequant) when K%32==0 and gs≥32.
|
||||
// Falls back to V7 otherwise. V7/V8 read A sequentially, so act-order
|
||||
// (b_q_perm != null) must skip them and use v5, which honors the perm.
|
||||
if (size_m >= 128 && b_q_perm == nullptr) {
|
||||
// Falls back to V7 otherwise.
|
||||
if (size_m >= 128) {
|
||||
const int k_split =
|
||||
compute_wmma_k_split_mn(size_m, size_n, size_k, 128, 64);
|
||||
const int groupsize = size_k / groups;
|
||||
@@ -2048,11 +1998,11 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
if (size_k % 32 == 0 && groupsize >= 32 && (size_k / k_split) % 32 == 0) {
|
||||
gemm_q4_wmma_kernel_128x64_k32<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
zero_offset);
|
||||
} else {
|
||||
gemm_q4_wmma_kernel_128x64_k16<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
zero_offset);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -2061,9 +2011,9 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
const int k_split = compute_wmma_k_split_mn(size_m, size_n, size_k, 64, 64);
|
||||
dim3 block(128);
|
||||
dim3 grid((size_n + 63) / 64, (size_m + 63) / 64, k_split);
|
||||
gemm_q4_wmma_kernel_64x64_4w<T><<<grid, block, 0, stream>>>(
|
||||
a, b_q_weight, b_qzeros, b_scales, c, size_m, size_n, size_k, groups,
|
||||
zero_offset, b_q_perm);
|
||||
gemm_q4_wmma_kernel_64x64_4w<T>
|
||||
<<<grid, block, 0, stream>>>(a, b_q_weight, b_qzeros, b_scales, c, size_m,
|
||||
size_n, size_k, groups, zero_offset);
|
||||
}
|
||||
|
||||
} // namespace gptq_rdna3_wmma
|
||||
@@ -2078,7 +2028,6 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
// b_q_weight[K/8, N] uint32 (already shuffled via gptq_shuffle)
|
||||
// b_qzeros [groups, N/8] uint32 (packed 4-bit zeros)
|
||||
// b_scales [groups, N] half or bfloat16
|
||||
// b_g_idx [K] or empty int32 (act-order permutation; empty=identity)
|
||||
// use_v2_format bool (true = GPTQv2, no +1 zero offset)
|
||||
//
|
||||
// Output:
|
||||
@@ -2091,8 +2040,7 @@ void launch_gemm_q4_wmma_64x64_4w(const T* a, const uint32_t* b_q_weight,
|
||||
|
||||
torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
torch::Tensor b_qzeros,
|
||||
torch::Tensor b_scales,
|
||||
torch::Tensor b_g_idx, bool use_v2_format) {
|
||||
torch::Tensor b_scales, bool use_v2_format) {
|
||||
TORCH_CHECK(a.is_cuda(), "a must be a CUDA/HIP tensor");
|
||||
TORCH_CHECK(b_q_weight.is_cuda(), "b_q_weight must be a CUDA/HIP tensor");
|
||||
TORCH_CHECK(b_qzeros.is_cuda(), "b_qzeros must be a CUDA/HIP tensor");
|
||||
@@ -2128,13 +2076,6 @@ torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
// negligible (< 1.5% of prefill time on gfx1100).
|
||||
at::Tensor c = torch::zeros({size_m, size_n}, opts);
|
||||
|
||||
const int* g_idx_ptr = nullptr;
|
||||
if (!b_g_idx.device().is_meta() && b_g_idx.numel() > 0) {
|
||||
TORCH_CHECK(b_g_idx.scalar_type() == torch::kInt32,
|
||||
"b_g_idx must be int32");
|
||||
g_idx_ptr = (const int*)b_g_idx.data_ptr();
|
||||
}
|
||||
|
||||
const int zero_offset = use_v2_format ? 0 : 1;
|
||||
|
||||
// launch_gemm_q4_wmma_64x64_4w dispatches:
|
||||
@@ -2148,15 +2089,15 @@ torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight,
|
||||
vllm::gptq_rdna3_wmma::launch_gemm_q4_wmma_64x64_4w<half>(
|
||||
(const half*)a.data_ptr(), (const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(), (const half*)b_scales.data_ptr(),
|
||||
g_idx_ptr, (half*)c.data_ptr(), size_m, size_n, size_k, groups,
|
||||
zero_offset, stream);
|
||||
(half*)c.data_ptr(), size_m, size_n, size_k, groups, zero_offset,
|
||||
stream);
|
||||
} else {
|
||||
vllm::gptq_rdna3_wmma::launch_gemm_q4_wmma_64x64_4w<
|
||||
vllm::gptq_rdna3_wmma::bf16_t>(
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)a.data_ptr(),
|
||||
(const uint32_t*)b_q_weight.data_ptr(),
|
||||
(const uint32_t*)b_qzeros.data_ptr(),
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)b_scales.data_ptr(), g_idx_ptr,
|
||||
(const vllm::gptq_rdna3_wmma::bf16_t*)b_scales.data_ptr(),
|
||||
(vllm::gptq_rdna3_wmma::bf16_t*)c.data_ptr(), size_m, size_n, size_k,
|
||||
groups, zero_offset, stream);
|
||||
}
|
||||
|
||||
@@ -56,12 +56,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
|
||||
// W4A16 GPTQ kernels for AMD RDNA3 (gfx1100).
|
||||
rocm_ops.def(
|
||||
"gptq_gemm_rdna3(Tensor a, Tensor b_q_weight, Tensor b_qzeros, "
|
||||
"Tensor b_scales, Tensor b_g_idx, bool use_v2_format) -> Tensor");
|
||||
"Tensor b_scales, bool use_v2_format) -> Tensor");
|
||||
rocm_ops.impl("gptq_gemm_rdna3", torch::kCUDA, &gptq_gemm_rdna3);
|
||||
|
||||
rocm_ops.def(
|
||||
"gptq_gemm_rdna3_wmma(Tensor a, Tensor b_q_weight, Tensor b_qzeros, "
|
||||
"Tensor b_scales, Tensor b_g_idx, bool use_v2_format) -> Tensor");
|
||||
"Tensor b_scales, bool use_v2_format) -> Tensor");
|
||||
rocm_ops.impl("gptq_gemm_rdna3_wmma", torch::kCUDA, &gptq_gemm_rdna3_wmma);
|
||||
|
||||
rocm_ops.def(
|
||||
|
||||
Reference in New Issue
Block a user