[Bugfix][CPU] Fix several bugs (#54042)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+33
-26
@@ -136,60 +136,67 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
}();
|
||||
|
||||
// allocate buffers
|
||||
int32_t common_buffer_offset = 0;
|
||||
int32_t w13_thread_buffer_offset = 0;
|
||||
int32_t ws_thread_buffer_offset = 0;
|
||||
// Sizes/offsets below are byte counts across the whole (token_num *
|
||||
// topk_num) batch, which for large hidden sizes and batch sizes can
|
||||
// exceed INT32_MAX. Use int64_t throughout to avoid silently wrapping
|
||||
// into a negative size that gets sign-extended into a huge value when
|
||||
// passed to ScratchPadManager::realloc(size_t).
|
||||
int64_t common_buffer_offset = 0;
|
||||
int64_t w13_thread_buffer_offset = 0;
|
||||
int64_t ws_thread_buffer_offset = 0;
|
||||
|
||||
// common buffers
|
||||
const int32_t token_num_per_group_buffer_size =
|
||||
const int64_t token_num_per_group_buffer_size =
|
||||
cpu_utils::round_up<64>(expert_num * sizeof(int32_t));
|
||||
const int32_t token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
const int64_t token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += token_num_per_group_buffer_size;
|
||||
|
||||
const int32_t cu_token_num_per_group_buffer_size =
|
||||
const int64_t cu_token_num_per_group_buffer_size =
|
||||
cpu_utils::round_up<64>((expert_num + 1) * sizeof(int32_t));
|
||||
const int32_t cu_token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
const int64_t cu_token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += cu_token_num_per_group_buffer_size;
|
||||
|
||||
const int32_t expand_token_id_buffer_size =
|
||||
cpu_utils::round_up<64>(token_num * topk_num * sizeof(int32_t));
|
||||
const int32_t expand_token_id_buffer_offset = common_buffer_offset;
|
||||
const int64_t expand_token_id_buffer_size = cpu_utils::round_up<64>(
|
||||
static_cast<int64_t>(token_num) * topk_num * sizeof(int32_t));
|
||||
const int64_t expand_token_id_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += expand_token_id_buffer_size;
|
||||
|
||||
const int32_t expand_token_id_index_buffer_size =
|
||||
cpu_utils::round_up<64>(token_num * topk_num * sizeof(int32_t));
|
||||
const int32_t expand_token_id_index_buffer_offset = common_buffer_offset;
|
||||
const int64_t expand_token_id_index_buffer_size = cpu_utils::round_up<64>(
|
||||
static_cast<int64_t>(token_num) * topk_num * sizeof(int32_t));
|
||||
const int64_t expand_token_id_index_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += expand_token_id_index_buffer_size;
|
||||
|
||||
const int32_t w13_gemm_output_buffer_size = cpu_utils::round_up<64>(
|
||||
token_num * topk_num * (output_size_13 / 2) * sizeof(scalar_t));
|
||||
const int32_t w13_gemm_output_buffer_offset = common_buffer_offset;
|
||||
const int64_t w13_gemm_output_buffer_size =
|
||||
cpu_utils::round_up<64>(static_cast<int64_t>(token_num) * topk_num *
|
||||
(output_size_13 / 2) * sizeof(scalar_t));
|
||||
const int64_t w13_gemm_output_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += w13_gemm_output_buffer_size;
|
||||
|
||||
const int32_t w2_gemm_output_buffer_size = cpu_utils::round_up<64>(
|
||||
token_num * topk_num * output_size_2 * sizeof(float));
|
||||
const int32_t w2_gemm_output_buffer_offset = common_buffer_offset;
|
||||
const int64_t w2_gemm_output_buffer_size =
|
||||
cpu_utils::round_up<64>(static_cast<int64_t>(token_num) * topk_num *
|
||||
output_size_2 * sizeof(float));
|
||||
const int64_t w2_gemm_output_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += w2_gemm_output_buffer_size;
|
||||
|
||||
// w13 GEMM thread buffers
|
||||
const int32_t w13_input_buffer_offset = w13_thread_buffer_offset;
|
||||
const int64_t w13_input_buffer_offset = w13_thread_buffer_offset;
|
||||
w13_thread_buffer_offset += w13_input_buffer_size;
|
||||
|
||||
const int32_t w13_output_buffer_size = cpu_utils::round_up<64>(
|
||||
const int64_t w13_output_buffer_size = cpu_utils::round_up<64>(
|
||||
gemm_m_tile_size * w13_n_tile_size * sizeof(float));
|
||||
const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
|
||||
const int64_t w13_output_buffer_offset = w13_thread_buffer_offset;
|
||||
w13_thread_buffer_offset += w13_output_buffer_size;
|
||||
|
||||
const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
|
||||
const int64_t w2_input_buffer_offset = w13_thread_buffer_offset;
|
||||
w13_thread_buffer_offset += w2_input_buffer_size;
|
||||
|
||||
// Weighted sum thread buffer
|
||||
const int32_t ws_output_buffer_size =
|
||||
const int64_t ws_output_buffer_size =
|
||||
cpu_utils::round_up<64>(output_size_2 * sizeof(float));
|
||||
const int32_t ws_output_buffer_offset = ws_thread_buffer_offset;
|
||||
const int64_t ws_output_buffer_offset = ws_thread_buffer_offset;
|
||||
ws_thread_buffer_offset += ws_output_buffer_size;
|
||||
|
||||
const int32_t buffer_size =
|
||||
const int64_t buffer_size =
|
||||
common_buffer_offset +
|
||||
std::max(w13_thread_buffer_offset, ws_thread_buffer_offset) * thread_num;
|
||||
cpu_utils::ScratchPadManager::get_scratchpad_manager()->realloc(buffer_size);
|
||||
|
||||
@@ -723,9 +723,10 @@ at::Tensor convert_scale_packed(at::Tensor& scale) {
|
||||
// bias : [N]
|
||||
// out : [*, N]
|
||||
//
|
||||
at::Tensor
|
||||
weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
|
||||
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
|
||||
void
|
||||
weight_packed_linear(at::Tensor& out, const at::Tensor& mat1, const at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
|
||||
TORCH_CHECK(is_vnni);
|
||||
auto packed_w = mat2;
|
||||
bool use_fma_gemm = false;
|
||||
if (packed_w.scalar_type() == at::kFloat) {
|
||||
use_fma_gemm = true;
|
||||
@@ -746,7 +747,6 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
|
||||
}
|
||||
|
||||
auto dispatch_type = mat1.scalar_type();
|
||||
auto out = at::empty({M, N}, mat1.options());
|
||||
// strides
|
||||
int64_t out_strideM = out.stride(0);
|
||||
int64_t mat1_strideM = mat1.stride(-2);
|
||||
@@ -784,9 +784,6 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
|
||||
out_strideM);
|
||||
}
|
||||
});
|
||||
|
||||
input_sizes[ndim - 1] = N;
|
||||
return out.view(input_sizes);
|
||||
}
|
||||
|
||||
// mat1 : [M, K]
|
||||
|
||||
@@ -58,9 +58,9 @@ std::vector<torch::Tensor> shm_recv_tensor_list(int64_t handle, int64_t src);
|
||||
|
||||
// SGL CPU kernels
|
||||
|
||||
at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
bool is_vnni);
|
||||
void weight_packed_linear(at::Tensor& out, const at::Tensor& mat1,
|
||||
const at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias, bool is_vnni);
|
||||
|
||||
at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
|
||||
@@ -507,8 +507,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
// sgl-kernels
|
||||
#if defined(__AVX512BF16__) && defined(__AVX512F__) && defined(__AVX512VNNI__)
|
||||
ops.def(
|
||||
"weight_packed_linear(Tensor(a0!) mat1, Tensor(a1!) mat2, Tensor(a2!)? "
|
||||
"bias, bool is_vnni) -> Tensor");
|
||||
"weight_packed_linear(Tensor(a0!) out, Tensor(a1) mat1, Tensor(a2) mat2, "
|
||||
"Tensor(a3)? "
|
||||
"bias, bool is_vnni) -> ()");
|
||||
ops.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear);
|
||||
ops.def("convert_weight_packed(Tensor! weight) -> Tensor");
|
||||
ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed);
|
||||
|
||||
Reference in New Issue
Block a user