[Bugfix][CPU] Fix several bugs (#54042)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-08-31 15:33:06 +08:00
committed by GitHub
parent 2cf82bcdd1
commit fdbf2ddbd2
11 changed files with 118 additions and 85 deletions
+33 -26
View File
@@ -136,60 +136,67 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
}();
// allocate buffers
int32_t common_buffer_offset = 0;
int32_t w13_thread_buffer_offset = 0;
int32_t ws_thread_buffer_offset = 0;
// Sizes/offsets below are byte counts across the whole (token_num *
// topk_num) batch, which for large hidden sizes and batch sizes can
// exceed INT32_MAX. Use int64_t throughout to avoid silently wrapping
// into a negative size that gets sign-extended into a huge value when
// passed to ScratchPadManager::realloc(size_t).
int64_t common_buffer_offset = 0;
int64_t w13_thread_buffer_offset = 0;
int64_t ws_thread_buffer_offset = 0;
// common buffers
const int32_t token_num_per_group_buffer_size =
const int64_t token_num_per_group_buffer_size =
cpu_utils::round_up<64>(expert_num * sizeof(int32_t));
const int32_t token_num_per_group_buffer_offset = common_buffer_offset;
const int64_t token_num_per_group_buffer_offset = common_buffer_offset;
common_buffer_offset += token_num_per_group_buffer_size;
const int32_t cu_token_num_per_group_buffer_size =
const int64_t cu_token_num_per_group_buffer_size =
cpu_utils::round_up<64>((expert_num + 1) * sizeof(int32_t));
const int32_t cu_token_num_per_group_buffer_offset = common_buffer_offset;
const int64_t cu_token_num_per_group_buffer_offset = common_buffer_offset;
common_buffer_offset += cu_token_num_per_group_buffer_size;
const int32_t expand_token_id_buffer_size =
cpu_utils::round_up<64>(token_num * topk_num * sizeof(int32_t));
const int32_t expand_token_id_buffer_offset = common_buffer_offset;
const int64_t expand_token_id_buffer_size = cpu_utils::round_up<64>(
static_cast<int64_t>(token_num) * topk_num * sizeof(int32_t));
const int64_t expand_token_id_buffer_offset = common_buffer_offset;
common_buffer_offset += expand_token_id_buffer_size;
const int32_t expand_token_id_index_buffer_size =
cpu_utils::round_up<64>(token_num * topk_num * sizeof(int32_t));
const int32_t expand_token_id_index_buffer_offset = common_buffer_offset;
const int64_t expand_token_id_index_buffer_size = cpu_utils::round_up<64>(
static_cast<int64_t>(token_num) * topk_num * sizeof(int32_t));
const int64_t expand_token_id_index_buffer_offset = common_buffer_offset;
common_buffer_offset += expand_token_id_index_buffer_size;
const int32_t w13_gemm_output_buffer_size = cpu_utils::round_up<64>(
token_num * topk_num * (output_size_13 / 2) * sizeof(scalar_t));
const int32_t w13_gemm_output_buffer_offset = common_buffer_offset;
const int64_t w13_gemm_output_buffer_size =
cpu_utils::round_up<64>(static_cast<int64_t>(token_num) * topk_num *
(output_size_13 / 2) * sizeof(scalar_t));
const int64_t w13_gemm_output_buffer_offset = common_buffer_offset;
common_buffer_offset += w13_gemm_output_buffer_size;
const int32_t w2_gemm_output_buffer_size = cpu_utils::round_up<64>(
token_num * topk_num * output_size_2 * sizeof(float));
const int32_t w2_gemm_output_buffer_offset = common_buffer_offset;
const int64_t w2_gemm_output_buffer_size =
cpu_utils::round_up<64>(static_cast<int64_t>(token_num) * topk_num *
output_size_2 * sizeof(float));
const int64_t w2_gemm_output_buffer_offset = common_buffer_offset;
common_buffer_offset += w2_gemm_output_buffer_size;
// w13 GEMM thread buffers
const int32_t w13_input_buffer_offset = w13_thread_buffer_offset;
const int64_t w13_input_buffer_offset = w13_thread_buffer_offset;
w13_thread_buffer_offset += w13_input_buffer_size;
const int32_t w13_output_buffer_size = cpu_utils::round_up<64>(
const int64_t w13_output_buffer_size = cpu_utils::round_up<64>(
gemm_m_tile_size * w13_n_tile_size * sizeof(float));
const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
const int64_t w13_output_buffer_offset = w13_thread_buffer_offset;
w13_thread_buffer_offset += w13_output_buffer_size;
const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
const int64_t w2_input_buffer_offset = w13_thread_buffer_offset;
w13_thread_buffer_offset += w2_input_buffer_size;
// Weighted sum thread buffer
const int32_t ws_output_buffer_size =
const int64_t ws_output_buffer_size =
cpu_utils::round_up<64>(output_size_2 * sizeof(float));
const int32_t ws_output_buffer_offset = ws_thread_buffer_offset;
const int64_t ws_output_buffer_offset = ws_thread_buffer_offset;
ws_thread_buffer_offset += ws_output_buffer_size;
const int32_t buffer_size =
const int64_t buffer_size =
common_buffer_offset +
std::max(w13_thread_buffer_offset, ws_thread_buffer_offset) * thread_num;
cpu_utils::ScratchPadManager::get_scratchpad_manager()->realloc(buffer_size);
+4 -7
View File
@@ -723,9 +723,10 @@ at::Tensor convert_scale_packed(at::Tensor& scale) {
// bias : [N]
// out : [*, N]
//
at::Tensor
weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
void
weight_packed_linear(at::Tensor& out, const at::Tensor& mat1, const at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
TORCH_CHECK(is_vnni);
auto packed_w = mat2;
bool use_fma_gemm = false;
if (packed_w.scalar_type() == at::kFloat) {
use_fma_gemm = true;
@@ -746,7 +747,6 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
}
auto dispatch_type = mat1.scalar_type();
auto out = at::empty({M, N}, mat1.options());
// strides
int64_t out_strideM = out.stride(0);
int64_t mat1_strideM = mat1.stride(-2);
@@ -784,9 +784,6 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
out_strideM);
}
});
input_sizes[ndim - 1] = N;
return out.view(input_sizes);
}
// mat1 : [M, K]
+6 -5
View File
@@ -58,9 +58,9 @@ std::vector<torch::Tensor> shm_recv_tensor_list(int64_t handle, int64_t src);
// SGL CPU kernels
at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
const std::optional<at::Tensor>& bias,
bool is_vnni);
void weight_packed_linear(at::Tensor& out, const at::Tensor& mat1,
const at::Tensor& mat2,
const std::optional<at::Tensor>& bias, bool is_vnni);
at::Tensor convert_weight_packed(at::Tensor& weight);
@@ -507,8 +507,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
// sgl-kernels
#if defined(__AVX512BF16__) && defined(__AVX512F__) && defined(__AVX512VNNI__)
ops.def(
"weight_packed_linear(Tensor(a0!) mat1, Tensor(a1!) mat2, Tensor(a2!)? "
"bias, bool is_vnni) -> Tensor");
"weight_packed_linear(Tensor(a0!) out, Tensor(a1) mat1, Tensor(a2) mat2, "
"Tensor(a3)? "
"bias, bool is_vnni) -> ()");
ops.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear);
ops.def("convert_weight_packed(Tensor! weight) -> Tensor");
ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed);