[CPU] Conv1d optimised kernel for aarch64 (#54093)

Signed-off-by: Anna Mayne <anna.mayne@arm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
almayne
2026-09-30 20:23:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent df5668e5c9
commit 93dd80f636
9 changed files with 460 additions and 205 deletions
+19 -7
View File
@@ -66,6 +66,10 @@ struct tinygemm_kernel {
}
};
#if defined(ARM_BF16_SUPPORT)
#include "conv_arm.h"
#endif
#if defined(CPU_CAPABILITY_AVX512)
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
@@ -543,16 +547,24 @@ at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) {
int64_t dim = weight.size(0);
int64_t width = weight.size(1);
constexpr int64_t BLOCK_N = block_size_n();
constexpr int64_t BLOCK_N =
#if defined(ARM_BF16_SUPPORT)
8;// 8 channel blocks for microtile
#else
block_size_n();
#endif
TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4");
TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim);
TORCH_CHECK(
dim % block_size_n() == 0,
"causal_conv1d_weight_pack: invalid dim size ",
dim);
const int64_t N = dim, K2 = width >> 1;
const int64_t NB = div_up(N, BLOCK_N);
auto packed_weight = at::empty_like(weight);
AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] {
// cast to float32 as vnni size is 2
// Copy each pair as an opaque 32-bit word. This is not a bf16->fp32 conversion.
const float* w_data = reinterpret_cast<float*>(weight.data_ptr<scalar_t>());
float* packed_data = reinterpret_cast<float*>(packed_weight.data_ptr<scalar_t>());
@@ -642,9 +654,9 @@ at::Tensor causal_conv1d_fwd_cpu(
const std::optional<at::Tensor>& has_initial_state,
bool silu_activation,
int64_t pad_slot_id,
bool is_vnni) {
bool is_weight_packed) {
CHECK_CONTIGUOUS(weight);
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
const bool is_var_seqlen = query_start_loc.has_value();
const int64_t input_ndim = is_var_seqlen ? 2 : 3;
@@ -755,10 +767,10 @@ at::Tensor causal_conv1d_update_cpu(
const std::optional<at::Tensor>& num_accepted_tokens,
const std::optional<at::Tensor>& conv_state_indices,
int64_t pad_slot_id,
bool is_vnni) {
bool is_weight_packed) {
CHECK_CONTIGUOUS(x);
CHECK_CONTIGUOUS(weight);
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
TORCH_CHECK(
x.dim() == 2 || x.dim() == 3,