[CPU] Conv1d optimised kernel for aarch64 (#54093)
Signed-off-by: Anna Mayne <anna.mayne@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
df5668e5c9
commit
93dd80f636
@@ -66,6 +66,10 @@ struct tinygemm_kernel {
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
#include "conv_arm.h"
|
||||
#endif
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
|
||||
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
@@ -543,16 +547,24 @@ at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) {
|
||||
|
||||
int64_t dim = weight.size(0);
|
||||
int64_t width = weight.size(1);
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
constexpr int64_t BLOCK_N =
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
8;// 8 channel blocks for microtile
|
||||
#else
|
||||
block_size_n();
|
||||
#endif
|
||||
TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4");
|
||||
TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim);
|
||||
TORCH_CHECK(
|
||||
dim % block_size_n() == 0,
|
||||
"causal_conv1d_weight_pack: invalid dim size ",
|
||||
dim);
|
||||
|
||||
const int64_t N = dim, K2 = width >> 1;
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
auto packed_weight = at::empty_like(weight);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] {
|
||||
// cast to float32 as vnni size is 2
|
||||
// Copy each pair as an opaque 32-bit word. This is not a bf16->fp32 conversion.
|
||||
const float* w_data = reinterpret_cast<float*>(weight.data_ptr<scalar_t>());
|
||||
float* packed_data = reinterpret_cast<float*>(packed_weight.data_ptr<scalar_t>());
|
||||
|
||||
@@ -642,9 +654,9 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
const std::optional<at::Tensor>& has_initial_state,
|
||||
bool silu_activation,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
bool is_weight_packed) {
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
const bool is_var_seqlen = query_start_loc.has_value();
|
||||
const int64_t input_ndim = is_var_seqlen ? 2 : 3;
|
||||
@@ -755,10 +767,10 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
const std::optional<at::Tensor>& num_accepted_tokens,
|
||||
const std::optional<at::Tensor>& conv_state_indices,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
bool is_weight_packed) {
|
||||
CHECK_CONTIGUOUS(x);
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
TORCH_CHECK(
|
||||
x.dim() == 2 || x.dim() == 3,
|
||||
|
||||
Reference in New Issue
Block a user