[CPU] Conv1d optimised kernel for aarch64 (#54093)

Signed-off-by: Anna Mayne <anna.mayne@arm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
almayne
2026-09-30 20:23:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent df5668e5c9
commit 93dd80f636
9 changed files with 460 additions and 205 deletions
+19 -7
View File
@@ -66,6 +66,10 @@ struct tinygemm_kernel {
}
};
#if defined(ARM_BF16_SUPPORT)
#include "conv_arm.h"
#endif
#if defined(CPU_CAPABILITY_AVX512)
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
@@ -543,16 +547,24 @@ at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) {
int64_t dim = weight.size(0);
int64_t width = weight.size(1);
constexpr int64_t BLOCK_N = block_size_n();
constexpr int64_t BLOCK_N =
#if defined(ARM_BF16_SUPPORT)
8;// 8 channel blocks for microtile
#else
block_size_n();
#endif
TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4");
TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim);
TORCH_CHECK(
dim % block_size_n() == 0,
"causal_conv1d_weight_pack: invalid dim size ",
dim);
const int64_t N = dim, K2 = width >> 1;
const int64_t NB = div_up(N, BLOCK_N);
auto packed_weight = at::empty_like(weight);
AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] {
// cast to float32 as vnni size is 2
// Copy each pair as an opaque 32-bit word. This is not a bf16->fp32 conversion.
const float* w_data = reinterpret_cast<float*>(weight.data_ptr<scalar_t>());
float* packed_data = reinterpret_cast<float*>(packed_weight.data_ptr<scalar_t>());
@@ -642,9 +654,9 @@ at::Tensor causal_conv1d_fwd_cpu(
const std::optional<at::Tensor>& has_initial_state,
bool silu_activation,
int64_t pad_slot_id,
bool is_vnni) {
bool is_weight_packed) {
CHECK_CONTIGUOUS(weight);
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
const bool is_var_seqlen = query_start_loc.has_value();
const int64_t input_ndim = is_var_seqlen ? 2 : 3;
@@ -755,10 +767,10 @@ at::Tensor causal_conv1d_update_cpu(
const std::optional<at::Tensor>& num_accepted_tokens,
const std::optional<at::Tensor>& conv_state_indices,
int64_t pad_slot_id,
bool is_vnni) {
bool is_weight_packed) {
CHECK_CONTIGUOUS(x);
CHECK_CONTIGUOUS(weight);
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
TORCH_CHECK(
x.dim() == 2 || x.dim() == 3,
+126
View File
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#pragma once
#if defined(ARM_BF16_SUPPORT)
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
static inline bfloat16x8_t load_bf16x8(const at::BFloat16* ptr) {
return vld1q_bf16(reinterpret_cast<const bfloat16_t*>(ptr));
}
// Input layout: A[m][channel]
// Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2]
// The packed weights are reused across all M positions.
static inline void tinygemm_neon_8(
const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B,
at::BFloat16* __restrict__ C, const at::BFloat16* __restrict__ bias,
const at::BFloat16* __restrict__ conv_states, bool has_initial_state,
int64_t M, int64_t lda, bool is_first_token) {
bfloat16x8_t va[4];
// Load 4 pairs of kernel weights into 4 registers
const bfloat16x8_t vb0 = load_bf16x8(B); // channels 0-3, taps 0-1
const bfloat16x8_t vb1 = load_bf16x8(B + 8); // channels 4-7, taps 0-1
const bfloat16x8_t vb2 = load_bf16x8(B + 16); // channels 0-3, taps 2-3
const bfloat16x8_t vb3 = load_bf16x8(B + 24); // channels 4-7, taps 2-3
// Load the input data. A single register holds the values for a
// single token across all 8 channels for this microtile.
auto load_a = [&](int64_t m) {
if (m < 0 && is_first_token) {
if (!has_initial_state) {
return vreinterpretq_bf16_u16(vdupq_n_u16(0));
}
return load_bf16x8(conv_states + (m + K - 1) * lda);
}
return load_bf16x8(A + m * lda);
};
// Load the previous inputs from the cached state if available.
va[1] = load_a(-3);
va[2] = load_a(-2);
va[3] = load_a(-1);
// Iterate over the sequence block, updating the previous conv inputs
// and reading the next token's input data (8 channels).
for (int64_t m = 0; m < M; ++m) {
va[0] = va[1];
va[1] = va[2];
va[2] = va[3];
va[3] = load_a(m);
float32x4_t vc0;
float32x4_t vc1;
if constexpr (has_bias) {
const bfloat16x8_t bias_vec = load_bf16x8(bias);
vc0 = vcvtq_low_f32_bf16(bias_vec);
vc1 = vcvtq_high_f32_bf16(bias_vec);
} else {
vc0 = vdupq_n_f32(0.0f);
vc1 = vdupq_n_f32(0.0f);
}
// Use the zip operation to match the input layout to the weights layout
bfloat16x8_t va_pair =
vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 0-1
vzip1q_u16(vreinterpretq_u16_bf16(va[0]),
vreinterpretq_u16_bf16(va[1])));
vc0 = vbfdotq_f32(vc0, va_pair,
vb0); // 4 independent length-two dot products
va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 0-1
vzip2q_u16(vreinterpretq_u16_bf16(va[0]),
vreinterpretq_u16_bf16(va[1])));
vc1 = vbfdotq_f32(vc1, va_pair, vb1);
va_pair = vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 2-3
vzip1q_u16(vreinterpretq_u16_bf16(va[2]),
vreinterpretq_u16_bf16(va[3])));
// Add to the previous result for the same channels
vc0 = vbfdotq_f32(vc0, va_pair, vb2);
va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 2-3
vzip2q_u16(vreinterpretq_u16_bf16(va[2]),
vreinterpretq_u16_bf16(va[3])));
vc1 = vbfdotq_f32(vc1, va_pair, vb3);
using fVec = at::vec::Vectorized<float>;
fVec out0(vc0);
fVec out1(vc1);
if constexpr (has_silu) {
const fVec one(1.0f);
out0 = out0 / (one + out0.neg().fexp_u20());
out1 = out1 / (one + out1.neg().fexp_u20());
}
// Convert the 2 x 4 fp32 results to 1 x 8 bf16
const bfloat16x8_t out =
vcombine_bf16(vcvt_bf16_f32(static_cast<float32x4_t>(out0)),
vcvt_bf16_f32(static_cast<float32x4_t>(out1)));
vst1q_bf16(reinterpret_cast<bfloat16_t*>(C + m * lda), out);
}
}
static inline void apply(const at::BFloat16* __restrict__ A,
const at::BFloat16* __restrict__ B,
at::BFloat16* __restrict__ C,
const at::BFloat16* __restrict__ bias,
const at::BFloat16* __restrict__ conv_states,
bool has_initial_state, int64_t M, int64_t lda,
bool is_first_token) {
static_assert(K == 4);
constexpr int64_t MICRO_N = 8;
static_assert(BLOCK_N % MICRO_N == 0);
// BLOCK_N is always 32 or 64. M is the block sequence length.
// Input layout: A[m][channel]
// Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2]
// Use an 8 channel microtile and iterate over the 8 channel blocks.
for (int64_t n = 0; n < BLOCK_N; n += MICRO_N) {
tinygemm_neon_8(A + n, B + n * K, C + n, has_bias ? bias + n : nullptr,
conv_states ? conv_states + n : nullptr,
has_initial_state, M, lda, is_first_token);
}
}
};
#endif
+29 -20
View File
@@ -281,14 +281,14 @@ at::Tensor causal_conv1d_fwd_cpu(
const std::optional<at::Tensor>& query_start_loc,
const std::optional<at::Tensor>& cache_indices,
const std::optional<at::Tensor>& has_initial_state, bool silu_activation,
int64_t pad_slot_id, bool is_vnni);
int64_t pad_slot_id, bool is_weight_packed);
at::Tensor causal_conv1d_update_cpu(
const at::Tensor& x, const at::Tensor& conv_states,
const at::Tensor& weight, const std::optional<at::Tensor>& bias,
bool silu_activation, const std::optional<at::Tensor>& num_accepted_tokens,
const std::optional<at::Tensor>& conv_state_indices, int64_t pad_slot_id,
bool is_vnni);
bool is_weight_packed);
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
const std::string& activation);
@@ -517,6 +517,33 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
#endif // (defined(__aarch64__) && !defined(__APPLE__))
#if (defined(__aarch64__) && defined(ARM_BF16_SUPPORT) && \
!defined(__APPLE__)) || \
(defined(__AVX512BF16__) && defined(__AVX512F__) && \
defined(__AVX512VNNI__))
// Adapted from sglang: casual_conv1d kernels
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
&causal_conv1d_weight_pack);
ops.def(
"causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? "
"conv_states, Tensor? query_start_loc,"
"Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, "
"int pad_slot_id, bool is_weight_packed) -> "
"Tensor");
ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu);
ops.def(
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
"weight, Tensor? bias, bool silu_activation,"
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
"pad_slot_id, "
"bool is_weight_packed) -> Tensor");
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
#endif
// Layernorm
// Apply Root Mean Square (RMS) Normalization to the input tensor.
ops.def(
@@ -665,24 +692,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"-> (Tensor, Tensor)");
ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec);
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
&causal_conv1d_weight_pack);
ops.def(
"causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? "
"conv_states, Tensor? query_start_loc,"
"Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, "
"int pad_slot_id, bool is_vnni) -> "
"Tensor");
ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu);
ops.def(
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
"weight, Tensor? bias, bool silu_activation,"
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
"pad_slot_id, "
"bool is_vnni) -> Tensor");
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
// Adapted from sglang: MLA CPU kernels (AMX-only, DeepSeek V2/V3/R1)
ops.def(
"decode_attention_cpu(Tensor query, Tensor k_buffer, Tensor v_buffer, "