[CPU] Conv1d optimised kernel for aarch64 (#54093)
Signed-off-by: Anna Mayne <anna.mayne@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
df5668e5c9
commit
93dd80f636
@@ -66,6 +66,10 @@ struct tinygemm_kernel {
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
#include "conv_arm.h"
|
||||
#endif
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
|
||||
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
@@ -543,16 +547,24 @@ at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) {
|
||||
|
||||
int64_t dim = weight.size(0);
|
||||
int64_t width = weight.size(1);
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
constexpr int64_t BLOCK_N =
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
8;// 8 channel blocks for microtile
|
||||
#else
|
||||
block_size_n();
|
||||
#endif
|
||||
TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4");
|
||||
TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim);
|
||||
TORCH_CHECK(
|
||||
dim % block_size_n() == 0,
|
||||
"causal_conv1d_weight_pack: invalid dim size ",
|
||||
dim);
|
||||
|
||||
const int64_t N = dim, K2 = width >> 1;
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
auto packed_weight = at::empty_like(weight);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] {
|
||||
// cast to float32 as vnni size is 2
|
||||
// Copy each pair as an opaque 32-bit word. This is not a bf16->fp32 conversion.
|
||||
const float* w_data = reinterpret_cast<float*>(weight.data_ptr<scalar_t>());
|
||||
float* packed_data = reinterpret_cast<float*>(packed_weight.data_ptr<scalar_t>());
|
||||
|
||||
@@ -642,9 +654,9 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
const std::optional<at::Tensor>& has_initial_state,
|
||||
bool silu_activation,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
bool is_weight_packed) {
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
const bool is_var_seqlen = query_start_loc.has_value();
|
||||
const int64_t input_ndim = is_var_seqlen ? 2 : 3;
|
||||
@@ -755,10 +767,10 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
const std::optional<at::Tensor>& num_accepted_tokens,
|
||||
const std::optional<at::Tensor>& conv_state_indices,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
bool is_weight_packed) {
|
||||
CHECK_CONTIGUOUS(x);
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
TORCH_CHECK(
|
||||
x.dim() == 2 || x.dim() == 3,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
|
||||
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
|
||||
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
static inline bfloat16x8_t load_bf16x8(const at::BFloat16* ptr) {
|
||||
return vld1q_bf16(reinterpret_cast<const bfloat16_t*>(ptr));
|
||||
}
|
||||
|
||||
// Input layout: A[m][channel]
|
||||
// Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2]
|
||||
// The packed weights are reused across all M positions.
|
||||
static inline void tinygemm_neon_8(
|
||||
const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B,
|
||||
at::BFloat16* __restrict__ C, const at::BFloat16* __restrict__ bias,
|
||||
const at::BFloat16* __restrict__ conv_states, bool has_initial_state,
|
||||
int64_t M, int64_t lda, bool is_first_token) {
|
||||
bfloat16x8_t va[4];
|
||||
// Load 4 pairs of kernel weights into 4 registers
|
||||
const bfloat16x8_t vb0 = load_bf16x8(B); // channels 0-3, taps 0-1
|
||||
const bfloat16x8_t vb1 = load_bf16x8(B + 8); // channels 4-7, taps 0-1
|
||||
const bfloat16x8_t vb2 = load_bf16x8(B + 16); // channels 0-3, taps 2-3
|
||||
const bfloat16x8_t vb3 = load_bf16x8(B + 24); // channels 4-7, taps 2-3
|
||||
|
||||
// Load the input data. A single register holds the values for a
|
||||
// single token across all 8 channels for this microtile.
|
||||
auto load_a = [&](int64_t m) {
|
||||
if (m < 0 && is_first_token) {
|
||||
if (!has_initial_state) {
|
||||
return vreinterpretq_bf16_u16(vdupq_n_u16(0));
|
||||
}
|
||||
return load_bf16x8(conv_states + (m + K - 1) * lda);
|
||||
}
|
||||
return load_bf16x8(A + m * lda);
|
||||
};
|
||||
|
||||
// Load the previous inputs from the cached state if available.
|
||||
va[1] = load_a(-3);
|
||||
va[2] = load_a(-2);
|
||||
va[3] = load_a(-1);
|
||||
|
||||
// Iterate over the sequence block, updating the previous conv inputs
|
||||
// and reading the next token's input data (8 channels).
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
va[0] = va[1];
|
||||
va[1] = va[2];
|
||||
va[2] = va[3];
|
||||
va[3] = load_a(m);
|
||||
|
||||
float32x4_t vc0;
|
||||
float32x4_t vc1;
|
||||
if constexpr (has_bias) {
|
||||
const bfloat16x8_t bias_vec = load_bf16x8(bias);
|
||||
vc0 = vcvtq_low_f32_bf16(bias_vec);
|
||||
vc1 = vcvtq_high_f32_bf16(bias_vec);
|
||||
} else {
|
||||
vc0 = vdupq_n_f32(0.0f);
|
||||
vc1 = vdupq_n_f32(0.0f);
|
||||
}
|
||||
|
||||
// Use the zip operation to match the input layout to the weights layout
|
||||
bfloat16x8_t va_pair =
|
||||
vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 0-1
|
||||
vzip1q_u16(vreinterpretq_u16_bf16(va[0]),
|
||||
vreinterpretq_u16_bf16(va[1])));
|
||||
vc0 = vbfdotq_f32(vc0, va_pair,
|
||||
vb0); // 4 independent length-two dot products
|
||||
va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 0-1
|
||||
vzip2q_u16(vreinterpretq_u16_bf16(va[0]),
|
||||
vreinterpretq_u16_bf16(va[1])));
|
||||
vc1 = vbfdotq_f32(vc1, va_pair, vb1);
|
||||
va_pair = vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 2-3
|
||||
vzip1q_u16(vreinterpretq_u16_bf16(va[2]),
|
||||
vreinterpretq_u16_bf16(va[3])));
|
||||
// Add to the previous result for the same channels
|
||||
vc0 = vbfdotq_f32(vc0, va_pair, vb2);
|
||||
va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 2-3
|
||||
vzip2q_u16(vreinterpretq_u16_bf16(va[2]),
|
||||
vreinterpretq_u16_bf16(va[3])));
|
||||
vc1 = vbfdotq_f32(vc1, va_pair, vb3);
|
||||
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
fVec out0(vc0);
|
||||
fVec out1(vc1);
|
||||
if constexpr (has_silu) {
|
||||
const fVec one(1.0f);
|
||||
out0 = out0 / (one + out0.neg().fexp_u20());
|
||||
out1 = out1 / (one + out1.neg().fexp_u20());
|
||||
}
|
||||
|
||||
// Convert the 2 x 4 fp32 results to 1 x 8 bf16
|
||||
const bfloat16x8_t out =
|
||||
vcombine_bf16(vcvt_bf16_f32(static_cast<float32x4_t>(out0)),
|
||||
vcvt_bf16_f32(static_cast<float32x4_t>(out1)));
|
||||
vst1q_bf16(reinterpret_cast<bfloat16_t*>(C + m * lda), out);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void apply(const at::BFloat16* __restrict__ A,
|
||||
const at::BFloat16* __restrict__ B,
|
||||
at::BFloat16* __restrict__ C,
|
||||
const at::BFloat16* __restrict__ bias,
|
||||
const at::BFloat16* __restrict__ conv_states,
|
||||
bool has_initial_state, int64_t M, int64_t lda,
|
||||
bool is_first_token) {
|
||||
static_assert(K == 4);
|
||||
constexpr int64_t MICRO_N = 8;
|
||||
static_assert(BLOCK_N % MICRO_N == 0);
|
||||
|
||||
// BLOCK_N is always 32 or 64. M is the block sequence length.
|
||||
// Input layout: A[m][channel]
|
||||
// Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2]
|
||||
// Use an 8 channel microtile and iterate over the 8 channel blocks.
|
||||
for (int64_t n = 0; n < BLOCK_N; n += MICRO_N) {
|
||||
tinygemm_neon_8(A + n, B + n * K, C + n, has_bias ? bias + n : nullptr,
|
||||
conv_states ? conv_states + n : nullptr,
|
||||
has_initial_state, M, lda, is_first_token);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
+29
-20
@@ -281,14 +281,14 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
const std::optional<at::Tensor>& query_start_loc,
|
||||
const std::optional<at::Tensor>& cache_indices,
|
||||
const std::optional<at::Tensor>& has_initial_state, bool silu_activation,
|
||||
int64_t pad_slot_id, bool is_vnni);
|
||||
int64_t pad_slot_id, bool is_weight_packed);
|
||||
|
||||
at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& x, const at::Tensor& conv_states,
|
||||
const at::Tensor& weight, const std::optional<at::Tensor>& bias,
|
||||
bool silu_activation, const std::optional<at::Tensor>& num_accepted_tokens,
|
||||
const std::optional<at::Tensor>& conv_state_indices, int64_t pad_slot_id,
|
||||
bool is_vnni);
|
||||
bool is_weight_packed);
|
||||
|
||||
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const std::string& activation);
|
||||
@@ -517,6 +517,33 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
|
||||
#endif // (defined(__aarch64__) && !defined(__APPLE__))
|
||||
|
||||
#if (defined(__aarch64__) && defined(ARM_BF16_SUPPORT) && \
|
||||
!defined(__APPLE__)) || \
|
||||
(defined(__AVX512BF16__) && defined(__AVX512F__) && \
|
||||
defined(__AVX512VNNI__))
|
||||
|
||||
// Adapted from sglang: casual_conv1d kernels
|
||||
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
|
||||
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
|
||||
&causal_conv1d_weight_pack);
|
||||
|
||||
ops.def(
|
||||
"causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? "
|
||||
"conv_states, Tensor? query_start_loc,"
|
||||
"Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, "
|
||||
"int pad_slot_id, bool is_weight_packed) -> "
|
||||
"Tensor");
|
||||
ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu);
|
||||
|
||||
ops.def(
|
||||
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
|
||||
"weight, Tensor? bias, bool silu_activation,"
|
||||
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
|
||||
"pad_slot_id, "
|
||||
"bool is_weight_packed) -> Tensor");
|
||||
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
|
||||
#endif
|
||||
|
||||
// Layernorm
|
||||
// Apply Root Mean Square (RMS) Normalization to the input tensor.
|
||||
ops.def(
|
||||
@@ -665,24 +692,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"-> (Tensor, Tensor)");
|
||||
ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec);
|
||||
|
||||
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
|
||||
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
|
||||
&causal_conv1d_weight_pack);
|
||||
ops.def(
|
||||
"causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? "
|
||||
"conv_states, Tensor? query_start_loc,"
|
||||
"Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, "
|
||||
"int pad_slot_id, bool is_vnni) -> "
|
||||
"Tensor");
|
||||
ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu);
|
||||
ops.def(
|
||||
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
|
||||
"weight, Tensor? bias, bool silu_activation,"
|
||||
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
|
||||
"pad_slot_id, "
|
||||
"bool is_vnni) -> Tensor");
|
||||
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
|
||||
|
||||
// Adapted from sglang: MLA CPU kernels (AMX-only, DeepSeek V2/V3/R1)
|
||||
ops.def(
|
||||
"decode_attention_cpu(Tensor query, Tensor k_buffer, Tensor v_buffer, "
|
||||
|
||||
Reference in New Issue
Block a user