[perf][cpu] Accelerate BF16 GELU with LUT impl on Arm CPUs (#37469)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
18013df6ae
commit
445b7093fd
@@ -0,0 +1,71 @@
|
||||
#include "cpu_types.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include <ATen/ops/empty.h>
|
||||
#include <ATen/ops/gelu.h>
|
||||
#include <c10/util/BFloat16.h>
|
||||
|
||||
constexpr uint32_t ActivationLutSize = 1u << 16;
|
||||
|
||||
at::Tensor gelu_reference(const at::Tensor& x) { return at::gelu(x, "none"); }
|
||||
|
||||
void maybe_init_activation_lut_bf16(
|
||||
uint16_t* lut, std::once_flag& once,
|
||||
at::Tensor (*activation)(const at::Tensor&)) {
|
||||
std::call_once(once, [&]() {
|
||||
auto lut_input =
|
||||
at::empty({static_cast<int64_t>(ActivationLutSize)},
|
||||
at::TensorOptions().device(at::kCPU).dtype(at::kFloat));
|
||||
auto* lut_input_ptr = lut_input.data_ptr<float>();
|
||||
#pragma omp parallel for
|
||||
for (uint32_t i = 0; i < ActivationLutSize; ++i) {
|
||||
lut_input_ptr[i] = c10::detail::f32_from_bits(static_cast<uint16_t>(i));
|
||||
}
|
||||
|
||||
auto lut_output = activation(lut_input);
|
||||
const auto* lut_output_ptr = lut_output.data_ptr<float>();
|
||||
#pragma omp parallel for
|
||||
for (uint32_t i = 0; i < ActivationLutSize; ++i) {
|
||||
lut[i] = c10::detail::round_to_nearest_even(lut_output_ptr[i]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const uint16_t* lut, const char* op_name) {
|
||||
TORCH_CHECK(input.scalar_type() == at::kBFloat16, op_name,
|
||||
": input must be bfloat16");
|
||||
TORCH_CHECK(out.scalar_type() == at::kBFloat16, op_name,
|
||||
": out must be bfloat16");
|
||||
TORCH_CHECK(input.is_contiguous(), op_name, ": input must be contiguous");
|
||||
TORCH_CHECK(out.is_contiguous(), op_name, ": out must be contiguous");
|
||||
|
||||
const auto* src =
|
||||
reinterpret_cast<const uint16_t*>(input.data_ptr<at::BFloat16>());
|
||||
auto* dst = reinterpret_cast<uint16_t*>(out.data_ptr<at::BFloat16>());
|
||||
const int64_t n = input.numel();
|
||||
|
||||
CPU_KERNEL_GUARD_IN(activation_lut_bf16_impl)
|
||||
#pragma omp parallel for
|
||||
for (int64_t i = 0; i < n; ++i) {
|
||||
dst[i] = lut[src[i]];
|
||||
}
|
||||
CPU_KERNEL_GUARD_OUT(activation_lut_bf16_impl)
|
||||
}
|
||||
|
||||
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const std::string& activation) {
|
||||
if (activation == "gelu") {
|
||||
static std::array<uint16_t, ActivationLutSize> lut{};
|
||||
static std::once_flag once;
|
||||
maybe_init_activation_lut_bf16(lut.data(), once, gelu_reference);
|
||||
activation_lut_bf16(out, input, lut.data(), "gelu_lut");
|
||||
return;
|
||||
}
|
||||
|
||||
TORCH_CHECK(false, "Unsupported activation: ", activation);
|
||||
}
|
||||
@@ -85,6 +85,9 @@ at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros,
|
||||
at::Tensor& w_scales,
|
||||
std::optional<at::Tensor> bias);
|
||||
|
||||
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const std::string& activation);
|
||||
|
||||
torch::Tensor get_scheduler_metadata(
|
||||
const int64_t num_req, const int64_t num_heads_q,
|
||||
const int64_t num_heads_kv, const int64_t head_dim,
|
||||
@@ -231,6 +234,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
|
||||
ops.impl("gelu_quick", torch::kCPU, &gelu_quick);
|
||||
|
||||
#if (defined(__aarch64__) && !defined(__APPLE__))
|
||||
|
||||
ops.def(
|
||||
"activation_lut_bf16(Tensor! out, Tensor input, str activation)"
|
||||
" -> ()");
|
||||
ops.impl("activation_lut_bf16", torch::kCPU, &activation_lut_bf16);
|
||||
|
||||
#endif // (defined(__aarch64__) && !defined(__APPLE__))
|
||||
|
||||
// Layernorm
|
||||
// Apply Root Mean Square (RMS) Normalization to the input tensor.
|
||||
ops.def(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <torch/library.h>
|
||||
#include <tuple>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user