[CPU] Speedup LM Head on Arm CPUs (#55352)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
6376c601ed
commit
e3f755b732
@@ -513,6 +513,17 @@ int64_t create_onednn_mm_handler(const torch::Tensor& b,
|
||||
args.ab_type = get_dnnl_type<scalar_t>();
|
||||
});
|
||||
|
||||
#ifdef VLLM_USE_ACL
|
||||
// TODO(fadara01): this is a hot-fix, remove once problem is addressed in the
|
||||
// AArch64 stack. hybrid kernels in ACL which are the fastest for LM head
|
||||
// layers are only enabled for BF16 x BF16 -> FP32 problem desc as they can't
|
||||
// convert FP32->BF16 internally. In this case do: oneDNN matmul with
|
||||
// bf16:bf16:fp32 then cast output down to bf16.
|
||||
if (b.scalar_type() == at::kBFloat16 && b.size(1) >= 64000) {
|
||||
args.c_type = dnnl::memory::data_type::f32;
|
||||
}
|
||||
#endif
|
||||
|
||||
return reinterpret_cast<int64_t>(new MatMulPrimitiveHandler(args));
|
||||
}
|
||||
|
||||
@@ -527,12 +538,22 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
|
||||
MatMulPrimitiveHandler* ptr =
|
||||
reinterpret_cast<MatMulPrimitiveHandler*>(handler_tensor.item<int64_t>());
|
||||
|
||||
torch::Tensor dnnl_output = c;
|
||||
// AArch64 specific case where we do a bf16 x bf16 -> fp32 matmul
|
||||
const bool convert_output =
|
||||
ptr->get_output_type() == dnnl::memory::data_type::f32 &&
|
||||
c.scalar_type() != at::kFloat;
|
||||
if (convert_output) {
|
||||
dnnl_output = torch::empty(c.sizes(), c.options().dtype(at::kFloat));
|
||||
}
|
||||
|
||||
// ACL matmuls expect contiguous source tensors
|
||||
#ifdef VLLM_USE_ACL
|
||||
torch::Tensor a_contig = a.contiguous();
|
||||
#endif
|
||||
|
||||
MatMulPrimitiveHandler::ExecArgs exec_args;
|
||||
exec_args.c_ptr = dnnl_output.data_ptr();
|
||||
|
||||
#ifdef VLLM_USE_ACL
|
||||
exec_args.a_m_size = a_contig.size(0);
|
||||
@@ -548,7 +569,7 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
|
||||
#ifdef VLLM_USE_ACL
|
||||
// ACL matmuls in oneDNN do not support a bias.
|
||||
// We handle a matmul with bias by doing: c = bias; c += matmul(a, b)
|
||||
c.copy_(bias.value());
|
||||
dnnl_output.copy_(bias.value());
|
||||
#else
|
||||
exec_args.bias_ptr = bias->data_ptr<scalar_t>();
|
||||
#endif
|
||||
@@ -563,8 +584,11 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
|
||||
exec_args.a_ptr = a.data_ptr<scalar_t>();
|
||||
|
||||
#endif
|
||||
exec_args.c_ptr = c.data_ptr<scalar_t>();
|
||||
|
||||
ptr->execute(exec_args);
|
||||
});
|
||||
|
||||
if (convert_output) {
|
||||
c.copy_(dnnl_output);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user