[CPU] Speedup LM Head on Arm CPUs (#55352)

Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Fadi Arafeh
2026-09-11 16:21:14 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 6376c601ed
commit e3f755b732
4 changed files with 37 additions and 6 deletions
+26 -2
View File
@@ -513,6 +513,17 @@ int64_t create_onednn_mm_handler(const torch::Tensor& b,
args.ab_type = get_dnnl_type<scalar_t>();
});
#ifdef VLLM_USE_ACL
// TODO(fadara01): this is a hot-fix, remove once problem is addressed in the
// AArch64 stack. hybrid kernels in ACL which are the fastest for LM head
// layers are only enabled for BF16 x BF16 -> FP32 problem desc as they can't
// convert FP32->BF16 internally. In this case do: oneDNN matmul with
// bf16:bf16:fp32 then cast output down to bf16.
if (b.scalar_type() == at::kBFloat16 && b.size(1) >= 64000) {
args.c_type = dnnl::memory::data_type::f32;
}
#endif
return reinterpret_cast<int64_t>(new MatMulPrimitiveHandler(args));
}
@@ -527,12 +538,22 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
MatMulPrimitiveHandler* ptr =
reinterpret_cast<MatMulPrimitiveHandler*>(handler_tensor.item<int64_t>());
torch::Tensor dnnl_output = c;
// AArch64 specific case where we do a bf16 x bf16 -> fp32 matmul
const bool convert_output =
ptr->get_output_type() == dnnl::memory::data_type::f32 &&
c.scalar_type() != at::kFloat;
if (convert_output) {
dnnl_output = torch::empty(c.sizes(), c.options().dtype(at::kFloat));
}
// ACL matmuls expect contiguous source tensors
#ifdef VLLM_USE_ACL
torch::Tensor a_contig = a.contiguous();
#endif
MatMulPrimitiveHandler::ExecArgs exec_args;
exec_args.c_ptr = dnnl_output.data_ptr();
#ifdef VLLM_USE_ACL
exec_args.a_m_size = a_contig.size(0);
@@ -548,7 +569,7 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
#ifdef VLLM_USE_ACL
// ACL matmuls in oneDNN do not support a bias.
// We handle a matmul with bias by doing: c = bias; c += matmul(a, b)
c.copy_(bias.value());
dnnl_output.copy_(bias.value());
#else
exec_args.bias_ptr = bias->data_ptr<scalar_t>();
#endif
@@ -563,8 +584,11 @@ void onednn_mm(torch::Tensor& c, // [M, OC], row-major
exec_args.a_ptr = a.data_ptr<scalar_t>();
#endif
exec_args.c_ptr = c.data_ptr<scalar_t>();
ptr->execute(exec_args);
});
if (convert_output) {
c.copy_(dnnl_output);
}
}