[CPU] Add MLA backend so DeepSeek-V2/V3 can run on CPU (#49453)
Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
ad5280255b
commit
13726c80fe
@@ -943,10 +943,14 @@ inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
|
||||
};
|
||||
|
||||
inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
|
||||
fmadd(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
|
||||
fmadd(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
|
||||
fmadd(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
|
||||
fmadd(acc.reg.val[3], a.reg.val[3], b.reg.val[3]);
|
||||
// NOTE: at::vec::fmadd(a, b, c) returns a * b + c and does not mutate its
|
||||
// arguments; we must capture the result back into acc, otherwise the FMA
|
||||
// is a no-op. On x86 fma is implemented as `acc = acc + a * b` which does
|
||||
// the assignment; the ARM version previously dropped the return value.
|
||||
acc.reg.val[0] = fmadd(a.reg.val[0], b.reg.val[0], acc.reg.val[0]);
|
||||
acc.reg.val[1] = fmadd(a.reg.val[1], b.reg.val[1], acc.reg.val[1]);
|
||||
acc.reg.val[2] = fmadd(a.reg.val[2], b.reg.val[2], acc.reg.val[2]);
|
||||
acc.reg.val[3] = fmadd(a.reg.val[3], b.reg.val[3], acc.reg.val[3]);
|
||||
};
|
||||
|
||||
inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
|
||||
@@ -967,10 +971,11 @@ inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
|
||||
std::tie(b0_low, b0_high) = convert_bfloat16_float(b.reg.val[0]);
|
||||
std::tie(b1_low, b1_high) = convert_bfloat16_float(b.reg.val[1]);
|
||||
|
||||
fmadd(acc.reg.val[0], a0_low, b0_low);
|
||||
fmadd(acc.reg.val[1], a0_high, b0_high);
|
||||
fmadd(acc.reg.val[2], a1_low, b1_low);
|
||||
fmadd(acc.reg.val[3], a1_high, b1_high);
|
||||
// Same fmadd semantic as the FP32 overload above: capture the result.
|
||||
acc.reg.val[0] = fmadd(a0_low, b0_low, acc.reg.val[0]);
|
||||
acc.reg.val[1] = fmadd(a0_high, b0_high, acc.reg.val[1]);
|
||||
acc.reg.val[2] = fmadd(a1_low, b1_low, acc.reg.val[2]);
|
||||
acc.reg.val[3] = fmadd(a1_high, b1_high, acc.reg.val[3]);
|
||||
};
|
||||
|
||||
template <>
|
||||
|
||||
@@ -57,6 +57,12 @@ void mla_decode_block_head(
|
||||
std::fill(max_val, max_val + HEAD_UNROLL, -FLT_MAX);
|
||||
|
||||
f32_vec_type acc_vec[BLOCK_SIZE][HEAD_UNROLL];
|
||||
// Explicitly zero-initialize the accumulator. On x86 the default
|
||||
// ctor of FP32Vec16 already sets zero, but on ARM (and any backend
|
||||
// whose Vectorized<float> default ctor is trivial) the array holds
|
||||
// undefined values and the fma below accumulates garbage.
|
||||
for (int b = 0; b < BLOCK_SIZE; ++b)
|
||||
for (int u = 0; u < HEAD_UNROLL; ++u) acc_vec[b][u] = f32_vec_type{0.0f};
|
||||
for (int i = 0; i < HEAD_DIM; i += QK_NUM_ELEM) {
|
||||
// load to registers
|
||||
qk_vec_type q_vec[HEAD_UNROLL];
|
||||
@@ -96,6 +102,9 @@ void mla_decode_block_head(
|
||||
}
|
||||
|
||||
f32_vec_type this_out[V_HEAD_DIM / V_NUM_ELEM][HEAD_UNROLL];
|
||||
// Same zero-initialization concern as acc_vec above.
|
||||
for (int i = 0; i < V_HEAD_DIM / V_NUM_ELEM; ++i)
|
||||
for (int u = 0; u < HEAD_UNROLL; ++u) this_out[i][u] = f32_vec_type{0.0f};
|
||||
|
||||
for (int block_offset = 0; block_offset < num_tokens; ++block_offset) {
|
||||
// load to registers
|
||||
|
||||
Reference in New Issue
Block a user