[CPU] Fold the MXFP4 block scale in 2 instructions instead of 4 (#51583)

Signed-off-by: ccaadaro <ccanadar@unex.es>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
ccaadaro
2026-08-14 08:19:21 +00:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 8e6d8e4f6a
commit bda4c3e8ee
2 changed files with 135 additions and 11 deletions
+16 -11
View File
@@ -204,7 +204,6 @@ inline std::tuple<__m512bh, __m512bh> cvt_mxfp4_e2m1_bf16_intrinsic_lut(__m256i
const __m512i lut = (__m512i)(_mm512_cvtne2ps_pbh(values, values));
const __m512i abs_mask = _mm512_set1_epi16(0x7FFF);
const __m512i zero = _mm512_setzero_si512();
// expand values to 16-bit integers
__m512i x0 = _mm512_cvtepu8_epi16(a);
@@ -214,17 +213,23 @@ inline std::tuple<__m512bh, __m512bh> cvt_mxfp4_e2m1_bf16_intrinsic_lut(__m256i
x0 = _mm512_permutexvar_epi16(x0, lut);
x1 = _mm512_permutexvar_epi16(x1, lut);
// check for zeros
__mmask32 mask0 = _mm512_cmp_epi16_mask(_mm512_and_si512(x0, abs_mask), zero, _MM_CMPINT_EQ);
__mmask32 mask1 = _mm512_cmp_epi16_mask(_mm512_and_si512(x1, abs_mask), zero, _MM_CMPINT_EQ);
// Emulate the bf16 multiply by the E8M0 scale as an integer add on the
// exponent field, forcing zeros to stay zero.
//
// vptestmw sets a lane's mask bit when (x & 0x7FFF) != 0, i.e. for the lanes
// that are not +0.0 or -0.0. That is the *complement* of the mask the old
// and/cmp/add/blend sequence built, which selected the zero lanes; the
// zero-masking form of vpaddw then writes 0 where the mask is clear, so the
// two agree lane by lane. That includes the LUT's -0.0 entry, which both
// versions turn into +0.0.
//
// 2 instructions per vector instead of 4, no extra ISA requirement:
// vptestmw is AVX512BW, which vpermw above already needs.
__mmask32 mask0 = _mm512_test_epi16_mask(x0, abs_mask);
__mmask32 mask1 = _mm512_test_epi16_mask(x1, abs_mask);
// emulate bf16 mul with scale factor
x0 = _mm512_add_epi16(x0, s0);
x1 = _mm512_add_epi16(x1, s1);
// blend with zero
x0 = _mm512_mask_blend_epi16(mask0, x0, zero);
x1 = _mm512_mask_blend_epi16(mask1, x1, zero);
x0 = _mm512_maskz_add_epi16(mask0, x0, s0);
x1 = _mm512_maskz_add_epi16(mask1, x1, s1);
return std::make_tuple(__m512bh(x0), __m512bh(x1));
}