[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Yuwen Zhou
2026-09-18 17:59:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 6df2b1a8c6
commit b346479065
20 changed files with 2924 additions and 46 deletions
@@ -1114,5 +1114,203 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
)
# ===========================================================================
# FP8 W8A8 MoE
# ===========================================================================
requires_cpu_fp8_w8a8 = pytest.mark.skipif(
not ops.cpu_has_amx_fp8(),
reason="requires native AMX-FP8 hardware",
)
FP8_W8A8_MAX = torch.finfo(torch.float8_e4m3fn).max
FP8_W8A8_QUANT_GROUP = 128 # quantization group size for both K and N dimensions
def _make_fp8_w8a8_weight_w13(
E: int, two_n: int, K: int, group_K: int = FP8_W8A8_QUANT_GROUP
) -> tuple[torch.Tensor, torch.Tensor]:
"""Create w13 [E, 2N, K] FP8 with group scales [E, 2N, G] where G = K//group_K."""
G = K // group_K
w_list, s_list = [], []
for _ in range(E):
w_f32 = torch.randn(two_n, K)
# Per-row, per-K-group quantization: scale [2N, G]
w_re = w_f32.view(two_n, G, group_K)
abs_max = w_re.abs().amax(dim=2, keepdim=True).clamp(min=1e-7) # [2N, G, 1]
scale = (abs_max / FP8_W8A8_MAX).squeeze(2) # [2N, G]
w_q = (
(w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
)
w_list.append(w_q.view(two_n, K).contiguous())
s_list.append(scale.float())
return torch.stack(w_list), torch.stack(s_list) # [E, 2N, K], [E, 2N, G]
def _make_fp8_w8a8_weight_w2(
E: int,
K: int,
N: int,
group_K: int = FP8_W8A8_QUANT_GROUP,
group_N: int = FP8_W8A8_QUANT_GROUP,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Create w2 [E, K, N] FP8 with block scales [E, K//gK, N//gN]."""
nK = K // group_K
nN = N // group_N
w_list, s_list = [], []
for _ in range(E):
w_f32 = torch.randn(K, N)
w_re = w_f32.view(nK, group_K, nN, group_N)
abs_max = w_re.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-7)
scale = (abs_max / FP8_W8A8_MAX).squeeze(1).squeeze(2) # [nK, nN]
w_q = (
(w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
)
w_list.append(w_q.view(K, N).contiguous())
s_list.append(scale.float())
return torch.stack(w_list), torch.stack(s_list) # [E, K, N], [E, K//gK, N//gN]
def _pack_fp8_w8a8_w13(
w13: torch.Tensor, # [E, 2N, K] FP8
w13_scale: torch.Tensor, # [E, 2N, G] float32
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack w13 for CPU FP8 W8A8 MoE kernel using float8_linear_prepack_cpu."""
E = w13.size(0)
packed_list, scale_list = [], []
for i in range(E):
pw, ps = torch.ops._C.float8_linear_prepack_cpu(
w13[i].contiguous(), w13_scale[i].contiguous()
)
packed_list.append(pw)
scale_list.append(ps)
return torch.stack(packed_list), torch.stack(scale_list)
def _ref_fp8_w8a8_moe(
hidden_states: torch.Tensor, # BF16 [M, K_hidden] (after FP8 round-trip dequant)
w13: torch.Tensor, # FP8 [E, 2N, K_hidden]
w13_scale: torch.Tensor, # float32 [E, 2N, G] (G = K_hidden//group_K)
w2: torch.Tensor, # FP8 [E, K_hidden, N]
w2_scale: torch.Tensor, # float32 [E, K_hidden//gK, N//gN]
topk_weights: torch.Tensor, # float32 [M, top_k]
topk_ids: torch.Tensor, # int64 [M, top_k]
N: int,
group_K: int = FP8_W8A8_QUANT_GROUP,
group_N: int = FP8_W8A8_QUANT_GROUP,
) -> torch.Tensor:
"""Reference implementation: dequantize weights then compute MoE."""
M, K_hidden = hidden_states.shape
top_k = topk_ids.shape[1]
output = torch.zeros(M, K_hidden, dtype=torch.bfloat16)
for tok in range(M):
x = hidden_states[tok].float() # [K_hidden]
for k_idx in range(top_k):
expert_id = topk_ids[tok, k_idx].item()
w = topk_weights[tok, k_idx].item()
# Stage 1: x @ w13.T — gate and up projections
w13_e = w13[expert_id].float() # [2N, K_hidden]
ws_e = w13_scale[expert_id].float() # [2N, G]
G = ws_e.shape[1]
w13_dq = torch.zeros_like(w13_e)
for g in range(G):
c0, c1 = g * group_K, (g + 1) * group_K
w13_dq[:, c0:c1] = w13_e[:, c0:c1] * ws_e[:, g : g + 1]
gate_up = (x.unsqueeze(0) @ w13_dq.T).squeeze(0) # [2N]
gate = gate_up[:N]
up = gate_up[N:]
act = F.silu(gate) * up # [N]
# Stage 2: act @ w2.T — down projection
w2_e = w2[expert_id].float() # [K_hidden, N]
ws2_e = w2_scale[expert_id].float() # [K_hidden//gK, N//gN]
nK2 = K_hidden // group_K
nN2 = N // group_N
w2_dq = torch.zeros_like(w2_e)
for gi in range(nK2):
for gj in range(nN2):
r0, r1 = gi * group_K, (gi + 1) * group_K
c0, c1 = gj * group_N, (gj + 1) * group_N
w2_dq[r0:r1, c0:c1] = w2_e[r0:r1, c0:c1] * ws2_e[gi, gj]
out_e = (act.unsqueeze(0) @ w2_dq.T).squeeze(0) # [K_hidden]
output[tok] += (w * out_e).to(torch.bfloat16)
return output
@requires_cpu_fp8_w8a8
@pytest.mark.parametrize(
"E,N,K,M,top_k",
[
# N and K must be multiples of FP8_W8A8_QUANT_GROUP=128
(4, 128, 256, 1, 2), # single token (decode phase)
(4, 128, 256, 8, 2),
(8, 256, 512, 16, 2),
(4, 256, 512, 4, 1),
],
)
def test_fp8_w8a8_cpu_fused_moe(E: int, N: int, K: int, M: int, top_k: int):
"""Test fused_experts_cpu FP8_W8A8 shape/dtype and accuracy vs a
dequantized BF16 reference."""
set_random_seed(42)
w13, w13_scale = _make_fp8_w8a8_weight_w13(E, 2 * N, K)
w2, w2_scale = _make_fp8_w8a8_weight_w2(E, K, N)
packed_w13, packed_w13_scale = _pack_fp8_w8a8_w13(w13, w13_scale)
packed_w2 = _prepack_experts(w2)
hidden_states = torch.randn(M, K, dtype=torch.bfloat16) * 0.1
x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(hidden_states, True, None)
assert x_fp8.dtype == torch.float8_e4m3fn
assert x_scales.shape == (M,)
gen = torch.Generator().manual_seed(0)
topk_weights = torch.softmax(torch.randn(M, top_k, generator=gen), dim=-1).float()
topk_ids = torch.zeros(M, top_k, dtype=torch.int32)
for i in range(M):
topk_ids[i] = torch.randperm(E, generator=torch.Generator().manual_seed(i))[
:top_k
].int()
output = torch.empty(M, K, dtype=torch.bfloat16)
ops.fused_experts_cpu(
output,
x_fp8,
packed_w13,
packed_w2,
topk_weights,
topk_ids,
ops.CPUQuantMethod.FP8_W8A8,
packed_w13_scale, # w1_scale: [E, Nc, G, BLOCK_N]
w2_scale, # w2_scale: [E, K//128, N//128]
None, # w1_zero
None, # w2_zero
[FP8_W8A8_QUANT_GROUP, FP8_W8A8_QUANT_GROUP], # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni (weights already packed)
x_scales, # a1_scale: per-token FP8 activation scales
)
assert output.shape == (M, K)
assert output.dtype == torch.bfloat16
assert not torch.isnan(output).any()
assert not torch.isinf(output).any()
x_dequant = (x_fp8.float() * x_scales.unsqueeze(1)).bfloat16()
ref = _ref_fp8_w8a8_moe(
x_dequant, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids.long(), N
)
torch.testing.assert_close(output.float(), ref.float(), atol=1e-2, rtol=1e-2)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))