[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
6df2b1a8c6
commit
b346479065
@@ -1114,5 +1114,203 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# FP8 W8A8 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
requires_cpu_fp8_w8a8 = pytest.mark.skipif(
|
||||
not ops.cpu_has_amx_fp8(),
|
||||
reason="requires native AMX-FP8 hardware",
|
||||
)
|
||||
|
||||
FP8_W8A8_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
FP8_W8A8_QUANT_GROUP = 128 # quantization group size for both K and N dimensions
|
||||
|
||||
|
||||
def _make_fp8_w8a8_weight_w13(
|
||||
E: int, two_n: int, K: int, group_K: int = FP8_W8A8_QUANT_GROUP
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Create w13 [E, 2N, K] FP8 with group scales [E, 2N, G] where G = K//group_K."""
|
||||
G = K // group_K
|
||||
w_list, s_list = [], []
|
||||
for _ in range(E):
|
||||
w_f32 = torch.randn(two_n, K)
|
||||
# Per-row, per-K-group quantization: scale [2N, G]
|
||||
w_re = w_f32.view(two_n, G, group_K)
|
||||
abs_max = w_re.abs().amax(dim=2, keepdim=True).clamp(min=1e-7) # [2N, G, 1]
|
||||
scale = (abs_max / FP8_W8A8_MAX).squeeze(2) # [2N, G]
|
||||
w_q = (
|
||||
(w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
|
||||
)
|
||||
w_list.append(w_q.view(two_n, K).contiguous())
|
||||
s_list.append(scale.float())
|
||||
return torch.stack(w_list), torch.stack(s_list) # [E, 2N, K], [E, 2N, G]
|
||||
|
||||
|
||||
def _make_fp8_w8a8_weight_w2(
|
||||
E: int,
|
||||
K: int,
|
||||
N: int,
|
||||
group_K: int = FP8_W8A8_QUANT_GROUP,
|
||||
group_N: int = FP8_W8A8_QUANT_GROUP,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Create w2 [E, K, N] FP8 with block scales [E, K//gK, N//gN]."""
|
||||
nK = K // group_K
|
||||
nN = N // group_N
|
||||
w_list, s_list = [], []
|
||||
for _ in range(E):
|
||||
w_f32 = torch.randn(K, N)
|
||||
w_re = w_f32.view(nK, group_K, nN, group_N)
|
||||
abs_max = w_re.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-7)
|
||||
scale = (abs_max / FP8_W8A8_MAX).squeeze(1).squeeze(2) # [nK, nN]
|
||||
w_q = (
|
||||
(w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
|
||||
)
|
||||
w_list.append(w_q.view(K, N).contiguous())
|
||||
s_list.append(scale.float())
|
||||
return torch.stack(w_list), torch.stack(s_list) # [E, K, N], [E, K//gK, N//gN]
|
||||
|
||||
|
||||
def _pack_fp8_w8a8_w13(
|
||||
w13: torch.Tensor, # [E, 2N, K] FP8
|
||||
w13_scale: torch.Tensor, # [E, 2N, G] float32
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pack w13 for CPU FP8 W8A8 MoE kernel using float8_linear_prepack_cpu."""
|
||||
E = w13.size(0)
|
||||
packed_list, scale_list = [], []
|
||||
for i in range(E):
|
||||
pw, ps = torch.ops._C.float8_linear_prepack_cpu(
|
||||
w13[i].contiguous(), w13_scale[i].contiguous()
|
||||
)
|
||||
packed_list.append(pw)
|
||||
scale_list.append(ps)
|
||||
return torch.stack(packed_list), torch.stack(scale_list)
|
||||
|
||||
|
||||
def _ref_fp8_w8a8_moe(
|
||||
hidden_states: torch.Tensor, # BF16 [M, K_hidden] (after FP8 round-trip dequant)
|
||||
w13: torch.Tensor, # FP8 [E, 2N, K_hidden]
|
||||
w13_scale: torch.Tensor, # float32 [E, 2N, G] (G = K_hidden//group_K)
|
||||
w2: torch.Tensor, # FP8 [E, K_hidden, N]
|
||||
w2_scale: torch.Tensor, # float32 [E, K_hidden//gK, N//gN]
|
||||
topk_weights: torch.Tensor, # float32 [M, top_k]
|
||||
topk_ids: torch.Tensor, # int64 [M, top_k]
|
||||
N: int,
|
||||
group_K: int = FP8_W8A8_QUANT_GROUP,
|
||||
group_N: int = FP8_W8A8_QUANT_GROUP,
|
||||
) -> torch.Tensor:
|
||||
"""Reference implementation: dequantize weights then compute MoE."""
|
||||
M, K_hidden = hidden_states.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
output = torch.zeros(M, K_hidden, dtype=torch.bfloat16)
|
||||
|
||||
for tok in range(M):
|
||||
x = hidden_states[tok].float() # [K_hidden]
|
||||
for k_idx in range(top_k):
|
||||
expert_id = topk_ids[tok, k_idx].item()
|
||||
w = topk_weights[tok, k_idx].item()
|
||||
|
||||
# Stage 1: x @ w13.T — gate and up projections
|
||||
w13_e = w13[expert_id].float() # [2N, K_hidden]
|
||||
ws_e = w13_scale[expert_id].float() # [2N, G]
|
||||
G = ws_e.shape[1]
|
||||
w13_dq = torch.zeros_like(w13_e)
|
||||
for g in range(G):
|
||||
c0, c1 = g * group_K, (g + 1) * group_K
|
||||
w13_dq[:, c0:c1] = w13_e[:, c0:c1] * ws_e[:, g : g + 1]
|
||||
gate_up = (x.unsqueeze(0) @ w13_dq.T).squeeze(0) # [2N]
|
||||
|
||||
gate = gate_up[:N]
|
||||
up = gate_up[N:]
|
||||
act = F.silu(gate) * up # [N]
|
||||
|
||||
# Stage 2: act @ w2.T — down projection
|
||||
w2_e = w2[expert_id].float() # [K_hidden, N]
|
||||
ws2_e = w2_scale[expert_id].float() # [K_hidden//gK, N//gN]
|
||||
nK2 = K_hidden // group_K
|
||||
nN2 = N // group_N
|
||||
w2_dq = torch.zeros_like(w2_e)
|
||||
for gi in range(nK2):
|
||||
for gj in range(nN2):
|
||||
r0, r1 = gi * group_K, (gi + 1) * group_K
|
||||
c0, c1 = gj * group_N, (gj + 1) * group_N
|
||||
w2_dq[r0:r1, c0:c1] = w2_e[r0:r1, c0:c1] * ws2_e[gi, gj]
|
||||
out_e = (act.unsqueeze(0) @ w2_dq.T).squeeze(0) # [K_hidden]
|
||||
|
||||
output[tok] += (w * out_e).to(torch.bfloat16)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize(
|
||||
"E,N,K,M,top_k",
|
||||
[
|
||||
# N and K must be multiples of FP8_W8A8_QUANT_GROUP=128
|
||||
(4, 128, 256, 1, 2), # single token (decode phase)
|
||||
(4, 128, 256, 8, 2),
|
||||
(8, 256, 512, 16, 2),
|
||||
(4, 256, 512, 4, 1),
|
||||
],
|
||||
)
|
||||
def test_fp8_w8a8_cpu_fused_moe(E: int, N: int, K: int, M: int, top_k: int):
|
||||
"""Test fused_experts_cpu FP8_W8A8 shape/dtype and accuracy vs a
|
||||
dequantized BF16 reference."""
|
||||
set_random_seed(42)
|
||||
|
||||
w13, w13_scale = _make_fp8_w8a8_weight_w13(E, 2 * N, K)
|
||||
w2, w2_scale = _make_fp8_w8a8_weight_w2(E, K, N)
|
||||
|
||||
packed_w13, packed_w13_scale = _pack_fp8_w8a8_w13(w13, w13_scale)
|
||||
packed_w2 = _prepack_experts(w2)
|
||||
|
||||
hidden_states = torch.randn(M, K, dtype=torch.bfloat16) * 0.1
|
||||
x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(hidden_states, True, None)
|
||||
assert x_fp8.dtype == torch.float8_e4m3fn
|
||||
assert x_scales.shape == (M,)
|
||||
|
||||
gen = torch.Generator().manual_seed(0)
|
||||
topk_weights = torch.softmax(torch.randn(M, top_k, generator=gen), dim=-1).float()
|
||||
topk_ids = torch.zeros(M, top_k, dtype=torch.int32)
|
||||
for i in range(M):
|
||||
topk_ids[i] = torch.randperm(E, generator=torch.Generator().manual_seed(i))[
|
||||
:top_k
|
||||
].int()
|
||||
|
||||
output = torch.empty(M, K, dtype=torch.bfloat16)
|
||||
ops.fused_experts_cpu(
|
||||
output,
|
||||
x_fp8,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
ops.CPUQuantMethod.FP8_W8A8,
|
||||
packed_w13_scale, # w1_scale: [E, Nc, G, BLOCK_N]
|
||||
w2_scale, # w2_scale: [E, K//128, N//128]
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
[FP8_W8A8_QUANT_GROUP, FP8_W8A8_QUANT_GROUP], # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni (weights already packed)
|
||||
x_scales, # a1_scale: per-token FP8 activation scales
|
||||
)
|
||||
|
||||
assert output.shape == (M, K)
|
||||
assert output.dtype == torch.bfloat16
|
||||
assert not torch.isnan(output).any()
|
||||
assert not torch.isinf(output).any()
|
||||
|
||||
x_dequant = (x_fp8.float() * x_scales.unsqueeze(1)).bfloat16()
|
||||
ref = _ref_fp8_w8a8_moe(
|
||||
x_dequant, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids.long(), N
|
||||
)
|
||||
torch.testing.assert_close(output.float(), ref.float(), atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU FP8 W8A16 block-scaled GEMM kernel (fp8_scaled_mm_cpu).
|
||||
"""Tests for CPU FP8 scaled-mm GEMM kernels.
|
||||
|
||||
Covers:
|
||||
* fp8_scaled_mm_cpu (W8A16)
|
||||
* fp8_scaled_mm_with_quant (W8A8)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_cpu_fp8_scaled_mm.py -v`.
|
||||
"""
|
||||
@@ -23,19 +27,31 @@ from vllm.platforms import current_platform
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
if not ops._supports_cpu_fp8_w8a16:
|
||||
pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True)
|
||||
requires_cpu_fp8_w8a16 = pytest.mark.skipif(
|
||||
not ops._supports_cpu_fp8_w8a16, reason="fp8_scaled_mm_cpu op not available"
|
||||
)
|
||||
requires_cpu_fp8_w8a8 = pytest.mark.skipif(
|
||||
not ops.cpu_has_amx_fp8(),
|
||||
reason="requires native AMX-FP8 hardware",
|
||||
)
|
||||
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
pytest.skip("requires AMX tile support", allow_module_level=True)
|
||||
|
||||
FP8_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
BLOCK_SIZE = [128, 128]
|
||||
GROUP_SIZE = 128
|
||||
|
||||
|
||||
def cdiv(a: int, b: int) -> int:
|
||||
return -(a // -b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A16 helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quantize_weight_block_fp8(
|
||||
weight: torch.Tensor,
|
||||
block_size: list[int],
|
||||
@@ -126,7 +142,7 @@ def ref_fp8_block_scaled_mm(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test parameters
|
||||
# W8A16 test: fp8_scaled_mm_cpu
|
||||
# ---------------------------------------------------------------------------
|
||||
M_SIZES = [1, 4, 16, 64, 128]
|
||||
# (N, K) — weight shape is [N, K], output has N columns.
|
||||
@@ -141,6 +157,7 @@ NK_SIZES = [
|
||||
]
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a16
|
||||
@pytest.mark.parametrize("M", M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
@@ -200,6 +217,77 @@ def ref_fp8_per_tensor_scaled_mm(
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A8 helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quantize_weight_per_tensor(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with a single per-tensor scale, broadcast
|
||||
to [N, 1] (the shape float8_linear_prepack_cpu expects)."""
|
||||
N = weight.shape[0]
|
||||
abs_max = weight.abs().max()
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q, scale.expand(N).contiguous().view(N, 1)
|
||||
|
||||
|
||||
def quantize_weight_per_channel(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with per-channel (per-row) scales [N, 1]."""
|
||||
abs_max = weight.abs().amax(dim=1, keepdim=True)
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q, scale.float()
|
||||
|
||||
|
||||
def quantize_weight_per_group(
|
||||
weight: torch.Tensor,
|
||||
group_size: int = GROUP_SIZE,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with per-group (128-block) scales [N, G]."""
|
||||
N, K = weight.shape
|
||||
G = K // group_size
|
||||
w_groups = weight.view(N, G, group_size)
|
||||
abs_max = w_groups.abs().amax(dim=2, keepdim=True)
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (w_groups.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q.view(N, K), scale.squeeze(2).float()
|
||||
|
||||
|
||||
def quantize_act_per_token(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference per-token (row) dynamic FP8 quantization of activations."""
|
||||
x_f = x.float()
|
||||
scale = (x_f.abs().amax(dim=1, keepdim=True) / FP8_MAX).clamp(min=1e-7)
|
||||
scaled = (x_f / scale).clamp(-FP8_MAX, FP8_MAX)
|
||||
# Round via fp16 to match the AVX10.2 kernel path.
|
||||
q = scaled.half().to(torch.float8_e4m3fn)
|
||||
return q, scale
|
||||
|
||||
|
||||
def reference_static_fp8_linear(
|
||||
x: torch.Tensor, # BF16 [M, K]
|
||||
weight_fp8: torch.Tensor, # FP8 [N, K]
|
||||
weight_scale: torch.Tensor, # float32 [N, 1]
|
||||
act_scale: torch.Tensor, # float32 scalar
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Static per-tensor: quantize/dequant x with act_scale, dequant weight, matmul."""
|
||||
x_scaled = (x.float() / act_scale.item()).clamp(-FP8_MAX, FP8_MAX)
|
||||
# Round via fp16 to match the AVX10.2 kernel path.
|
||||
x_q = x_scaled.half().to(torch.float8_e4m3fn)
|
||||
x_dq = x_q.float() * act_scale.item()
|
||||
w_dq = weight_fp8.float() * weight_scale.view(-1, 1)
|
||||
out = torch.mm(x_dq, w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
NK_SIZES_PER_TENSOR = [
|
||||
(32, 64),
|
||||
(5120, 5120),
|
||||
@@ -274,3 +362,136 @@ def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
|
||||
)
|
||||
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
|
||||
assert not supported
|
||||
|
||||
|
||||
def reference_dynamic_fp8_linear(
|
||||
x: torch.Tensor, # BF16 [M, K]
|
||||
weight_fp8: torch.Tensor, # FP8 [N, K]
|
||||
weight_scale: torch.Tensor, # float32 [N, G]
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dynamic per-token act x per-group weight: quantize x per-token,
|
||||
dequant weight per-group, matmul."""
|
||||
N, K = weight_fp8.shape
|
||||
G = weight_scale.shape[1]
|
||||
group_size = K // G
|
||||
x_q, x_scale = quantize_act_per_token(x)
|
||||
x_dq = x_q.float() * x_scale
|
||||
w_dq = (
|
||||
weight_fp8.float().view(N, G, group_size) * weight_scale.view(N, G, 1)
|
||||
).view(N, K)
|
||||
out = torch.mm(x_dq, w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A8 tests: fp8_scaled_mm_with_quant
|
||||
# ---------------------------------------------------------------------------
|
||||
W8A8_M_SIZES = [1, 4, 16, 64]
|
||||
W8A8_NK_SIZES = [(128, 256), (256, 512), (64, 128)]
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("static_activation", [True, False])
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_tensor(
|
||||
M: int, N: int, K: int, use_bias: bool, static_activation: bool
|
||||
):
|
||||
"""CPUFP8W8A8ScaledMMLinearKernel with per-tensor weight scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_tensor(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
|
||||
if static_activation:
|
||||
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_static_fp8_linear(
|
||||
x, w_fp8, w_scale, act_scale, bias, out_dtype
|
||||
)
|
||||
atol = 0.05
|
||||
else:
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
atol = 0.05
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("static_activation", [True, False])
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_channel(
|
||||
M: int, N: int, K: int, use_bias: bool, static_activation: bool
|
||||
):
|
||||
"""CPUFP8W8A8ScaledMMLinearKernel with per-channel weight scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_channel(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
|
||||
if static_activation:
|
||||
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_static_fp8_linear(
|
||||
x, w_fp8, w_scale, act_scale, bias, out_dtype
|
||||
)
|
||||
atol = 0.05
|
||||
else:
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
atol = 0.05
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_group(M: int, N: int, K: int, use_bias: bool):
|
||||
"""CPUFp8W8A8BlockScaledMMKernel: per-group (128-block) weight scale +
|
||||
dynamic per-token activation scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_group(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=0.05)
|
||||
|
||||
Reference in New Issue
Block a user