[Bugfix][CPU] Add per-tensor FP8 W8A16 kernel to fix Ministral crash on CPU (#56985)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-16 15:49:17 +08:00
committed by GitHub
parent f12fe10c5c
commit b68408043d
6 changed files with 233 additions and 7 deletions
@@ -9,6 +9,15 @@ import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
CPUFp8PerTensorScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
if not current_platform.is_cpu():
@@ -17,6 +26,9 @@ if not current_platform.is_cpu():
if not ops._supports_cpu_fp8_w8a16:
pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("requires AMX tile support", allow_module_level=True)
BLOCK_SIZE = [128, 128]
@@ -160,3 +172,104 @@ def test_cpu_fp8_scaled_mm(M: int, N: int, K: int, use_bias: bool):
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
def quantize_weight_per_tensor_fp8(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with a single per-tensor scale."""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
scale = weight.abs().amax() / fp8_max
scale = torch.where(scale == 0, torch.ones_like(scale), scale)
q = (weight / scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
return q, scale
def ref_fp8_per_tensor_scaled_mm(
x: torch.Tensor,
fp8_weight: torch.Tensor,
scale: torch.Tensor,
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
w_dq = fp8_weight.float() * scale
out = torch.mm(x.float(), w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
NK_SIZES_PER_TENSOR = [
(32, 64),
(5120, 5120),
]
@pytest.mark.parametrize("M", [1, 64])
@pytest.mark.parametrize("N,K", NK_SIZES_PER_TENSOR)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_per_tensor_scaled_mm_kernel(
M: int, N: int, K: int, use_bias: bool, default_vllm_config
):
"""CPUFp8PerTensorScaledMMLinearKernel correctness against float reference.
Exercises the full kernel class (not just the raw op), including the
weight-orientation fixup: Fp8LinearMethod stores `layer.weight` as
[K, N] (torch._scaled_mm convention) before calling
process_weights_after_loading, so the kernel must transpose back to
[N, K] before VNNI-packing.
"""
torch.manual_seed(0)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
fp8_weight, scale = quantize_weight_per_tensor_fp8(w_f32)
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
ref_out = ref_fp8_per_tensor_scaled_mm(x, fp8_weight, scale, bias, out_dtype)
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(N, K),
input_dtype=out_dtype,
out_dtype=out_dtype,
)
kernel = CPUFp8PerTensorScaledMMLinearKernel(
config,
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
)
layer = torch.nn.Module()
# Fp8LinearMethod stores the weight transposed to [K, N] before calling
# process_weights_after_loading.
layer.register_parameter(
"weight", torch.nn.Parameter(fp8_weight.t().contiguous(), requires_grad=False)
)
layer.register_parameter(
"weight_scale", torch.nn.Parameter(scale.clone(), requires_grad=False)
)
kernel.process_weights_after_loading(layer)
kernel_out = kernel.apply_weights(layer, x, bias)
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
@pytest.mark.parametrize("n", [16, 48])
def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
"""The AMX tinygemm kernel tiles N in chunks of 32 and cannot handle a
remainder tile smaller than that, so can_implement must reject any N
that isn't a multiple of 32 rather than let it crash the process.
"""
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(n, 64),
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
)
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
assert not supported