From a5bbd81e2e872eba255da9b9f8b86063c0d0cef0 Mon Sep 17 00:00:00 2001 From: Ma Jian Date: Sat, 23 May 2026 12:33:18 +0800 Subject: [PATCH] [XPU]feat: enable FP8 block-scaled quantization on XPU (#42952) Signed-off-by: Ma Jian Co-authored-by: Kunshang Ji --- vllm/model_executor/kernels/linear/__init__.py | 3 +++ .../model_executor/kernels/linear/scaled_mm/triton.py | 4 ++-- .../layers/quantization/input_quant_fp8.py | 11 ++++++++++- .../layers/quantization/utils/fp8_utils.py | 10 ++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 1899990bdd..1476cd707f 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -307,6 +307,9 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ PlatformEnum.CPU: [ CPUFp8BlockScaledMMKernel, ], + PlatformEnum.XPU: [ + TritonFp8BlockScaledMMKernel, + ], } _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = { diff --git a/vllm/model_executor/kernels/linear/scaled_mm/triton.py b/vllm/model_executor/kernels/linear/scaled_mm/triton.py index bb4392a6de..7003e727bf 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/triton.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/triton.py @@ -159,8 +159,8 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel): class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): @classmethod def is_supported(cls, compute_capability=None): - if not current_platform.is_cuda_alike(): - return False, "only cuda like devices are supported." + if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): + return False, "only cuda-like and xpu devices are supported." return True, None def apply_block_scaled_mm( diff --git a/vllm/model_executor/layers/quantization/input_quant_fp8.py b/vllm/model_executor/layers/quantization/input_quant_fp8.py index 0ae4ed3337..35e0b4533f 100644 --- a/vllm/model_executor/layers/quantization/input_quant_fp8.py +++ b/vllm/model_executor/layers/quantization/input_quant_fp8.py @@ -173,7 +173,16 @@ class QuantFP8(CustomOp): scale_ub: torch.Tensor | None = None, use_triton: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: - # XPU can use same code path as CUDA. + if self.is_group_quant and not self.static: + from vllm.model_executor.layers.quantization.utils import fp8_utils + + return fp8_utils.per_token_group_quant_fp8( + x, + group_size=self.group_size, + column_major_scales=self.column_major_scales, + dtype=_FP8_DTYPE, + use_ue8m0=self.use_ue8m0, + ) return self.forward_cuda(x, scale, scale_ub, use_triton) def forward_native( diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index f1a0209c6e..8b20c13a97 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -573,7 +573,7 @@ def per_token_group_quant_fp8( shape = x.shape[:-1] + (x.shape[-1] // group_size,) x_s = torch.empty(shape, device=x.device, dtype=torch.float32) - # prefer CUDA kernel if available + # prefer CUDA/XPU kernel if available # TODO(bnell): this causes some fp8 moe test to fail. if current_platform.is_cuda() and x.is_contiguous(): torch.ops._C.per_token_group_fp8_quant( @@ -590,6 +590,12 @@ def per_token_group_quant_fp8( ) return x_q, x_s + if current_platform.is_xpu() and x.is_contiguous(): + torch.ops._C.per_token_group_fp8_quant( + x, x_q, x_s, group_size, eps, fp8_min, fp8_max, use_ue8m0 + ) + return x_q, x_s + # TRITON FALLBACK M = x.numel() // group_size N = group_size @@ -861,7 +867,7 @@ def w8a8_triton_block_scaled_mm( # Triton cannot currently bind E8M0 scale tensors directly. On ROCm, # DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format, # so decode them to fp32 before launching the kernel. - if current_platform.is_rocm(): + if current_platform.is_rocm() or current_platform.is_xpu(): if As.dtype == torch.float8_e8m0fnu: As = _upcast_e8m0_to_fp32(As).contiguous() if Bs.dtype == torch.float8_e8m0fnu: