From 1b19bd758936496751432eccabf8adb7b5d8936a Mon Sep 17 00:00:00 2001 From: zofia <110436990+zufangzhu@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:59:20 +0800 Subject: [PATCH] [MXFP8] [XPU] add a new compressed tensor schema and add a xpu mxfp8 gemm kernel (#38707) Signed-off-by: Zhu, Zufang --- .../model_executor/kernels/linear/__init__.py | 8 +++ .../kernels/linear/mxfp8/xpu.py | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 vllm/model_executor/kernels/linear/mxfp8/xpu.py diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index b032f863d5..d0c0dd9f32 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -71,6 +71,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( from vllm.model_executor.kernels.linear.mxfp8.marlin import ( MarlinMxfp8LinearKernel, ) +from vllm.model_executor.kernels.linear.mxfp8.xpu import ( + XPUMxFp8LinearKernel, +) from vllm.model_executor.kernels.linear.nvfp4 import ( NvFp4LinearKernel, NvFp4LinearLayerConfig, @@ -243,6 +246,10 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = { PlatformEnum.ROCM: [ EmulationMxfp8LinearKernel, ], + PlatformEnum.XPU: [ + XPUMxFp8LinearKernel, + EmulationMxfp8LinearKernel, + ], } _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = { @@ -742,6 +749,7 @@ __all__ = [ "Mxfp8LinearLayerConfig", "FlashInferCutlassMxfp8LinearKernel", "MarlinMxfp8LinearKernel", + "XPUMxFp8LinearKernel", "EmulationMxfp8LinearKernel", "CutlassNvFp4LinearKernel", "EmulationNvFp4LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mxfp8/xpu.py b/vllm/model_executor/kernels/linear/mxfp8/xpu.py new file mode 100644 index 0000000000..d64e175c29 --- /dev/null +++ b/vllm/model_executor/kernels/linear/mxfp8/xpu.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + xpu_mxfp8_quantize as quant_mxfp8, +) +from vllm.model_executor.utils import replace_parameter +from vllm.platforms import current_platform + +from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig + + +class XPUMxFp8LinearKernel(Mxfp8LinearKernel): + """MXFP8 W8A8 GEMM on XPU.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_xpu(): + return False, "XPUMxFp8 only support on XPU" + return True, None + + @classmethod + def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu) + weight_scale = weight_scale.t().contiguous() + replace_parameter(layer, "weight", layer.weight.t()) + replace_parameter(layer, "weight_scale", weight_scale.data) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + out_dtype = x.dtype + x_fp8, x_scale = quant_mxfp8(x) + return torch.ops._xpu_C.fp8_gemm( + x_fp8, + layer.weight, + out_dtype, + x_scale, + layer.weight_scale, + bias, + )