From 1a308c449c93d25b11dc79a199ed67db185a18df Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Sat, 4 Jul 2026 18:10:01 +0800 Subject: [PATCH] [XPU] Add W8A8 FP8 linear kernel with multi-granularity quant support (#43645) Signed-off-by: Chaojun,Zhang Signed-off-by: Chaojun Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .buildkite/intel_jobs/test-intel.yaml | 49 +++++++- docs/features/quantization/online.md | 2 + vllm/config/kernel.py | 7 +- .../model_executor/kernels/linear/__init__.py | 15 ++- .../kernels/linear/scaled_mm/xpu.py | 111 +++++++++++++++++- .../compressed_tensors/compressed_tensors.py | 16 ++- 6 files changed, 188 insertions(+), 12 deletions(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 4b8ba21256..203e63f25e 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -49,6 +49,30 @@ steps: VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 ' + - label: "XPU W8A8 FP8 Linear Examples" + depends_on: + - image-build-xpu + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + ' - label: "XPU V1 test" depends_on: - image-build-xpu @@ -120,4 +144,27 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s quantization/test_auto_round.py' \ No newline at end of file + pytest -v -s quantization/test_auto_round.py' + - label: "XPU compressed tensors FP8 test" + depends_on: + - image-build-xpu + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/quantization/test_compressed_tensors.py + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8' \ No newline at end of file diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index 4b9571b38b..a4da52557a 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -62,6 +62,8 @@ weight name. Unset fields fall back to the `--quantization` shorthand's defaults, or for already-quantized checkpoints to whatever the checkpoint declares. +On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). + The CLI accepts the same shape as JSON or as dotted keys: ```bash diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 1d5b421715..e98ab1b3e0 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -156,6 +156,8 @@ LinearBackend = Literal[ "conch", "exllama", "emulation", + "xpu", + "xpu_woq", ] @@ -217,7 +219,10 @@ class KernelConfig: - "fbgemm": Use FBGEMM kernels - "conch": Use Conch mixed-precision kernels - "exllama": Use Exllama mixed-precision kernels - - "emulation": Use slow dequant-to-BF16 emulation (for testing only)""" + - "emulation": Use slow dequant-to-BF16 emulation (for testing only) + - "xpu": Use XPU kernels + - "xpu_woq": Use XPU kernels for weight-only quantization (e.g. W8A16) + """ @field_validator("moe_backend", mode="before") @classmethod diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 4ac8d49cd5..f5e3a16d71 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -171,7 +171,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import ( ) from vllm.model_executor.kernels.linear.scaled_mm.xpu import ( XPUFp8BlockScaledMMKernel, - XPUFP8ScaledMMLinearKernel, + XPUW8A8FP8LinearKernel, + XPUW8A16FP8LinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.zentorch import ( ZentorchInt8ScaledMMLinearKernel, @@ -266,6 +267,13 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { EmulationMxfp8LinearKernel, EmulationNvFp4LinearKernel, }, + "xpu": { + XPUW8A8FP8LinearKernel, + XPUFp8BlockScaledMMKernel, + }, + "xpu_woq": { + XPUW8A16FP8LinearKernel, + }, } @@ -311,7 +319,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = ChannelWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.XPU: [ - XPUFP8ScaledMMLinearKernel, + XPUW8A16FP8LinearKernel, + XPUW8A8FP8LinearKernel, ], } @@ -351,7 +360,7 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel] # To be added ], PlatformEnum.XPU: [ - XPUFP8ScaledMMLinearKernel, + XPUW8A16FP8LinearKernel, ], } diff --git a/vllm/model_executor/kernels/linear/scaled_mm/xpu.py b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py index 6afa52bf87..3d23b02bea 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/xpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py @@ -6,8 +6,11 @@ from collections.abc import Sequence import torch from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kFp8StaticTokenSym, ) from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform @@ -16,13 +19,112 @@ from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig -class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): +class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel): + _SUPPORTED_ACT_QUANT_KEYS = { + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8StaticTensorSym, + kFp8StaticTokenSym, + } + _SUPPORTED_WEIGHT_QUANT_KEYS = { + kFp8StaticChannelSym, + kFp8StaticTensorSym, + } + @classmethod def is_supported( cls, compute_capability: int | None = None ) -> tuple[bool, str | None]: if not current_platform.is_xpu(): - return False, "XPUFP8ScaledMM only support on XPU" + return False, "XPUW8A8FP8Linear only support on XPU" + return True, None + + @classmethod + def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]: + if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS: + return ( + False, + "XPUW8A8FP8Linear only support per-channel and per-tensor quantization", + ) + if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS: + return ( + False, + "XPUW8A8FP8Linear only support per-tensor and per-token activation " + "quantization", + ) + if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}: + return False, "XPUW8A8FP8Linear only support FP8 weight dtype" + if c.activation_quant_key.dtype not in { + torch.float8_e5m2, + torch.float8_e4m3fn, + }: + return False, "XPUW8A8FP8Linear only support FP8 activation dtype" + return True, None + + def __init__( + self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str] + ) -> None: + super().__init__(c, layer_param_names) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Ensure weight is stored as C-contiguous [K, N] (KN layout). + + Checkpoints store weight as [N, K]; fp8_gemm requires [K, N], + C-contiguous. Three incoming layouts are possible: + • [N, K] C-contiguous ← direct checkpoint → .t().contiguous() + • [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous() + • [K, N] C-contiguous ← already correct → no-op + + For square weights (K == N) the shape is ambiguous; contiguity is used + as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose); + Fortran-order ≡ fp8.py already transposed (needs only contiguous). + """ + K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1]) + N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0]) + w = layer.weight + + if w.shape not in {(K, N), (N, K)}: + raise ValueError( + f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), " + f"but got {tuple(w.shape)}" + ) + + needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous() + layer_weight = w.t() if needs_transpose else w + replace_parameter(layer, "weight", layer_weight) + ws = layer.weight_scale + if ws.numel() == 1: + replace_parameter(layer, "weight_scale", ws.reshape(1)) + + def apply_scaled_mm( + self, + *, + A: torch.Tensor, + B: torch.Tensor, + out_dtype: torch.dtype, + As: torch.Tensor, + Bs: torch.Tensor, + bias: torch.Tensor | None, + output_shape: list, + ) -> torch.Tensor: + # B is C-contiguous [K, N] from process_weights_after_loading. + # fp8_gemm routes on scale dtype (float32) and numel: + # As [1] → per-tensor (numel==1 branch) + # As [M,1] → per-token (group={1,K} branch, broadcast across K) + # Bs [1] → per-tensor + # Bs [N] → per-channel (mask=bit1 branch) + # No shape manipulation needed here. + output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias) + return output.view(*output_shape) + + +class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel): + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_xpu(): + return False, "XPUW8A16FP8Linear only support on XPU" return True, None @classmethod @@ -30,10 +132,11 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}: return ( False, - "XPUFP8ScaledMM only support per-channel and per-tensor quantization", + "XPUW8A16FP8Linear only support per-channel and per-tensor " + "quantization", ) if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}: - return False, "XPUFP8ScaledMM only support FP8 weight dtype" + return False, "XPUW8A16FP8Linear only support FP8 weight dtype" return True, None def __init__( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 2091a1cb6e..3789cd172f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -13,6 +13,7 @@ from compressed_tensors.quantization import ( ) from compressed_tensors.transform import TransformConfig +from vllm.config import get_current_vllm_config_or_none from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -747,9 +748,18 @@ class CompressedTensorsConfig(QuantizationConfig): act_quant_format = is_activation_quantization_format(format) if act_quant_format: if self._is_fp8_w8a8(weight_quant, input_quant): - is_fp8_w8a8_supported = self._check_scheme_supported( - CompressedTensorsW8A8Fp8.get_min_capability(), error=False - ) + if current_platform.is_xpu(): + # On XPU, --linear-backend xpu opts into W8A8 FP8 + # linear kernel; otherwise default to W8A16. + config = get_current_vllm_config_or_none() + is_fp8_w8a8_supported = ( + config is not None + and config.kernel_config.linear_backend == "xpu" + ) + else: + is_fp8_w8a8_supported = self._check_scheme_supported( + CompressedTensorsW8A8Fp8.get_min_capability(), error=False + ) if is_fp8_w8a8_supported: return CompressedTensorsW8A8Fp8( weight_quant=weight_quant,