[XPU] Add W8A8 FP8 linear kernel with multi-granularity quant support (#43645)

Signed-off-by: Chaojun,Zhang <chaojun.zhang@intel.com>
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Chaojun Zhang
2026-07-04 18:10:01 +08:00
committed by GitHub
co-authored by Copilot
parent e7c9df9449
commit 1a308c449c
6 changed files with 188 additions and 12 deletions
+48 -1
View File
@@ -49,6 +49,30 @@ steps:
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
'
- label: "XPU W8A8 FP8 Linear Examples"
depends_on:
- image-build-xpu
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
'
- label: "XPU V1 test"
depends_on:
- image-build-xpu
@@ -120,4 +144,27 @@ steps:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_auto_round.py'
pytest -v -s quantization/test_auto_round.py'
- label: "XPU compressed tensors FP8 test"
depends_on:
- image-build-xpu
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/quantization/test_compressed_tensors.py
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
+2
View File
@@ -62,6 +62,8 @@ weight name. Unset fields fall back to the `--quantization` shorthand's
defaults, or for already-quantized checkpoints to whatever the checkpoint
declares.
On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16).
The CLI accepts the same shape as JSON or as dotted keys:
```bash
+6 -1
View File
@@ -156,6 +156,8 @@ LinearBackend = Literal[
"conch",
"exllama",
"emulation",
"xpu",
"xpu_woq",
]
@@ -217,7 +219,10 @@ class KernelConfig:
- "fbgemm": Use FBGEMM kernels
- "conch": Use Conch mixed-precision kernels
- "exllama": Use Exllama mixed-precision kernels
- "emulation": Use slow dequant-to-BF16 emulation (for testing only)"""
- "emulation": Use slow dequant-to-BF16 emulation (for testing only)
- "xpu": Use XPU kernels
- "xpu_woq": Use XPU kernels for weight-only quantization (e.g. W8A16)
"""
@field_validator("moe_backend", mode="before")
@classmethod
+12 -3
View File
@@ -171,7 +171,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
XPUFP8ScaledMMLinearKernel,
XPUW8A8FP8LinearKernel,
XPUW8A16FP8LinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
@@ -266,6 +267,13 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
EmulationMxfp8LinearKernel,
EmulationNvFp4LinearKernel,
},
"xpu": {
XPUW8A8FP8LinearKernel,
XPUFp8BlockScaledMMKernel,
},
"xpu_woq": {
XPUW8A16FP8LinearKernel,
},
}
@@ -311,7 +319,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.XPU: [
XPUFP8ScaledMMLinearKernel,
XPUW8A16FP8LinearKernel,
XPUW8A8FP8LinearKernel,
],
}
@@ -351,7 +360,7 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
# To be added
],
PlatformEnum.XPU: [
XPUFP8ScaledMMLinearKernel,
XPUW8A16FP8LinearKernel,
],
}
@@ -6,8 +6,11 @@ from collections.abc import Sequence
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
@@ -16,13 +19,112 @@ from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
_SUPPORTED_ACT_QUANT_KEYS = {
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
}
_SUPPORTED_WEIGHT_QUANT_KEYS = {
kFp8StaticChannelSym,
kFp8StaticTensorSym,
}
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUFP8ScaledMM only support on XPU"
return False, "XPUW8A8FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
)
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
if c.activation_quant_key.dtype not in {
torch.float8_e5m2,
torch.float8_e4m3fn,
}:
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
super().__init__(c, layer_param_names)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
C-contiguous. Three incoming layouts are possible:
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
• [K, N] C-contiguous ← already correct → no-op
For square weights (K == N) the shape is ambiguous; contiguity is used
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
"""
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
w = layer.weight
if w.shape not in {(K, N), (N, K)}:
raise ValueError(
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
f"but got {tuple(w.shape)}"
)
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
layer_weight = w.t() if needs_transpose else w
replace_parameter(layer, "weight", layer_weight)
ws = layer.weight_scale
if ws.numel() == 1:
replace_parameter(layer, "weight_scale", ws.reshape(1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
# B is C-contiguous [K, N] from process_weights_after_loading.
# fp8_gemm routes on scale dtype (float32) and numel:
# As [1] → per-tensor (numel==1 branch)
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
# Bs [1] → per-tensor
# Bs [N] → per-channel (mask=bit1 branch)
# No shape manipulation needed here.
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
return output.view(*output_shape)
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A16FP8Linear only support on XPU"
return True, None
@classmethod
@@ -30,10 +132,11 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
return (
False,
"XPUFP8ScaledMM only support per-channel and per-tensor quantization",
"XPUW8A16FP8Linear only support per-channel and per-tensor "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUFP8ScaledMM only support FP8 weight dtype"
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
return True, None
def __init__(
@@ -13,6 +13,7 @@ from compressed_tensors.quantization import (
)
from compressed_tensors.transform import TransformConfig
from vllm.config import get_current_vllm_config_or_none
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -747,9 +748,18 @@ class CompressedTensorsConfig(QuantizationConfig):
act_quant_format = is_activation_quantization_format(format)
if act_quant_format:
if self._is_fp8_w8a8(weight_quant, input_quant):
is_fp8_w8a8_supported = self._check_scheme_supported(
CompressedTensorsW8A8Fp8.get_min_capability(), error=False
)
if current_platform.is_xpu():
# On XPU, --linear-backend xpu opts into W8A8 FP8
# linear kernel; otherwise default to W8A16.
config = get_current_vllm_config_or_none()
is_fp8_w8a8_supported = (
config is not None
and config.kernel_config.linear_backend == "xpu"
)
else:
is_fp8_w8a8_supported = self._check_scheme_supported(
CompressedTensorsW8A8Fp8.get_min_capability(), error=False
)
if is_fp8_w8a8_supported:
return CompressedTensorsW8A8Fp8(
weight_quant=weight_quant,