diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index a7e6fa87f5..c278b10bc4 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -70,8 +70,10 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 ' - label: "XPU V1 test" depends_on: diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index a4da52557a..67e28a270d 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -62,7 +62,7 @@ weight name. Unset fields fall back to the `--quantization` shorthand's defaults, or for already-quantized checkpoints to whatever the checkpoint declares. -On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). +On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). Setting `--linear-backend torch` also forces W8A8 but runs the GEMM through `torch._scaled_mm` instead of the custom XPU kernel. The CLI accepts the same shape as JSON or as dotted keys: diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index aaf46875e8..3d8449a219 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -363,6 +363,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = PlatformEnum.XPU: [ XPUW8A16FP8LinearKernel, XPUW8A8FP8LinearKernel, + PerTensorTorchFP8ScaledMMLinearKernel, + ChannelWiseTorchFP8ScaledMMLinearKernel, ], } diff --git a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py index a04be2e097..943dd2bb03 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py @@ -32,8 +32,12 @@ class TorchFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): def is_supported( cls, compute_capability: int | None = None ) -> tuple[bool, str | None]: - if not (current_platform.is_cuda_alike() or current_platform.is_cpu()): - return False, "requires ROCm, CUDA or CPU." + if not ( + current_platform.is_cuda_alike() + or current_platform.is_cpu() + or current_platform.is_xpu() + ): + return False, "requires ROCm, CUDA, CPU or XPU." if compute_capability is not None and compute_capability < 89: return False, "requires compute capability 89 and above." diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 6fea370824..473cad8741 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -811,12 +811,11 @@ class CompressedTensorsConfig(QuantizationConfig): if act_quant_format: if self._is_fp8_w8a8(weight_quant, input_quant): if current_platform.is_xpu(): - # On XPU, --linear-backend xpu opts into W8A8 FP8 + # On XPU, --linear-backend xpu or torch opts into W8A8 FP8 # linear kernel; otherwise default to W8A16. config = get_current_vllm_config_or_none() - is_fp8_w8a8_supported = ( - config is not None - and config.kernel_config.linear_backend == "xpu" + is_fp8_w8a8_supported = config is not None and ( + config.kernel_config.linear_backend in ("xpu", "torch") ) else: is_fp8_w8a8_supported = self._check_scheme_supported(