[XPU] [Linear] add torch as xpu linear backend (#49664)
Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -70,8 +70,10 @@ steps:
|
|||||||
- >-
|
- >-
|
||||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||||
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
|
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
|
||||||
|
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
|
||||||
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
|
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
|
||||||
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
|
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 &&
|
||||||
|
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
|
||||||
'
|
'
|
||||||
- label: "XPU V1 test"
|
- label: "XPU V1 test"
|
||||||
depends_on:
|
depends_on:
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ weight name. Unset fields fall back to the `--quantization` shorthand's
|
|||||||
defaults, or for already-quantized checkpoints to whatever the checkpoint
|
defaults, or for already-quantized checkpoints to whatever the checkpoint
|
||||||
declares.
|
declares.
|
||||||
|
|
||||||
On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16).
|
On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). Setting `--linear-backend torch` also forces W8A8 but runs the GEMM through `torch._scaled_mm` instead of the custom XPU kernel.
|
||||||
|
|
||||||
The CLI accepts the same shape as JSON or as dotted keys:
|
The CLI accepts the same shape as JSON or as dotted keys:
|
||||||
|
|
||||||
|
|||||||
@@ -363,6 +363,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
|||||||
PlatformEnum.XPU: [
|
PlatformEnum.XPU: [
|
||||||
XPUW8A16FP8LinearKernel,
|
XPUW8A16FP8LinearKernel,
|
||||||
XPUW8A8FP8LinearKernel,
|
XPUW8A8FP8LinearKernel,
|
||||||
|
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||||
|
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,8 +32,12 @@ class TorchFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
|||||||
def is_supported(
|
def is_supported(
|
||||||
cls, compute_capability: int | None = None
|
cls, compute_capability: int | None = None
|
||||||
) -> tuple[bool, str | None]:
|
) -> tuple[bool, str | None]:
|
||||||
if not (current_platform.is_cuda_alike() or current_platform.is_cpu()):
|
if not (
|
||||||
return False, "requires ROCm, CUDA or CPU."
|
current_platform.is_cuda_alike()
|
||||||
|
or current_platform.is_cpu()
|
||||||
|
or current_platform.is_xpu()
|
||||||
|
):
|
||||||
|
return False, "requires ROCm, CUDA, CPU or XPU."
|
||||||
|
|
||||||
if compute_capability is not None and compute_capability < 89:
|
if compute_capability is not None and compute_capability < 89:
|
||||||
return False, "requires compute capability 89 and above."
|
return False, "requires compute capability 89 and above."
|
||||||
|
|||||||
@@ -811,12 +811,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
|||||||
if act_quant_format:
|
if act_quant_format:
|
||||||
if self._is_fp8_w8a8(weight_quant, input_quant):
|
if self._is_fp8_w8a8(weight_quant, input_quant):
|
||||||
if current_platform.is_xpu():
|
if current_platform.is_xpu():
|
||||||
# On XPU, --linear-backend xpu opts into W8A8 FP8
|
# On XPU, --linear-backend xpu or torch opts into W8A8 FP8
|
||||||
# linear kernel; otherwise default to W8A16.
|
# linear kernel; otherwise default to W8A16.
|
||||||
config = get_current_vllm_config_or_none()
|
config = get_current_vllm_config_or_none()
|
||||||
is_fp8_w8a8_supported = (
|
is_fp8_w8a8_supported = config is not None and (
|
||||||
config is not None
|
config.kernel_config.linear_backend in ("xpu", "torch")
|
||||||
and config.kernel_config.linear_backend == "xpu"
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
is_fp8_w8a8_supported = self._check_scheme_supported(
|
is_fp8_w8a8_supported = self._check_scheme_supported(
|
||||||
|
|||||||
Reference in New Issue
Block a user