[XPU][INC] Add int4 w4a8 (dynamic int8 activation) backend for INC linear layers (#50501)
Signed-off-by: Thanaji Rao Thakkalapelli <thanaji.rao.thakkalapelli@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
f8e0602713
commit
9fd750f00d
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from .inc_wna16_linear import INCXPULinearMethod
|
||||
|
||||
|
||||
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
|
||||
"""XPU linear method for INC int4 weights with dynamic int8 activations.
|
||||
|
||||
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
|
||||
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
|
||||
no extra repacking is needed. Activations are dynamically quantized
|
||||
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
|
||||
instead of upconverting the weights to the activation dtype. That is a win
|
||||
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
|
||||
is dominated by dequantization rather than by weight bandwidth.
|
||||
|
||||
Both scale tensors must be float16: the kernel reads them as fp16
|
||||
regardless of the activation dtype, so bf16 scales are silently
|
||||
misinterpreted. The activations themselves may be bf16 or fp16.
|
||||
|
||||
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
|
||||
device-dependent, so it is not the default: it was measured faster on B70
|
||||
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
|
||||
"""
|
||||
|
||||
# int8 activations only pay off once the GEMM is compute-bound. Below this
|
||||
# token count the per-token quantization overhead dominates and w4a16 is
|
||||
# faster, so fall back to it per-call.
|
||||
_MIN_TOKENS_FOR_INT8 = 512
|
||||
|
||||
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
|
||||
# parallelism shards these, so an aligned model dimension can still yield an
|
||||
# unaligned partition.
|
||||
_DIM_ALIGNMENT = 8
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
unaligned = [
|
||||
(name, size)
|
||||
for name, size in (
|
||||
("input", input_size_per_partition),
|
||||
("output", output_size_per_partition),
|
||||
)
|
||||
if size % self._DIM_ALIGNMENT != 0
|
||||
]
|
||||
if unaligned:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
|
||||
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
|
||||
+ ", ".join(f"{name}={size}" for name, size in unaligned)
|
||||
+ f". Partition shape: ({input_size_per_partition}, "
|
||||
f"{output_size_per_partition})."
|
||||
)
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
|
||||
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
|
||||
layer.scales_fp16 = Parameter(
|
||||
layer.scales.data.to(torch.float16), requires_grad=False
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
|
||||
return super().apply_weights(layer, x, bias)
|
||||
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale.to(torch.float16),
|
||||
x_zero,
|
||||
layer.qweight,
|
||||
layer.scales_fp16,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None,
|
||||
bias,
|
||||
)
|
||||
return out.to(x.dtype).reshape(out_shape)
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
@@ -21,6 +22,32 @@ logger = init_logger(__name__)
|
||||
|
||||
XPU_WNA16_SUPPORTED_BITS = {2, 4}
|
||||
|
||||
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
|
||||
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
|
||||
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
|
||||
|
||||
|
||||
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
|
||||
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
|
||||
|
||||
The backend is requested explicitly, so an unusable configuration is an
|
||||
error rather than something to silently fall back from.
|
||||
"""
|
||||
import torch
|
||||
|
||||
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
|
||||
"which this build of vllm-xpu-kernels does not provide. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
|
||||
f"positive multiple of 32, got {layer_config.group_size}. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
|
||||
|
||||
class INCWna16Scheme(INCScheme):
|
||||
@staticmethod
|
||||
@@ -38,12 +65,31 @@ class INCWna16Scheme(INCScheme):
|
||||
if current_platform.is_xpu():
|
||||
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
|
||||
from .inc_ark_ops import get_ark_state
|
||||
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCXPULinearMethod,
|
||||
)
|
||||
|
||||
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
|
||||
if backend in XPU_ONEDNN_BACKENDS:
|
||||
if layer_config.bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
|
||||
f"int4, got int{layer_config.bits}. Layer: {prefix}."
|
||||
)
|
||||
if backend == "w4a8":
|
||||
_check_xpu_w4a8_supported(layer_config, prefix)
|
||||
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
|
||||
return INCLinearMethod(INCXPULinearMethod(layer_config))
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if backend == "ark" and not is_ark_available:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
|
||||
f"auto_round_kernel is unavailable: "
|
||||
f"{ark_error or 'unknown error'}. Layer: {prefix}."
|
||||
)
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
elif layer_config.bits == 2:
|
||||
|
||||
Reference in New Issue
Block a user