[XPU][INC] Add int4 w4a8 (dynamic int8 activation) backend for INC linear layers (#50501)

Signed-off-by: Thanaji Rao Thakkalapelli <thanaji.rao.thakkalapelli@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Thanaji Rao Thakkalapelli
2026-08-21 14:38:59 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent f8e0602713
commit 9fd750f00d
4 changed files with 787 additions and 0 deletions
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from .inc_wna16_linear import INCXPULinearMethod
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
"""XPU linear method for INC int4 weights with dynamic int8 activations.
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
no extra repacking is needed. Activations are dynamically quantized
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
instead of upconverting the weights to the activation dtype. That is a win
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
is dominated by dequantization rather than by weight bandwidth.
Both scale tensors must be float16: the kernel reads them as fp16
regardless of the activation dtype, so bf16 scales are silently
misinterpreted. The activations themselves may be bf16 or fp16.
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
device-dependent, so it is not the default: it was measured faster on B70
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
"""
# int8 activations only pay off once the GEMM is compute-bound. Below this
# token count the per-token quantization overhead dominates and w4a16 is
# faster, so fall back to it per-call.
_MIN_TOKENS_FOR_INT8 = 512
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
# parallelism shards these, so an aligned model dimension can still yield an
# unaligned partition.
_DIM_ALIGNMENT = 8
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
unaligned = [
(name, size)
for name, size in (
("input", input_size_per_partition),
("output", output_size_per_partition),
)
if size % self._DIM_ALIGNMENT != 0
]
if unaligned:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
+ ", ".join(f"{name}={size}" for name, size in unaligned)
+ f". Partition shape: ({input_size_per_partition}, "
f"{output_size_per_partition})."
)
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
layer.scales_fp16 = Parameter(
layer.scales.data.to(torch.float16), requires_grad=False
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
return super().apply_weights(layer, x, bias)
from vllm._xpu_ops import xpu_ops as ops
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale.to(torch.float16),
x_zero,
layer.qweight,
layer.scales_fp16,
layer.qzeros,
self.group_size,
None,
bias,
)
return out.to(x.dtype).reshape(out_shape)
@@ -3,6 +3,7 @@
from typing import TYPE_CHECKING
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
@@ -21,6 +22,32 @@ logger = init_logger(__name__)
XPU_WNA16_SUPPORTED_BITS = {2, 4}
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
The backend is requested explicitly, so an unusable configuration is an
error rather than something to silently fall back from.
"""
import torch
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
f"positive multiple of 32, got {layer_config.group_size}. "
f"Layer: {prefix}."
)
class INCWna16Scheme(INCScheme):
@staticmethod
@@ -38,12 +65,31 @@ class INCWna16Scheme(INCScheme):
if current_platform.is_xpu():
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
from .inc_wna16_linear import (
INCARKLinearMethod,
INCXPULinearMethod,
)
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
if backend in XPU_ONEDNN_BACKENDS:
if layer_config.bits != 4:
raise NotImplementedError(
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
f"int4, got int{layer_config.bits}. Layer: {prefix}."
)
if backend == "w4a8":
_check_xpu_w4a8_supported(layer_config, prefix)
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
return INCLinearMethod(INCXPULinearMethod(layer_config))
is_ark_available, ark_error, _, _ = get_ark_state()
if backend == "ark" and not is_ark_available:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
f"auto_round_kernel is unavailable: "
f"{ark_error or 'unknown error'}. Layer: {prefix}."
)
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
elif layer_config.bits == 2: