[XPU][INC] Add int4 w4a8 (dynamic int8 activation) backend for INC linear layers (#50501)

Signed-off-by: Thanaji Rao Thakkalapelli <thanaji.rao.thakkalapelli@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Thanaji Rao Thakkalapelli
2026-08-21 14:38:59 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent f8e0602713
commit 9fd750f00d
4 changed files with 787 additions and 0 deletions
+618
View File
@@ -33,6 +33,9 @@ from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
INCLinearScheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_w4a8_linear import (
INCXPUW4A8LinearMethod,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
INCARKLinearMethod,
INCWNA16LinearScheme,
@@ -1482,3 +1485,618 @@ class TestGetLayerConfigFusedQKV:
DummyLayer(), "model.layers.0.self_attn.qkv_proj"
)
assert bits == 8
# ---------------------------------------------------------------------------
# INC XPU w4a8: int4 weights with dynamic per-token int8 activations
#
# The w4a8 path keeps int4 weights but dynamically quantizes activations to
# per-token symmetric int8, so the GEMM stays on the int8 datapath instead of
# upconverting weights to the activation dtype. It is opt-in through
# VLLM_XPU_INC_WNA16_BACKEND because which kernel is fastest depends on the
# device: ARK can use XMX int8 on some XPUs, and the default ("auto")
# preference order is deliberately left unchanged.
#
# The backend-selection tests stub out ARK and the platform; the weight-creation
# tests only need the TP-rank stubs. Both run anywhere. The apply_weights tests
# stub out int4_gemm_w4a8 — what needs guarding there is the calling convention
# (scale dtypes, argument order, weight reuse, batch-size gate), which is where
# this path is easy to get silently wrong, since the kernel accepts bf16 scales
# and returns plausible-looking garbage rather than raising. The ones that cross
# the token threshold reach ``vllm._xpu_ops`` for the activation quantizer, so
# they are XPU-image only; the small-batch fallback test returns before that
# import and must stay unmarked to keep covering the gate off-XPU. The
# end-to-end tests load a real int4 checkpoint and additionally need the op.
# ---------------------------------------------------------------------------
_BACKEND_ENV = "VLLM_XPU_INC_WNA16_BACKEND"
_ARK_STATE = (
"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state"
)
_QUANT_REF = "vllm._xpu_ops.xpu_ops.dynamic_per_token_int8_quant_ref"
IN_FEATURES = 128
OUT_FEATURES = 64
# auto-round int4/sym/g128 checkpoints, which is what this backend serves. The
# GPTQ- and AWQ-packed variants take different repack branches in
# ``process_weights_after_loading``, so both are worth loading.
E2E_MODELS = [
pytest.param("OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", id="auto_round:auto_gptq"),
pytest.param(
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", id="auto_round:auto_awq"
),
]
def _has_xpu_ops() -> bool:
"""True when ``vllm._xpu_ops`` is importable.
``apply_weights`` imports it for the activation quantizer, and the stubs
below patch through it, so tests that exercise that path cannot even be set
up on a build without vllm-xpu-kernels installed.
"""
if not current_platform.is_xpu():
return False
try:
import vllm._xpu_ops # noqa: F401
except ImportError:
return False
return True
def _has_w4a8_kernel() -> bool:
"""True when this build can actually run the w4a8 GEMM.
The op is registered as a side effect of importing ``vllm._xpu_ops``, so it
is not visible on ``torch.ops._xpu_C`` until that import happens.
"""
return _has_xpu_ops() and hasattr(torch.ops._xpu_C, "int4_gemm_w4a8")
requires_xpu_ops = pytest.mark.skipif(
not _has_xpu_ops(),
reason="needs an XPU image with vllm._xpu_ops available",
)
requires_w4a8_kernel = pytest.mark.skipif(
not _has_w4a8_kernel(),
reason="needs an XPU with the int4_gemm_w4a8 op",
)
def _fake_xpu(monkeypatch, ark_available: bool = True) -> None:
class DummyQuantLinear:
pass
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
monkeypatch.setattr(
_ARK_STATE,
lambda: (
(True, None, object(), DummyQuantLinear)
if ark_available
else (False, "missing", None, None)
),
)
def _with_w4a8_kernel(monkeypatch) -> None:
"""Present the op so the w4a8 support check passes off-XPU."""
class FakeOps:
int4_gemm_w4a8 = object()
monkeypatch.setattr(torch.ops, "_xpu_C", FakeOps, raising=False)
def _dispatch(layer_config=None):
return INCWna16Scheme().get_linear_method(
object(), object(), "layer", layer_config or make_layer_config()
)
@pytest.fixture
def single_tp_rank(monkeypatch):
"""Parameter creation reads the TP rank/size; there is no distributed init."""
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
lambda: 1,
)
def _create_weights(
method,
input_size_per_partition: int = IN_FEATURES,
output_partition_sizes: list[int] | None = None,
) -> torch.nn.Module:
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
output_partition_sizes = (
[OUT_FEATURES] if output_partition_sizes is None else output_partition_sizes
)
method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size_per_partition,
output_size=sum(output_partition_sizes),
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
return layer
@pytest.fixture
def w4a8_layer(single_tp_rank):
"""A w4a8 method plus a layer whose weights are created but not processed."""
method = INCXPUW4A8LinearMethod(make_layer_config())
return method, _create_weights(method)
def _stub_quant(monkeypatch, captured=None):
def fake_quant(x, use_sym, bits):
if captured is not None:
captured["quant_args"] = (tuple(x.shape), use_sym, bits)
m, k = x.shape
return (
torch.zeros(m, k, dtype=torch.int8),
torch.ones(m, 1, dtype=x.dtype),
torch.zeros(m, 1, dtype=torch.int32),
)
monkeypatch.setattr(_QUANT_REF, fake_quant, raising=False)
def _stub_gemm(monkeypatch, captured=None):
def fake_gemm(*args):
if captured is not None:
captured["gemm_args"] = args
# The real kernel always emits float16, whatever the model dtype.
return torch.zeros(args[0].shape[0], OUT_FEATURES, dtype=torch.float16)
monkeypatch.setattr(torch.ops._xpu_C, "int4_gemm_w4a8", fake_gemm, raising=False)
# ---------------------------------------------------------------------------
# Backend selection
# ---------------------------------------------------------------------------
def test_default_backend_keeps_ark_preference(monkeypatch) -> None:
"""The "auto" default must not change upstream behaviour.
ARK still wins whenever it is importable.
"""
monkeypatch.delenv(_BACKEND_ENV, raising=False)
_fake_xpu(monkeypatch, ark_available=True)
_with_w4a8_kernel(monkeypatch)
method = _dispatch()
assert isinstance(method, INCLinearMethod)
assert isinstance(method.scheme, INCARKLinearMethod)
def test_w4a8_backend_overrides_available_ark(monkeypatch) -> None:
"""An explicit request wins even though ARK would otherwise be chosen."""
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
_fake_xpu(monkeypatch, ark_available=True)
_with_w4a8_kernel(monkeypatch)
method = _dispatch()
assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
assert not isinstance(method.scheme, INCARKLinearMethod)
# w4a8 subclasses w4a16 so it can fall back for small batches.
assert isinstance(method.scheme, INCXPULinearMethod)
def test_w4a16_backend_overrides_available_ark(monkeypatch) -> None:
monkeypatch.setenv(_BACKEND_ENV, "w4a16")
_fake_xpu(monkeypatch, ark_available=True)
method = _dispatch()
assert isinstance(method.scheme, INCXPULinearMethod)
assert not isinstance(method.scheme, INCXPUW4A8LinearMethod)
assert not isinstance(method.scheme, INCARKLinearMethod)
def test_ark_backend_requested_but_unavailable_raises(monkeypatch) -> None:
"""Explicitly asking for ARK must fail loudly, not fall back silently."""
monkeypatch.setenv(_BACKEND_ENV, "ark")
_fake_xpu(monkeypatch, ark_available=False)
with pytest.raises(NotImplementedError, match="auto_round_kernel is unavailable"):
_dispatch()
@pytest.mark.parametrize("backend", ["w4a16", "w4a8"])
def test_onednn_backends_reject_int2(monkeypatch, backend) -> None:
"""The oneDNN int4 GEMMs cannot serve int2; ARK is the only int2 path."""
monkeypatch.setenv(_BACKEND_ENV, backend)
_fake_xpu(monkeypatch, ark_available=True)
_with_w4a8_kernel(monkeypatch)
with pytest.raises(NotImplementedError, match="only supports int4"):
_dispatch(make_layer_config(bits=2))
@pytest.mark.parametrize("group_size", [48, -1, 0])
def test_w4a8_rejects_unaligned_group_size(monkeypatch, group_size) -> None:
"""The kernel needs a group size that is a positive multiple of 32."""
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
_fake_xpu(monkeypatch, ark_available=True)
_with_w4a8_kernel(monkeypatch)
with pytest.raises(NotImplementedError, match="multiple of 32"):
_dispatch(make_layer_config(group_size=group_size))
@pytest.mark.parametrize("group_size", [128, 64, 32])
def test_w4a8_accepts_aligned_group_size(monkeypatch, group_size) -> None:
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
_fake_xpu(monkeypatch, ark_available=True)
_with_w4a8_kernel(monkeypatch)
method = _dispatch(make_layer_config(group_size=group_size))
assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
def test_w4a8_requires_the_kernel(monkeypatch) -> None:
"""Older vllm-xpu-kernels builds lack the op entirely."""
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
_fake_xpu(monkeypatch, ark_available=True)
class FakeOpsNoW4A8:
pass
monkeypatch.setattr(torch.ops, "_xpu_C", FakeOpsNoW4A8, raising=False)
with pytest.raises(NotImplementedError, match="int4_gemm_w4a8"):
_dispatch()
def test_invalid_backend_value_raises(monkeypatch) -> None:
import vllm.envs as envs
monkeypatch.setenv(_BACKEND_ENV, "onednn")
with pytest.raises(ValueError, match=_BACKEND_ENV):
_ = envs.VLLM_XPU_INC_WNA16_BACKEND
# ---------------------------------------------------------------------------
# Partition shape
# ---------------------------------------------------------------------------
#
# ``XPUW4A8IntLinearKernel.can_implement`` rejects partition shapes whose in/out
# dims are not multiples of 8, but that check is unreachable from the INC path:
# the shapes only exist once ``create_weights`` is called, well after backend
# selection. Unaligned dims must not reach the kernel — the int4 packing divides
# both dims by 8, so an unaligned size is silently truncated into a weight tensor
# that is too small rather than rejected.
def test_rejects_unaligned_input_partition(single_tp_rank) -> None:
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
with pytest.raises(NotImplementedError, match=r"multiples of 8.*input=132"):
_create_weights(method, input_size_per_partition=132)
def test_rejects_unaligned_output_partition(single_tp_rank) -> None:
method = INCXPUW4A8LinearMethod(make_layer_config())
with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=60"):
_create_weights(method, output_partition_sizes=[60])
def test_rejects_unaligned_sum_of_output_partitions(single_tp_rank) -> None:
"""A fused QKV/MLP layer is only as aligned as the sum of its shards."""
method = INCXPUW4A8LinearMethod(make_layer_config())
with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=76"):
_create_weights(method, output_partition_sizes=[OUT_FEATURES, 12])
def test_reports_both_unaligned_dims(single_tp_rank) -> None:
"""The message names every offending dim, not just the first."""
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
with pytest.raises(NotImplementedError) as excinfo:
_create_weights(
method, input_size_per_partition=132, output_partition_sizes=[60]
)
assert "input=132" in str(excinfo.value)
assert "output=60" in str(excinfo.value)
def test_accepts_aligned_partition_shape(single_tp_rank) -> None:
"""Multiples of 8 that are not powers of two are still fine."""
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
layer = _create_weights(
method, input_size_per_partition=96, output_partition_sizes=[24, 48]
)
assert layer.qweight.shape == (96 // 8, 72)
def test_w4a16_accepts_unaligned_partition_shape(single_tp_rank) -> None:
"""The constraint is a w4a8 kernel requirement; w4a16 must stay unaffected."""
method = INCXPULinearMethod(make_layer_config(group_size=32))
layer = _create_weights(method, output_partition_sizes=[60])
assert layer.scales.shape[1] == 60
# ---------------------------------------------------------------------------
# Weight processing
# ---------------------------------------------------------------------------
def test_keeps_fp16_scales(w4a8_layer) -> None:
"""The kernel reads scales as fp16 regardless of activation dtype.
Passing bf16 scales does not raise — it silently returns wrong results — so
an fp16 copy must exist, while the bf16 ``scales`` survive for the
small-batch w4a16 fallback.
"""
method, layer = w4a8_layer
layer.scales.data.fill_(0.125) # exactly representable in both dtypes
method.process_weights_after_loading(layer)
assert layer.scales_fp16.dtype is torch.float16
assert layer.scales_fp16.shape == layer.scales.shape
assert torch.equal(layer.scales_fp16.data.float(), layer.scales.data.float())
# The w4a16 fallback still needs the original activation-dtype scales.
assert layer.scales.dtype is torch.bfloat16
def test_reuses_w4a16_weight_layout(w4a8_layer) -> None:
"""w4a8 and w4a16 accept bit-identical weights, so there is no repacking."""
method, layer = w4a8_layer
qweight_before = layer.qweight.data.clone()
method.process_weights_after_loading(layer)
assert layer.qweight.dtype is torch.int32
assert layer.qweight.shape == (IN_FEATURES // 8, OUT_FEATURES)
assert torch.equal(layer.qweight.data, qweight_before)
# Symmetric int4: the zero point is the constant 8, as for w4a16.
assert layer.qzeros.dtype is torch.int8
assert layer.qzeros.tolist() == [8]
# ---------------------------------------------------------------------------
# apply_weights
# ---------------------------------------------------------------------------
def test_falls_back_to_w4a16_for_small_batches(monkeypatch, w4a8_layer) -> None:
"""Below the token threshold, per-token quant overhead dominates."""
method, layer = w4a8_layer
method.process_weights_after_loading(layer)
calls = []
def record_fallback(self, layer, x, bias=None):
calls.append(tuple(x.shape))
return torch.zeros(x.shape[:-1] + (OUT_FEATURES,), dtype=x.dtype)
monkeypatch.setattr(INCXPULinearMethod, "apply_weights", record_fallback)
monkeypatch.setattr(
torch.ops._xpu_C,
"int4_gemm_w4a8",
lambda *args: pytest.fail("small batches must not call the w4a8 kernel"),
raising=False,
)
tokens = method._MIN_TOKENS_FOR_INT8 - 1
x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
out = method.apply_weights(layer, x)
assert calls == [(tokens, IN_FEATURES)]
assert out.shape == (tokens, OUT_FEATURES)
@requires_xpu_ops
def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
"""At/above the threshold, activations are int8-quantized per token."""
method, layer = w4a8_layer
method.process_weights_after_loading(layer)
monkeypatch.setattr(
INCXPULinearMethod,
"apply_weights",
lambda self, layer, x, bias=None: pytest.fail(
"large batches must not take the w4a16 fallback"
),
)
captured: dict[str, tuple] = {}
_stub_quant(monkeypatch, captured)
_stub_gemm(monkeypatch, captured)
tokens = method._MIN_TOKENS_FOR_INT8
x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
out = method.apply_weights(layer, x)
# Symmetric 8-bit per-token quantization over the flattened activations.
assert captured["quant_args"] == ((tokens, IN_FEATURES), True, 8)
(
quant_x,
x_scale,
x_zero,
qweight,
w_scale,
w_zp,
group_size,
g_idx,
bias,
) = captured["gemm_args"]
assert quant_x.dtype is torch.int8
# Both scales must reach the kernel as fp16, whatever the model dtype.
assert x_scale.dtype is torch.float16
assert w_scale.dtype is torch.float16
assert w_scale is layer.scales_fp16
assert x_zero.dtype is torch.int32
assert qweight is layer.qweight
assert w_zp is layer.qzeros
assert group_size == 128
assert g_idx is None
assert bias is None
# The kernel emits fp16; the result is cast back to the activation dtype.
assert out.dtype is torch.bfloat16
assert out.shape == (tokens, OUT_FEATURES)
@requires_xpu_ops
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
def test_scales_are_fp16_for_any_activation_dtype(
monkeypatch, w4a8_layer, act_dtype
) -> None:
"""fp16 scales are required for bf16 *and* fp16 activations alike."""
method, layer = w4a8_layer
layer.scales.data = layer.scales.data.to(act_dtype)
method.process_weights_after_loading(layer)
captured: dict[str, tuple] = {}
_stub_quant(monkeypatch, captured)
_stub_gemm(monkeypatch, captured)
x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=act_dtype)
out = method.apply_weights(layer, x)
assert captured["gemm_args"][1].dtype is torch.float16
assert captured["gemm_args"][4].dtype is torch.float16
assert out.dtype is act_dtype
@requires_xpu_ops
def test_forwards_bias_to_kernel(monkeypatch, w4a8_layer) -> None:
"""Bias is applied by the kernel, not added afterwards."""
method, layer = w4a8_layer
method.process_weights_after_loading(layer)
captured: dict[str, tuple] = {}
_stub_quant(monkeypatch)
_stub_gemm(monkeypatch, captured)
bias = torch.zeros(OUT_FEATURES, dtype=torch.bfloat16)
x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=torch.bfloat16)
method.apply_weights(layer, x, bias)
assert captured["gemm_args"][8] is bias
@requires_xpu_ops
def test_preserves_leading_dims(monkeypatch, w4a8_layer) -> None:
"""3D activations must round-trip through the flatten/reshape."""
method, layer = w4a8_layer
method.process_weights_after_loading(layer)
_stub_quant(monkeypatch)
_stub_gemm(monkeypatch)
tokens = method._MIN_TOKENS_FOR_INT8
x = torch.zeros(4, tokens, IN_FEATURES, dtype=torch.bfloat16)
out = method.apply_weights(layer, x)
assert out.shape == (4, tokens, OUT_FEATURES)
assert out.dtype is torch.bfloat16
@requires_xpu_ops
def test_batch_gate_uses_flattened_token_count(monkeypatch, w4a8_layer) -> None:
"""The gate counts total tokens, not the leading dimension."""
method, layer = w4a8_layer
method.process_weights_after_loading(layer)
monkeypatch.setattr(
INCXPULinearMethod,
"apply_weights",
lambda self, layer, x, bias=None: pytest.fail(
"flattened token count is at the threshold; must use w4a8"
),
)
_stub_quant(monkeypatch)
_stub_gemm(monkeypatch)
tokens = method._MIN_TOKENS_FOR_INT8
# Each leading slice is below the threshold, but 4 * (tokens // 4) is not.
x = torch.zeros(4, tokens // 4, IN_FEATURES, dtype=torch.bfloat16)
out = method.apply_weights(layer, x)
assert out.shape == (4, tokens // 4, OUT_FEATURES)
# ---------------------------------------------------------------------------
# End to end on real weights
# ---------------------------------------------------------------------------
@requires_w4a8_kernel
@pytest.mark.parametrize("model", E2E_MODELS)
def test_e2e_generates_with_real_weights(vllm_runner, monkeypatch, model) -> None:
"""The whole path — real int4 checkpoint, real kernel, real output.
The stubbed tests above cannot catch a wrong-but-plausible calling
convention, because the fake kernel accepts anything. Greedy decoding of a
factual prompt does: wrongly set scales or a bad weight layout turn the
answer into noise rather than raising.
"""
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
with vllm_runner(
model, max_model_len=512, enforce_eager=True, gpu_memory_utilization=0.55
) as vllm_model:
out = vllm_model.generate_greedy(["The capital of France is"], max_tokens=8)
assert "Paris" in out[0][1]
@requires_w4a8_kernel
def test_e2e_selects_w4a8_for_every_linear(vllm_runner, monkeypatch) -> None:
"""Guard against the validation silently becoming dead code.
If backend selection stopped routing here, the generation test above would
still pass on the w4a16 path, so assert the method is actually in use and
that every partition shape it accepted honours the alignment rule.
"""
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
# apply_model pickles the closure below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
E2E_MODELS[0].values[0],
max_model_len=512,
enforce_eager=True,
gpu_memory_utilization=0.55,
) as vllm_model:
def check_model(model):
shapes = []
for module in model.modules():
scheme = getattr(getattr(module, "quant_method", None), "scheme", None)
if isinstance(scheme, INCXPUW4A8LinearMethod):
# qweight is the packed [in // 8, out] layout.
packed_in, out = module.qweight.shape
shapes.append((packed_in * 8, out))
assert shapes, "no linear layer used the w4a8 method"
assert all(i % 8 == 0 and o % 8 == 0 for i, o in shapes), shapes
vllm_model.apply_model(check_model)
+11
View File
@@ -313,6 +313,7 @@ if TYPE_CHECKING:
VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32
VLLM_XPU_ENABLE_XPU_GRAPH: bool = False
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
@@ -2111,6 +2112,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_XPU_USE_SAMPLER_KERNEL": lambda: bool(
int(os.getenv("VLLM_XPU_USE_SAMPLER_KERNEL", "1"))
),
# Kernel backend for INC weight-only intN (WNA16) linear layers on XPU.
# "auto" keeps the default preference order (ARK when importable, else the
# oneDNN w4a16 path). "ark" forces the auto_round_kernel backend, "w4a16"
# forces oneDNN int4_gemm_w4a16, and "w4a8" additionally quantizes
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
# The two oneDNN backends are int4-only; ARK also serves int2.
# Which one is fastest is device-dependent, so this is left as an opt-in.
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
),
# Enable simple KV offload.
"VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool(
int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0"))
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from .inc_wna16_linear import INCXPULinearMethod
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
"""XPU linear method for INC int4 weights with dynamic int8 activations.
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
no extra repacking is needed. Activations are dynamically quantized
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
instead of upconverting the weights to the activation dtype. That is a win
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
is dominated by dequantization rather than by weight bandwidth.
Both scale tensors must be float16: the kernel reads them as fp16
regardless of the activation dtype, so bf16 scales are silently
misinterpreted. The activations themselves may be bf16 or fp16.
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
device-dependent, so it is not the default: it was measured faster on B70
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
"""
# int8 activations only pay off once the GEMM is compute-bound. Below this
# token count the per-token quantization overhead dominates and w4a16 is
# faster, so fall back to it per-call.
_MIN_TOKENS_FOR_INT8 = 512
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
# parallelism shards these, so an aligned model dimension can still yield an
# unaligned partition.
_DIM_ALIGNMENT = 8
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
unaligned = [
(name, size)
for name, size in (
("input", input_size_per_partition),
("output", output_size_per_partition),
)
if size % self._DIM_ALIGNMENT != 0
]
if unaligned:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
+ ", ".join(f"{name}={size}" for name, size in unaligned)
+ f". Partition shape: ({input_size_per_partition}, "
f"{output_size_per_partition})."
)
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
layer.scales_fp16 = Parameter(
layer.scales.data.to(torch.float16), requires_grad=False
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
return super().apply_weights(layer, x, bias)
from vllm._xpu_ops import xpu_ops as ops
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale.to(torch.float16),
x_zero,
layer.qweight,
layer.scales_fp16,
layer.qzeros,
self.group_size,
None,
bias,
)
return out.to(x.dtype).reshape(out_shape)
@@ -3,6 +3,7 @@
from typing import TYPE_CHECKING
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
@@ -21,6 +22,32 @@ logger = init_logger(__name__)
XPU_WNA16_SUPPORTED_BITS = {2, 4}
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
The backend is requested explicitly, so an unusable configuration is an
error rather than something to silently fall back from.
"""
import torch
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
f"positive multiple of 32, got {layer_config.group_size}. "
f"Layer: {prefix}."
)
class INCWna16Scheme(INCScheme):
@staticmethod
@@ -38,12 +65,31 @@ class INCWna16Scheme(INCScheme):
if current_platform.is_xpu():
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
from .inc_wna16_linear import (
INCARKLinearMethod,
INCXPULinearMethod,
)
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
if backend in XPU_ONEDNN_BACKENDS:
if layer_config.bits != 4:
raise NotImplementedError(
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
f"int4, got int{layer_config.bits}. Layer: {prefix}."
)
if backend == "w4a8":
_check_xpu_w4a8_supported(layer_config, prefix)
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
return INCLinearMethod(INCXPULinearMethod(layer_config))
is_ark_available, ark_error, _, _ = get_ark_state()
if backend == "ark" and not is_ark_available:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
f"auto_round_kernel is unavailable: "
f"{ark_error or 'unknown error'}. Layer: {prefix}."
)
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
elif layer_config.bits == 2: