[XPU][INC] Add int4 w4a8 (dynamic int8 activation) backend for INC linear layers (#50501)
Signed-off-by: Thanaji Rao Thakkalapelli <thanaji.rao.thakkalapelli@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
f8e0602713
commit
9fd750f00d
@@ -33,6 +33,9 @@ from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
|
||||
INCLinearScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_w4a8_linear import (
|
||||
INCXPUW4A8LinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCWNA16LinearScheme,
|
||||
@@ -1482,3 +1485,618 @@ class TestGetLayerConfigFusedQKV:
|
||||
DummyLayer(), "model.layers.0.self_attn.qkv_proj"
|
||||
)
|
||||
assert bits == 8
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# INC XPU w4a8: int4 weights with dynamic per-token int8 activations
|
||||
#
|
||||
# The w4a8 path keeps int4 weights but dynamically quantizes activations to
|
||||
# per-token symmetric int8, so the GEMM stays on the int8 datapath instead of
|
||||
# upconverting weights to the activation dtype. It is opt-in through
|
||||
# VLLM_XPU_INC_WNA16_BACKEND because which kernel is fastest depends on the
|
||||
# device: ARK can use XMX int8 on some XPUs, and the default ("auto")
|
||||
# preference order is deliberately left unchanged.
|
||||
#
|
||||
# The backend-selection tests stub out ARK and the platform; the weight-creation
|
||||
# tests only need the TP-rank stubs. Both run anywhere. The apply_weights tests
|
||||
# stub out int4_gemm_w4a8 — what needs guarding there is the calling convention
|
||||
# (scale dtypes, argument order, weight reuse, batch-size gate), which is where
|
||||
# this path is easy to get silently wrong, since the kernel accepts bf16 scales
|
||||
# and returns plausible-looking garbage rather than raising. The ones that cross
|
||||
# the token threshold reach ``vllm._xpu_ops`` for the activation quantizer, so
|
||||
# they are XPU-image only; the small-batch fallback test returns before that
|
||||
# import and must stay unmarked to keep covering the gate off-XPU. The
|
||||
# end-to-end tests load a real int4 checkpoint and additionally need the op.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BACKEND_ENV = "VLLM_XPU_INC_WNA16_BACKEND"
|
||||
_ARK_STATE = (
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state"
|
||||
)
|
||||
_QUANT_REF = "vllm._xpu_ops.xpu_ops.dynamic_per_token_int8_quant_ref"
|
||||
|
||||
IN_FEATURES = 128
|
||||
OUT_FEATURES = 64
|
||||
|
||||
# auto-round int4/sym/g128 checkpoints, which is what this backend serves. The
|
||||
# GPTQ- and AWQ-packed variants take different repack branches in
|
||||
# ``process_weights_after_loading``, so both are worth loading.
|
||||
E2E_MODELS = [
|
||||
pytest.param("OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", id="auto_round:auto_gptq"),
|
||||
pytest.param(
|
||||
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", id="auto_round:auto_awq"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _has_xpu_ops() -> bool:
|
||||
"""True when ``vllm._xpu_ops`` is importable.
|
||||
|
||||
``apply_weights`` imports it for the activation quantizer, and the stubs
|
||||
below patch through it, so tests that exercise that path cannot even be set
|
||||
up on a build without vllm-xpu-kernels installed.
|
||||
"""
|
||||
if not current_platform.is_xpu():
|
||||
return False
|
||||
try:
|
||||
import vllm._xpu_ops # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _has_w4a8_kernel() -> bool:
|
||||
"""True when this build can actually run the w4a8 GEMM.
|
||||
|
||||
The op is registered as a side effect of importing ``vllm._xpu_ops``, so it
|
||||
is not visible on ``torch.ops._xpu_C`` until that import happens.
|
||||
"""
|
||||
return _has_xpu_ops() and hasattr(torch.ops._xpu_C, "int4_gemm_w4a8")
|
||||
|
||||
|
||||
requires_xpu_ops = pytest.mark.skipif(
|
||||
not _has_xpu_ops(),
|
||||
reason="needs an XPU image with vllm._xpu_ops available",
|
||||
)
|
||||
|
||||
requires_w4a8_kernel = pytest.mark.skipif(
|
||||
not _has_w4a8_kernel(),
|
||||
reason="needs an XPU with the int4_gemm_w4a8 op",
|
||||
)
|
||||
|
||||
|
||||
def _fake_xpu(monkeypatch, ark_available: bool = True) -> None:
|
||||
class DummyQuantLinear:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
_ARK_STATE,
|
||||
lambda: (
|
||||
(True, None, object(), DummyQuantLinear)
|
||||
if ark_available
|
||||
else (False, "missing", None, None)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _with_w4a8_kernel(monkeypatch) -> None:
|
||||
"""Present the op so the w4a8 support check passes off-XPU."""
|
||||
|
||||
class FakeOps:
|
||||
int4_gemm_w4a8 = object()
|
||||
|
||||
monkeypatch.setattr(torch.ops, "_xpu_C", FakeOps, raising=False)
|
||||
|
||||
|
||||
def _dispatch(layer_config=None):
|
||||
return INCWna16Scheme().get_linear_method(
|
||||
object(), object(), "layer", layer_config or make_layer_config()
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def single_tp_rank(monkeypatch):
|
||||
"""Parameter creation reads the TP rank/size; there is no distributed init."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
|
||||
|
||||
def _create_weights(
|
||||
method,
|
||||
input_size_per_partition: int = IN_FEATURES,
|
||||
output_partition_sizes: list[int] | None = None,
|
||||
) -> torch.nn.Module:
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
output_partition_sizes = (
|
||||
[OUT_FEATURES] if output_partition_sizes is None else output_partition_sizes
|
||||
)
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size_per_partition,
|
||||
output_size=sum(output_partition_sizes),
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def w4a8_layer(single_tp_rank):
|
||||
"""A w4a8 method plus a layer whose weights are created but not processed."""
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config())
|
||||
return method, _create_weights(method)
|
||||
|
||||
|
||||
def _stub_quant(monkeypatch, captured=None):
|
||||
def fake_quant(x, use_sym, bits):
|
||||
if captured is not None:
|
||||
captured["quant_args"] = (tuple(x.shape), use_sym, bits)
|
||||
m, k = x.shape
|
||||
return (
|
||||
torch.zeros(m, k, dtype=torch.int8),
|
||||
torch.ones(m, 1, dtype=x.dtype),
|
||||
torch.zeros(m, 1, dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(_QUANT_REF, fake_quant, raising=False)
|
||||
|
||||
|
||||
def _stub_gemm(monkeypatch, captured=None):
|
||||
def fake_gemm(*args):
|
||||
if captured is not None:
|
||||
captured["gemm_args"] = args
|
||||
# The real kernel always emits float16, whatever the model dtype.
|
||||
return torch.zeros(args[0].shape[0], OUT_FEATURES, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(torch.ops._xpu_C, "int4_gemm_w4a8", fake_gemm, raising=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backend selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_default_backend_keeps_ark_preference(monkeypatch) -> None:
|
||||
"""The "auto" default must not change upstream behaviour.
|
||||
|
||||
ARK still wins whenever it is importable.
|
||||
"""
|
||||
monkeypatch.delenv(_BACKEND_ENV, raising=False)
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
_with_w4a8_kernel(monkeypatch)
|
||||
|
||||
method = _dispatch()
|
||||
|
||||
assert isinstance(method, INCLinearMethod)
|
||||
assert isinstance(method.scheme, INCARKLinearMethod)
|
||||
|
||||
|
||||
def test_w4a8_backend_overrides_available_ark(monkeypatch) -> None:
|
||||
"""An explicit request wins even though ARK would otherwise be chosen."""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
_with_w4a8_kernel(monkeypatch)
|
||||
|
||||
method = _dispatch()
|
||||
|
||||
assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
|
||||
assert not isinstance(method.scheme, INCARKLinearMethod)
|
||||
# w4a8 subclasses w4a16 so it can fall back for small batches.
|
||||
assert isinstance(method.scheme, INCXPULinearMethod)
|
||||
|
||||
|
||||
def test_w4a16_backend_overrides_available_ark(monkeypatch) -> None:
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a16")
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
|
||||
method = _dispatch()
|
||||
|
||||
assert isinstance(method.scheme, INCXPULinearMethod)
|
||||
assert not isinstance(method.scheme, INCXPUW4A8LinearMethod)
|
||||
assert not isinstance(method.scheme, INCARKLinearMethod)
|
||||
|
||||
|
||||
def test_ark_backend_requested_but_unavailable_raises(monkeypatch) -> None:
|
||||
"""Explicitly asking for ARK must fail loudly, not fall back silently."""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "ark")
|
||||
_fake_xpu(monkeypatch, ark_available=False)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="auto_round_kernel is unavailable"):
|
||||
_dispatch()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["w4a16", "w4a8"])
|
||||
def test_onednn_backends_reject_int2(monkeypatch, backend) -> None:
|
||||
"""The oneDNN int4 GEMMs cannot serve int2; ARK is the only int2 path."""
|
||||
monkeypatch.setenv(_BACKEND_ENV, backend)
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
_with_w4a8_kernel(monkeypatch)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="only supports int4"):
|
||||
_dispatch(make_layer_config(bits=2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_size", [48, -1, 0])
|
||||
def test_w4a8_rejects_unaligned_group_size(monkeypatch, group_size) -> None:
|
||||
"""The kernel needs a group size that is a positive multiple of 32."""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
_with_w4a8_kernel(monkeypatch)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="multiple of 32"):
|
||||
_dispatch(make_layer_config(group_size=group_size))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_size", [128, 64, 32])
|
||||
def test_w4a8_accepts_aligned_group_size(monkeypatch, group_size) -> None:
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
_with_w4a8_kernel(monkeypatch)
|
||||
|
||||
method = _dispatch(make_layer_config(group_size=group_size))
|
||||
|
||||
assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
|
||||
|
||||
|
||||
def test_w4a8_requires_the_kernel(monkeypatch) -> None:
|
||||
"""Older vllm-xpu-kernels builds lack the op entirely."""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
_fake_xpu(monkeypatch, ark_available=True)
|
||||
|
||||
class FakeOpsNoW4A8:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(torch.ops, "_xpu_C", FakeOpsNoW4A8, raising=False)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="int4_gemm_w4a8"):
|
||||
_dispatch()
|
||||
|
||||
|
||||
def test_invalid_backend_value_raises(monkeypatch) -> None:
|
||||
import vllm.envs as envs
|
||||
|
||||
monkeypatch.setenv(_BACKEND_ENV, "onednn")
|
||||
with pytest.raises(ValueError, match=_BACKEND_ENV):
|
||||
_ = envs.VLLM_XPU_INC_WNA16_BACKEND
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Partition shape
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# ``XPUW4A8IntLinearKernel.can_implement`` rejects partition shapes whose in/out
|
||||
# dims are not multiples of 8, but that check is unreachable from the INC path:
|
||||
# the shapes only exist once ``create_weights`` is called, well after backend
|
||||
# selection. Unaligned dims must not reach the kernel — the int4 packing divides
|
||||
# both dims by 8, so an unaligned size is silently truncated into a weight tensor
|
||||
# that is too small rather than rejected.
|
||||
|
||||
|
||||
def test_rejects_unaligned_input_partition(single_tp_rank) -> None:
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
|
||||
|
||||
with pytest.raises(NotImplementedError, match=r"multiples of 8.*input=132"):
|
||||
_create_weights(method, input_size_per_partition=132)
|
||||
|
||||
|
||||
def test_rejects_unaligned_output_partition(single_tp_rank) -> None:
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config())
|
||||
|
||||
with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=60"):
|
||||
_create_weights(method, output_partition_sizes=[60])
|
||||
|
||||
|
||||
def test_rejects_unaligned_sum_of_output_partitions(single_tp_rank) -> None:
|
||||
"""A fused QKV/MLP layer is only as aligned as the sum of its shards."""
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config())
|
||||
|
||||
with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=76"):
|
||||
_create_weights(method, output_partition_sizes=[OUT_FEATURES, 12])
|
||||
|
||||
|
||||
def test_reports_both_unaligned_dims(single_tp_rank) -> None:
|
||||
"""The message names every offending dim, not just the first."""
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
|
||||
|
||||
with pytest.raises(NotImplementedError) as excinfo:
|
||||
_create_weights(
|
||||
method, input_size_per_partition=132, output_partition_sizes=[60]
|
||||
)
|
||||
|
||||
assert "input=132" in str(excinfo.value)
|
||||
assert "output=60" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_accepts_aligned_partition_shape(single_tp_rank) -> None:
|
||||
"""Multiples of 8 that are not powers of two are still fine."""
|
||||
method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
|
||||
|
||||
layer = _create_weights(
|
||||
method, input_size_per_partition=96, output_partition_sizes=[24, 48]
|
||||
)
|
||||
|
||||
assert layer.qweight.shape == (96 // 8, 72)
|
||||
|
||||
|
||||
def test_w4a16_accepts_unaligned_partition_shape(single_tp_rank) -> None:
|
||||
"""The constraint is a w4a8 kernel requirement; w4a16 must stay unaffected."""
|
||||
method = INCXPULinearMethod(make_layer_config(group_size=32))
|
||||
|
||||
layer = _create_weights(method, output_partition_sizes=[60])
|
||||
|
||||
assert layer.scales.shape[1] == 60
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weight processing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_keeps_fp16_scales(w4a8_layer) -> None:
|
||||
"""The kernel reads scales as fp16 regardless of activation dtype.
|
||||
|
||||
Passing bf16 scales does not raise — it silently returns wrong results — so
|
||||
an fp16 copy must exist, while the bf16 ``scales`` survive for the
|
||||
small-batch w4a16 fallback.
|
||||
"""
|
||||
method, layer = w4a8_layer
|
||||
layer.scales.data.fill_(0.125) # exactly representable in both dtypes
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.scales_fp16.dtype is torch.float16
|
||||
assert layer.scales_fp16.shape == layer.scales.shape
|
||||
assert torch.equal(layer.scales_fp16.data.float(), layer.scales.data.float())
|
||||
# The w4a16 fallback still needs the original activation-dtype scales.
|
||||
assert layer.scales.dtype is torch.bfloat16
|
||||
|
||||
|
||||
def test_reuses_w4a16_weight_layout(w4a8_layer) -> None:
|
||||
"""w4a8 and w4a16 accept bit-identical weights, so there is no repacking."""
|
||||
method, layer = w4a8_layer
|
||||
qweight_before = layer.qweight.data.clone()
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.qweight.dtype is torch.int32
|
||||
assert layer.qweight.shape == (IN_FEATURES // 8, OUT_FEATURES)
|
||||
assert torch.equal(layer.qweight.data, qweight_before)
|
||||
# Symmetric int4: the zero point is the constant 8, as for w4a16.
|
||||
assert layer.qzeros.dtype is torch.int8
|
||||
assert layer.qzeros.tolist() == [8]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_weights
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_falls_back_to_w4a16_for_small_batches(monkeypatch, w4a8_layer) -> None:
|
||||
"""Below the token threshold, per-token quant overhead dominates."""
|
||||
method, layer = w4a8_layer
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
calls = []
|
||||
|
||||
def record_fallback(self, layer, x, bias=None):
|
||||
calls.append(tuple(x.shape))
|
||||
return torch.zeros(x.shape[:-1] + (OUT_FEATURES,), dtype=x.dtype)
|
||||
|
||||
monkeypatch.setattr(INCXPULinearMethod, "apply_weights", record_fallback)
|
||||
monkeypatch.setattr(
|
||||
torch.ops._xpu_C,
|
||||
"int4_gemm_w4a8",
|
||||
lambda *args: pytest.fail("small batches must not call the w4a8 kernel"),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
tokens = method._MIN_TOKENS_FOR_INT8 - 1
|
||||
x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
|
||||
out = method.apply_weights(layer, x)
|
||||
|
||||
assert calls == [(tokens, IN_FEATURES)]
|
||||
assert out.shape == (tokens, OUT_FEATURES)
|
||||
|
||||
|
||||
@requires_xpu_ops
|
||||
def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
|
||||
"""At/above the threshold, activations are int8-quantized per token."""
|
||||
method, layer = w4a8_layer
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
monkeypatch.setattr(
|
||||
INCXPULinearMethod,
|
||||
"apply_weights",
|
||||
lambda self, layer, x, bias=None: pytest.fail(
|
||||
"large batches must not take the w4a16 fallback"
|
||||
),
|
||||
)
|
||||
captured: dict[str, tuple] = {}
|
||||
_stub_quant(monkeypatch, captured)
|
||||
_stub_gemm(monkeypatch, captured)
|
||||
|
||||
tokens = method._MIN_TOKENS_FOR_INT8
|
||||
x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
|
||||
out = method.apply_weights(layer, x)
|
||||
|
||||
# Symmetric 8-bit per-token quantization over the flattened activations.
|
||||
assert captured["quant_args"] == ((tokens, IN_FEATURES), True, 8)
|
||||
|
||||
(
|
||||
quant_x,
|
||||
x_scale,
|
||||
x_zero,
|
||||
qweight,
|
||||
w_scale,
|
||||
w_zp,
|
||||
group_size,
|
||||
g_idx,
|
||||
bias,
|
||||
) = captured["gemm_args"]
|
||||
assert quant_x.dtype is torch.int8
|
||||
# Both scales must reach the kernel as fp16, whatever the model dtype.
|
||||
assert x_scale.dtype is torch.float16
|
||||
assert w_scale.dtype is torch.float16
|
||||
assert w_scale is layer.scales_fp16
|
||||
assert x_zero.dtype is torch.int32
|
||||
assert qweight is layer.qweight
|
||||
assert w_zp is layer.qzeros
|
||||
assert group_size == 128
|
||||
assert g_idx is None
|
||||
assert bias is None
|
||||
|
||||
# The kernel emits fp16; the result is cast back to the activation dtype.
|
||||
assert out.dtype is torch.bfloat16
|
||||
assert out.shape == (tokens, OUT_FEATURES)
|
||||
|
||||
|
||||
@requires_xpu_ops
|
||||
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
|
||||
def test_scales_are_fp16_for_any_activation_dtype(
|
||||
monkeypatch, w4a8_layer, act_dtype
|
||||
) -> None:
|
||||
"""fp16 scales are required for bf16 *and* fp16 activations alike."""
|
||||
method, layer = w4a8_layer
|
||||
layer.scales.data = layer.scales.data.to(act_dtype)
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
captured: dict[str, tuple] = {}
|
||||
_stub_quant(monkeypatch, captured)
|
||||
_stub_gemm(monkeypatch, captured)
|
||||
|
||||
x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=act_dtype)
|
||||
out = method.apply_weights(layer, x)
|
||||
|
||||
assert captured["gemm_args"][1].dtype is torch.float16
|
||||
assert captured["gemm_args"][4].dtype is torch.float16
|
||||
assert out.dtype is act_dtype
|
||||
|
||||
|
||||
@requires_xpu_ops
|
||||
def test_forwards_bias_to_kernel(monkeypatch, w4a8_layer) -> None:
|
||||
"""Bias is applied by the kernel, not added afterwards."""
|
||||
method, layer = w4a8_layer
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
captured: dict[str, tuple] = {}
|
||||
_stub_quant(monkeypatch)
|
||||
_stub_gemm(monkeypatch, captured)
|
||||
|
||||
bias = torch.zeros(OUT_FEATURES, dtype=torch.bfloat16)
|
||||
x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=torch.bfloat16)
|
||||
method.apply_weights(layer, x, bias)
|
||||
|
||||
assert captured["gemm_args"][8] is bias
|
||||
|
||||
|
||||
@requires_xpu_ops
|
||||
def test_preserves_leading_dims(monkeypatch, w4a8_layer) -> None:
|
||||
"""3D activations must round-trip through the flatten/reshape."""
|
||||
method, layer = w4a8_layer
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
_stub_quant(monkeypatch)
|
||||
_stub_gemm(monkeypatch)
|
||||
|
||||
tokens = method._MIN_TOKENS_FOR_INT8
|
||||
x = torch.zeros(4, tokens, IN_FEATURES, dtype=torch.bfloat16)
|
||||
out = method.apply_weights(layer, x)
|
||||
|
||||
assert out.shape == (4, tokens, OUT_FEATURES)
|
||||
assert out.dtype is torch.bfloat16
|
||||
|
||||
|
||||
@requires_xpu_ops
|
||||
def test_batch_gate_uses_flattened_token_count(monkeypatch, w4a8_layer) -> None:
|
||||
"""The gate counts total tokens, not the leading dimension."""
|
||||
method, layer = w4a8_layer
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
monkeypatch.setattr(
|
||||
INCXPULinearMethod,
|
||||
"apply_weights",
|
||||
lambda self, layer, x, bias=None: pytest.fail(
|
||||
"flattened token count is at the threshold; must use w4a8"
|
||||
),
|
||||
)
|
||||
_stub_quant(monkeypatch)
|
||||
_stub_gemm(monkeypatch)
|
||||
|
||||
tokens = method._MIN_TOKENS_FOR_INT8
|
||||
# Each leading slice is below the threshold, but 4 * (tokens // 4) is not.
|
||||
x = torch.zeros(4, tokens // 4, IN_FEATURES, dtype=torch.bfloat16)
|
||||
out = method.apply_weights(layer, x)
|
||||
|
||||
assert out.shape == (4, tokens // 4, OUT_FEATURES)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End to end on real weights
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@requires_w4a8_kernel
|
||||
@pytest.mark.parametrize("model", E2E_MODELS)
|
||||
def test_e2e_generates_with_real_weights(vllm_runner, monkeypatch, model) -> None:
|
||||
"""The whole path — real int4 checkpoint, real kernel, real output.
|
||||
|
||||
The stubbed tests above cannot catch a wrong-but-plausible calling
|
||||
convention, because the fake kernel accepts anything. Greedy decoding of a
|
||||
factual prompt does: wrongly set scales or a bad weight layout turn the
|
||||
answer into noise rather than raising.
|
||||
"""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
|
||||
with vllm_runner(
|
||||
model, max_model_len=512, enforce_eager=True, gpu_memory_utilization=0.55
|
||||
) as vllm_model:
|
||||
out = vllm_model.generate_greedy(["The capital of France is"], max_tokens=8)
|
||||
|
||||
assert "Paris" in out[0][1]
|
||||
|
||||
|
||||
@requires_w4a8_kernel
|
||||
def test_e2e_selects_w4a8_for_every_linear(vllm_runner, monkeypatch) -> None:
|
||||
"""Guard against the validation silently becoming dead code.
|
||||
|
||||
If backend selection stopped routing here, the generation test above would
|
||||
still pass on the w4a16 path, so assert the method is actually in use and
|
||||
that every partition shape it accepted honours the alignment rule.
|
||||
"""
|
||||
monkeypatch.setenv(_BACKEND_ENV, "w4a8")
|
||||
# apply_model pickles the closure below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
with vllm_runner(
|
||||
E2E_MODELS[0].values[0],
|
||||
max_model_len=512,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.55,
|
||||
) as vllm_model:
|
||||
|
||||
def check_model(model):
|
||||
shapes = []
|
||||
for module in model.modules():
|
||||
scheme = getattr(getattr(module, "quant_method", None), "scheme", None)
|
||||
if isinstance(scheme, INCXPUW4A8LinearMethod):
|
||||
# qweight is the packed [in // 8, out] layout.
|
||||
packed_in, out = module.qweight.shape
|
||||
shapes.append((packed_in * 8, out))
|
||||
|
||||
assert shapes, "no linear layer used the w4a8 method"
|
||||
assert all(i % 8 == 0 and o % 8 == 0 for i, o in shapes), shapes
|
||||
|
||||
vllm_model.apply_model(check_model)
|
||||
|
||||
@@ -313,6 +313,7 @@ if TYPE_CHECKING:
|
||||
VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32
|
||||
VLLM_XPU_ENABLE_XPU_GRAPH: bool = False
|
||||
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
|
||||
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
|
||||
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
|
||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
@@ -2111,6 +2112,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_XPU_USE_SAMPLER_KERNEL": lambda: bool(
|
||||
int(os.getenv("VLLM_XPU_USE_SAMPLER_KERNEL", "1"))
|
||||
),
|
||||
# Kernel backend for INC weight-only intN (WNA16) linear layers on XPU.
|
||||
# "auto" keeps the default preference order (ARK when importable, else the
|
||||
# oneDNN w4a16 path). "ark" forces the auto_round_kernel backend, "w4a16"
|
||||
# forces oneDNN int4_gemm_w4a16, and "w4a8" additionally quantizes
|
||||
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
|
||||
# The two oneDNN backends are int4-only; ARK also serves int2.
|
||||
# Which one is fastest is device-dependent, so this is left as an opt-in.
|
||||
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
|
||||
),
|
||||
# Enable simple KV offload.
|
||||
"VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0"))
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from .inc_wna16_linear import INCXPULinearMethod
|
||||
|
||||
|
||||
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
|
||||
"""XPU linear method for INC int4 weights with dynamic int8 activations.
|
||||
|
||||
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
|
||||
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
|
||||
no extra repacking is needed. Activations are dynamically quantized
|
||||
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
|
||||
instead of upconverting the weights to the activation dtype. That is a win
|
||||
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
|
||||
is dominated by dequantization rather than by weight bandwidth.
|
||||
|
||||
Both scale tensors must be float16: the kernel reads them as fp16
|
||||
regardless of the activation dtype, so bf16 scales are silently
|
||||
misinterpreted. The activations themselves may be bf16 or fp16.
|
||||
|
||||
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
|
||||
device-dependent, so it is not the default: it was measured faster on B70
|
||||
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
|
||||
"""
|
||||
|
||||
# int8 activations only pay off once the GEMM is compute-bound. Below this
|
||||
# token count the per-token quantization overhead dominates and w4a16 is
|
||||
# faster, so fall back to it per-call.
|
||||
_MIN_TOKENS_FOR_INT8 = 512
|
||||
|
||||
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
|
||||
# parallelism shards these, so an aligned model dimension can still yield an
|
||||
# unaligned partition.
|
||||
_DIM_ALIGNMENT = 8
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
unaligned = [
|
||||
(name, size)
|
||||
for name, size in (
|
||||
("input", input_size_per_partition),
|
||||
("output", output_size_per_partition),
|
||||
)
|
||||
if size % self._DIM_ALIGNMENT != 0
|
||||
]
|
||||
if unaligned:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
|
||||
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
|
||||
+ ", ".join(f"{name}={size}" for name, size in unaligned)
|
||||
+ f". Partition shape: ({input_size_per_partition}, "
|
||||
f"{output_size_per_partition})."
|
||||
)
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
|
||||
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
|
||||
layer.scales_fp16 = Parameter(
|
||||
layer.scales.data.to(torch.float16), requires_grad=False
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
|
||||
return super().apply_weights(layer, x, bias)
|
||||
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale.to(torch.float16),
|
||||
x_zero,
|
||||
layer.qweight,
|
||||
layer.scales_fp16,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None,
|
||||
bias,
|
||||
)
|
||||
return out.to(x.dtype).reshape(out_shape)
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
@@ -21,6 +22,32 @@ logger = init_logger(__name__)
|
||||
|
||||
XPU_WNA16_SUPPORTED_BITS = {2, 4}
|
||||
|
||||
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
|
||||
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
|
||||
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
|
||||
|
||||
|
||||
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
|
||||
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
|
||||
|
||||
The backend is requested explicitly, so an unusable configuration is an
|
||||
error rather than something to silently fall back from.
|
||||
"""
|
||||
import torch
|
||||
|
||||
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
|
||||
"which this build of vllm-xpu-kernels does not provide. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
|
||||
f"positive multiple of 32, got {layer_config.group_size}. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
|
||||
|
||||
class INCWna16Scheme(INCScheme):
|
||||
@staticmethod
|
||||
@@ -38,12 +65,31 @@ class INCWna16Scheme(INCScheme):
|
||||
if current_platform.is_xpu():
|
||||
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
|
||||
from .inc_ark_ops import get_ark_state
|
||||
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCXPULinearMethod,
|
||||
)
|
||||
|
||||
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
|
||||
if backend in XPU_ONEDNN_BACKENDS:
|
||||
if layer_config.bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
|
||||
f"int4, got int{layer_config.bits}. Layer: {prefix}."
|
||||
)
|
||||
if backend == "w4a8":
|
||||
_check_xpu_w4a8_supported(layer_config, prefix)
|
||||
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
|
||||
return INCLinearMethod(INCXPULinearMethod(layer_config))
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if backend == "ark" and not is_ark_available:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
|
||||
f"auto_round_kernel is unavailable: "
|
||||
f"{ark_error or 'unknown error'}. Layer: {prefix}."
|
||||
)
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
elif layer_config.bits == 2:
|
||||
|
||||
Reference in New Issue
Block a user