From 9fd750f00de5ab8fa172c34e219b3345411e9a91 Mon Sep 17 00:00:00 2001 From: Thanaji Rao Thakkalapelli Date: Fri, 21 Aug 2026 01:38:59 -0500 Subject: [PATCH] [XPU][INC] Add int4 w4a8 (dynamic int8 activation) backend for INC linear layers (#50501) Signed-off-by: Thanaji Rao Thakkalapelli Co-authored-by: Kunshang Ji --- tests/quantization/test_auto_round.py | 618 ++++++++++++++++++ vllm/envs.py | 11 + .../inc/schemes/inc_w4a8_linear.py | 112 ++++ .../inc/schemes/inc_wna16_scheme.py | 46 ++ 4 files changed, 787 insertions(+) create mode 100644 vllm/model_executor/layers/quantization/inc/schemes/inc_w4a8_linear.py diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index 86bd857923..9d48926853 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -33,6 +33,9 @@ from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import ( INCLinearScheme, ) +from vllm.model_executor.layers.quantization.inc.schemes.inc_w4a8_linear import ( + INCXPUW4A8LinearMethod, +) from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import ( INCARKLinearMethod, INCWNA16LinearScheme, @@ -1482,3 +1485,618 @@ class TestGetLayerConfigFusedQKV: DummyLayer(), "model.layers.0.self_attn.qkv_proj" ) assert bits == 8 + + +# --------------------------------------------------------------------------- +# INC XPU w4a8: int4 weights with dynamic per-token int8 activations +# +# The w4a8 path keeps int4 weights but dynamically quantizes activations to +# per-token symmetric int8, so the GEMM stays on the int8 datapath instead of +# upconverting weights to the activation dtype. It is opt-in through +# VLLM_XPU_INC_WNA16_BACKEND because which kernel is fastest depends on the +# device: ARK can use XMX int8 on some XPUs, and the default ("auto") +# preference order is deliberately left unchanged. +# +# The backend-selection tests stub out ARK and the platform; the weight-creation +# tests only need the TP-rank stubs. Both run anywhere. The apply_weights tests +# stub out int4_gemm_w4a8 — what needs guarding there is the calling convention +# (scale dtypes, argument order, weight reuse, batch-size gate), which is where +# this path is easy to get silently wrong, since the kernel accepts bf16 scales +# and returns plausible-looking garbage rather than raising. The ones that cross +# the token threshold reach ``vllm._xpu_ops`` for the activation quantizer, so +# they are XPU-image only; the small-batch fallback test returns before that +# import and must stay unmarked to keep covering the gate off-XPU. The +# end-to-end tests load a real int4 checkpoint and additionally need the op. +# --------------------------------------------------------------------------- + +_BACKEND_ENV = "VLLM_XPU_INC_WNA16_BACKEND" +_ARK_STATE = ( + "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state" +) +_QUANT_REF = "vllm._xpu_ops.xpu_ops.dynamic_per_token_int8_quant_ref" + +IN_FEATURES = 128 +OUT_FEATURES = 64 + +# auto-round int4/sym/g128 checkpoints, which is what this backend serves. The +# GPTQ- and AWQ-packed variants take different repack branches in +# ``process_weights_after_loading``, so both are worth loading. +E2E_MODELS = [ + pytest.param("OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", id="auto_round:auto_gptq"), + pytest.param( + "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", id="auto_round:auto_awq" + ), +] + + +def _has_xpu_ops() -> bool: + """True when ``vllm._xpu_ops`` is importable. + + ``apply_weights`` imports it for the activation quantizer, and the stubs + below patch through it, so tests that exercise that path cannot even be set + up on a build without vllm-xpu-kernels installed. + """ + if not current_platform.is_xpu(): + return False + try: + import vllm._xpu_ops # noqa: F401 + except ImportError: + return False + return True + + +def _has_w4a8_kernel() -> bool: + """True when this build can actually run the w4a8 GEMM. + + The op is registered as a side effect of importing ``vllm._xpu_ops``, so it + is not visible on ``torch.ops._xpu_C`` until that import happens. + """ + return _has_xpu_ops() and hasattr(torch.ops._xpu_C, "int4_gemm_w4a8") + + +requires_xpu_ops = pytest.mark.skipif( + not _has_xpu_ops(), + reason="needs an XPU image with vllm._xpu_ops available", +) + +requires_w4a8_kernel = pytest.mark.skipif( + not _has_w4a8_kernel(), + reason="needs an XPU with the int4_gemm_w4a8 op", +) + + +def _fake_xpu(monkeypatch, ark_available: bool = True) -> None: + class DummyQuantLinear: + pass + + monkeypatch.setattr(current_platform, "is_xpu", lambda: True) + monkeypatch.setattr(current_platform, "is_cpu", lambda: False) + monkeypatch.setattr( + _ARK_STATE, + lambda: ( + (True, None, object(), DummyQuantLinear) + if ark_available + else (False, "missing", None, None) + ), + ) + + +def _with_w4a8_kernel(monkeypatch) -> None: + """Present the op so the w4a8 support check passes off-XPU.""" + + class FakeOps: + int4_gemm_w4a8 = object() + + monkeypatch.setattr(torch.ops, "_xpu_C", FakeOps, raising=False) + + +def _dispatch(layer_config=None): + return INCWna16Scheme().get_linear_method( + object(), object(), "layer", layer_config or make_layer_config() + ) + + +@pytest.fixture +def single_tp_rank(monkeypatch): + """Parameter creation reads the TP rank/size; there is no distributed init.""" + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_world_size", + lambda: 1, + ) + + +def _create_weights( + method, + input_size_per_partition: int = IN_FEATURES, + output_partition_sizes: list[int] | None = None, +) -> torch.nn.Module: + class DummyLayer(torch.nn.Module): + pass + + layer = DummyLayer() + output_partition_sizes = ( + [OUT_FEATURES] if output_partition_sizes is None else output_partition_sizes + ) + method.create_weights( + layer=layer, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + input_size=input_size_per_partition, + output_size=sum(output_partition_sizes), + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + return layer + + +@pytest.fixture +def w4a8_layer(single_tp_rank): + """A w4a8 method plus a layer whose weights are created but not processed.""" + method = INCXPUW4A8LinearMethod(make_layer_config()) + return method, _create_weights(method) + + +def _stub_quant(monkeypatch, captured=None): + def fake_quant(x, use_sym, bits): + if captured is not None: + captured["quant_args"] = (tuple(x.shape), use_sym, bits) + m, k = x.shape + return ( + torch.zeros(m, k, dtype=torch.int8), + torch.ones(m, 1, dtype=x.dtype), + torch.zeros(m, 1, dtype=torch.int32), + ) + + monkeypatch.setattr(_QUANT_REF, fake_quant, raising=False) + + +def _stub_gemm(monkeypatch, captured=None): + def fake_gemm(*args): + if captured is not None: + captured["gemm_args"] = args + # The real kernel always emits float16, whatever the model dtype. + return torch.zeros(args[0].shape[0], OUT_FEATURES, dtype=torch.float16) + + monkeypatch.setattr(torch.ops._xpu_C, "int4_gemm_w4a8", fake_gemm, raising=False) + + +# --------------------------------------------------------------------------- +# Backend selection +# --------------------------------------------------------------------------- + + +def test_default_backend_keeps_ark_preference(monkeypatch) -> None: + """The "auto" default must not change upstream behaviour. + + ARK still wins whenever it is importable. + """ + monkeypatch.delenv(_BACKEND_ENV, raising=False) + _fake_xpu(monkeypatch, ark_available=True) + _with_w4a8_kernel(monkeypatch) + + method = _dispatch() + + assert isinstance(method, INCLinearMethod) + assert isinstance(method.scheme, INCARKLinearMethod) + + +def test_w4a8_backend_overrides_available_ark(monkeypatch) -> None: + """An explicit request wins even though ARK would otherwise be chosen.""" + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + _fake_xpu(monkeypatch, ark_available=True) + _with_w4a8_kernel(monkeypatch) + + method = _dispatch() + + assert isinstance(method.scheme, INCXPUW4A8LinearMethod) + assert not isinstance(method.scheme, INCARKLinearMethod) + # w4a8 subclasses w4a16 so it can fall back for small batches. + assert isinstance(method.scheme, INCXPULinearMethod) + + +def test_w4a16_backend_overrides_available_ark(monkeypatch) -> None: + monkeypatch.setenv(_BACKEND_ENV, "w4a16") + _fake_xpu(monkeypatch, ark_available=True) + + method = _dispatch() + + assert isinstance(method.scheme, INCXPULinearMethod) + assert not isinstance(method.scheme, INCXPUW4A8LinearMethod) + assert not isinstance(method.scheme, INCARKLinearMethod) + + +def test_ark_backend_requested_but_unavailable_raises(monkeypatch) -> None: + """Explicitly asking for ARK must fail loudly, not fall back silently.""" + monkeypatch.setenv(_BACKEND_ENV, "ark") + _fake_xpu(monkeypatch, ark_available=False) + + with pytest.raises(NotImplementedError, match="auto_round_kernel is unavailable"): + _dispatch() + + +@pytest.mark.parametrize("backend", ["w4a16", "w4a8"]) +def test_onednn_backends_reject_int2(monkeypatch, backend) -> None: + """The oneDNN int4 GEMMs cannot serve int2; ARK is the only int2 path.""" + monkeypatch.setenv(_BACKEND_ENV, backend) + _fake_xpu(monkeypatch, ark_available=True) + _with_w4a8_kernel(monkeypatch) + + with pytest.raises(NotImplementedError, match="only supports int4"): + _dispatch(make_layer_config(bits=2)) + + +@pytest.mark.parametrize("group_size", [48, -1, 0]) +def test_w4a8_rejects_unaligned_group_size(monkeypatch, group_size) -> None: + """The kernel needs a group size that is a positive multiple of 32.""" + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + _fake_xpu(monkeypatch, ark_available=True) + _with_w4a8_kernel(monkeypatch) + + with pytest.raises(NotImplementedError, match="multiple of 32"): + _dispatch(make_layer_config(group_size=group_size)) + + +@pytest.mark.parametrize("group_size", [128, 64, 32]) +def test_w4a8_accepts_aligned_group_size(monkeypatch, group_size) -> None: + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + _fake_xpu(monkeypatch, ark_available=True) + _with_w4a8_kernel(monkeypatch) + + method = _dispatch(make_layer_config(group_size=group_size)) + + assert isinstance(method.scheme, INCXPUW4A8LinearMethod) + + +def test_w4a8_requires_the_kernel(monkeypatch) -> None: + """Older vllm-xpu-kernels builds lack the op entirely.""" + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + _fake_xpu(monkeypatch, ark_available=True) + + class FakeOpsNoW4A8: + pass + + monkeypatch.setattr(torch.ops, "_xpu_C", FakeOpsNoW4A8, raising=False) + + with pytest.raises(NotImplementedError, match="int4_gemm_w4a8"): + _dispatch() + + +def test_invalid_backend_value_raises(monkeypatch) -> None: + import vllm.envs as envs + + monkeypatch.setenv(_BACKEND_ENV, "onednn") + with pytest.raises(ValueError, match=_BACKEND_ENV): + _ = envs.VLLM_XPU_INC_WNA16_BACKEND + + +# --------------------------------------------------------------------------- +# Partition shape +# --------------------------------------------------------------------------- +# +# ``XPUW4A8IntLinearKernel.can_implement`` rejects partition shapes whose in/out +# dims are not multiples of 8, but that check is unreachable from the INC path: +# the shapes only exist once ``create_weights`` is called, well after backend +# selection. Unaligned dims must not reach the kernel — the int4 packing divides +# both dims by 8, so an unaligned size is silently truncated into a weight tensor +# that is too small rather than rejected. + + +def test_rejects_unaligned_input_partition(single_tp_rank) -> None: + method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32)) + + with pytest.raises(NotImplementedError, match=r"multiples of 8.*input=132"): + _create_weights(method, input_size_per_partition=132) + + +def test_rejects_unaligned_output_partition(single_tp_rank) -> None: + method = INCXPUW4A8LinearMethod(make_layer_config()) + + with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=60"): + _create_weights(method, output_partition_sizes=[60]) + + +def test_rejects_unaligned_sum_of_output_partitions(single_tp_rank) -> None: + """A fused QKV/MLP layer is only as aligned as the sum of its shards.""" + method = INCXPUW4A8LinearMethod(make_layer_config()) + + with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=76"): + _create_weights(method, output_partition_sizes=[OUT_FEATURES, 12]) + + +def test_reports_both_unaligned_dims(single_tp_rank) -> None: + """The message names every offending dim, not just the first.""" + method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32)) + + with pytest.raises(NotImplementedError) as excinfo: + _create_weights( + method, input_size_per_partition=132, output_partition_sizes=[60] + ) + + assert "input=132" in str(excinfo.value) + assert "output=60" in str(excinfo.value) + + +def test_accepts_aligned_partition_shape(single_tp_rank) -> None: + """Multiples of 8 that are not powers of two are still fine.""" + method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32)) + + layer = _create_weights( + method, input_size_per_partition=96, output_partition_sizes=[24, 48] + ) + + assert layer.qweight.shape == (96 // 8, 72) + + +def test_w4a16_accepts_unaligned_partition_shape(single_tp_rank) -> None: + """The constraint is a w4a8 kernel requirement; w4a16 must stay unaffected.""" + method = INCXPULinearMethod(make_layer_config(group_size=32)) + + layer = _create_weights(method, output_partition_sizes=[60]) + + assert layer.scales.shape[1] == 60 + + +# --------------------------------------------------------------------------- +# Weight processing +# --------------------------------------------------------------------------- + + +def test_keeps_fp16_scales(w4a8_layer) -> None: + """The kernel reads scales as fp16 regardless of activation dtype. + + Passing bf16 scales does not raise — it silently returns wrong results — so + an fp16 copy must exist, while the bf16 ``scales`` survive for the + small-batch w4a16 fallback. + """ + method, layer = w4a8_layer + layer.scales.data.fill_(0.125) # exactly representable in both dtypes + + method.process_weights_after_loading(layer) + + assert layer.scales_fp16.dtype is torch.float16 + assert layer.scales_fp16.shape == layer.scales.shape + assert torch.equal(layer.scales_fp16.data.float(), layer.scales.data.float()) + # The w4a16 fallback still needs the original activation-dtype scales. + assert layer.scales.dtype is torch.bfloat16 + + +def test_reuses_w4a16_weight_layout(w4a8_layer) -> None: + """w4a8 and w4a16 accept bit-identical weights, so there is no repacking.""" + method, layer = w4a8_layer + qweight_before = layer.qweight.data.clone() + + method.process_weights_after_loading(layer) + + assert layer.qweight.dtype is torch.int32 + assert layer.qweight.shape == (IN_FEATURES // 8, OUT_FEATURES) + assert torch.equal(layer.qweight.data, qweight_before) + # Symmetric int4: the zero point is the constant 8, as for w4a16. + assert layer.qzeros.dtype is torch.int8 + assert layer.qzeros.tolist() == [8] + + +# --------------------------------------------------------------------------- +# apply_weights +# --------------------------------------------------------------------------- + + +def test_falls_back_to_w4a16_for_small_batches(monkeypatch, w4a8_layer) -> None: + """Below the token threshold, per-token quant overhead dominates.""" + method, layer = w4a8_layer + method.process_weights_after_loading(layer) + + calls = [] + + def record_fallback(self, layer, x, bias=None): + calls.append(tuple(x.shape)) + return torch.zeros(x.shape[:-1] + (OUT_FEATURES,), dtype=x.dtype) + + monkeypatch.setattr(INCXPULinearMethod, "apply_weights", record_fallback) + monkeypatch.setattr( + torch.ops._xpu_C, + "int4_gemm_w4a8", + lambda *args: pytest.fail("small batches must not call the w4a8 kernel"), + raising=False, + ) + + tokens = method._MIN_TOKENS_FOR_INT8 - 1 + x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16) + out = method.apply_weights(layer, x) + + assert calls == [(tokens, IN_FEATURES)] + assert out.shape == (tokens, OUT_FEATURES) + + +@requires_xpu_ops +def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None: + """At/above the threshold, activations are int8-quantized per token.""" + method, layer = w4a8_layer + method.process_weights_after_loading(layer) + + monkeypatch.setattr( + INCXPULinearMethod, + "apply_weights", + lambda self, layer, x, bias=None: pytest.fail( + "large batches must not take the w4a16 fallback" + ), + ) + captured: dict[str, tuple] = {} + _stub_quant(monkeypatch, captured) + _stub_gemm(monkeypatch, captured) + + tokens = method._MIN_TOKENS_FOR_INT8 + x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16) + out = method.apply_weights(layer, x) + + # Symmetric 8-bit per-token quantization over the flattened activations. + assert captured["quant_args"] == ((tokens, IN_FEATURES), True, 8) + + ( + quant_x, + x_scale, + x_zero, + qweight, + w_scale, + w_zp, + group_size, + g_idx, + bias, + ) = captured["gemm_args"] + assert quant_x.dtype is torch.int8 + # Both scales must reach the kernel as fp16, whatever the model dtype. + assert x_scale.dtype is torch.float16 + assert w_scale.dtype is torch.float16 + assert w_scale is layer.scales_fp16 + assert x_zero.dtype is torch.int32 + assert qweight is layer.qweight + assert w_zp is layer.qzeros + assert group_size == 128 + assert g_idx is None + assert bias is None + + # The kernel emits fp16; the result is cast back to the activation dtype. + assert out.dtype is torch.bfloat16 + assert out.shape == (tokens, OUT_FEATURES) + + +@requires_xpu_ops +@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16]) +def test_scales_are_fp16_for_any_activation_dtype( + monkeypatch, w4a8_layer, act_dtype +) -> None: + """fp16 scales are required for bf16 *and* fp16 activations alike.""" + method, layer = w4a8_layer + layer.scales.data = layer.scales.data.to(act_dtype) + method.process_weights_after_loading(layer) + + captured: dict[str, tuple] = {} + _stub_quant(monkeypatch, captured) + _stub_gemm(monkeypatch, captured) + + x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=act_dtype) + out = method.apply_weights(layer, x) + + assert captured["gemm_args"][1].dtype is torch.float16 + assert captured["gemm_args"][4].dtype is torch.float16 + assert out.dtype is act_dtype + + +@requires_xpu_ops +def test_forwards_bias_to_kernel(monkeypatch, w4a8_layer) -> None: + """Bias is applied by the kernel, not added afterwards.""" + method, layer = w4a8_layer + method.process_weights_after_loading(layer) + + captured: dict[str, tuple] = {} + _stub_quant(monkeypatch) + _stub_gemm(monkeypatch, captured) + + bias = torch.zeros(OUT_FEATURES, dtype=torch.bfloat16) + x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=torch.bfloat16) + method.apply_weights(layer, x, bias) + + assert captured["gemm_args"][8] is bias + + +@requires_xpu_ops +def test_preserves_leading_dims(monkeypatch, w4a8_layer) -> None: + """3D activations must round-trip through the flatten/reshape.""" + method, layer = w4a8_layer + method.process_weights_after_loading(layer) + + _stub_quant(monkeypatch) + _stub_gemm(monkeypatch) + + tokens = method._MIN_TOKENS_FOR_INT8 + x = torch.zeros(4, tokens, IN_FEATURES, dtype=torch.bfloat16) + out = method.apply_weights(layer, x) + + assert out.shape == (4, tokens, OUT_FEATURES) + assert out.dtype is torch.bfloat16 + + +@requires_xpu_ops +def test_batch_gate_uses_flattened_token_count(monkeypatch, w4a8_layer) -> None: + """The gate counts total tokens, not the leading dimension.""" + method, layer = w4a8_layer + method.process_weights_after_loading(layer) + + monkeypatch.setattr( + INCXPULinearMethod, + "apply_weights", + lambda self, layer, x, bias=None: pytest.fail( + "flattened token count is at the threshold; must use w4a8" + ), + ) + _stub_quant(monkeypatch) + _stub_gemm(monkeypatch) + + tokens = method._MIN_TOKENS_FOR_INT8 + # Each leading slice is below the threshold, but 4 * (tokens // 4) is not. + x = torch.zeros(4, tokens // 4, IN_FEATURES, dtype=torch.bfloat16) + out = method.apply_weights(layer, x) + + assert out.shape == (4, tokens // 4, OUT_FEATURES) + + +# --------------------------------------------------------------------------- +# End to end on real weights +# --------------------------------------------------------------------------- + + +@requires_w4a8_kernel +@pytest.mark.parametrize("model", E2E_MODELS) +def test_e2e_generates_with_real_weights(vllm_runner, monkeypatch, model) -> None: + """The whole path — real int4 checkpoint, real kernel, real output. + + The stubbed tests above cannot catch a wrong-but-plausible calling + convention, because the fake kernel accepts anything. Greedy decoding of a + factual prompt does: wrongly set scales or a bad weight layout turn the + answer into noise rather than raising. + """ + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + + with vllm_runner( + model, max_model_len=512, enforce_eager=True, gpu_memory_utilization=0.55 + ) as vllm_model: + out = vllm_model.generate_greedy(["The capital of France is"], max_tokens=8) + + assert "Paris" in out[0][1] + + +@requires_w4a8_kernel +def test_e2e_selects_w4a8_for_every_linear(vllm_runner, monkeypatch) -> None: + """Guard against the validation silently becoming dead code. + + If backend selection stopped routing here, the generation test above would + still pass on the w4a16 path, so assert the method is actually in use and + that every partition shape it accepted honours the alignment rule. + """ + monkeypatch.setenv(_BACKEND_ENV, "w4a8") + # apply_model pickles the closure below. + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + with vllm_runner( + E2E_MODELS[0].values[0], + max_model_len=512, + enforce_eager=True, + gpu_memory_utilization=0.55, + ) as vllm_model: + + def check_model(model): + shapes = [] + for module in model.modules(): + scheme = getattr(getattr(module, "quant_method", None), "scheme", None) + if isinstance(scheme, INCXPUW4A8LinearMethod): + # qweight is the packed [in // 8, out] layout. + packed_in, out = module.qweight.shape + shapes.append((packed_in * 8, out)) + + assert shapes, "no linear layer used the w4a8 method" + assert all(i % 8 == 0 and o % 8 == 0 for i, o in shapes), shapes + + vllm_model.apply_model(check_model) diff --git a/vllm/envs.py b/vllm/envs.py index e410dd7f54..e33e2fd272 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -313,6 +313,7 @@ if TYPE_CHECKING: VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32 VLLM_XPU_ENABLE_XPU_GRAPH: bool = False VLLM_XPU_USE_SAMPLER_KERNEL: bool = True + VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto" VLLM_LORA_ENABLE_DUAL_STREAM: bool = False VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" @@ -2111,6 +2112,16 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_XPU_USE_SAMPLER_KERNEL": lambda: bool( int(os.getenv("VLLM_XPU_USE_SAMPLER_KERNEL", "1")) ), + # Kernel backend for INC weight-only intN (WNA16) linear layers on XPU. + # "auto" keeps the default preference order (ARK when importable, else the + # oneDNN w4a16 path). "ark" forces the auto_round_kernel backend, "w4a16" + # forces oneDNN int4_gemm_w4a16, and "w4a8" additionally quantizes + # activations to per-token int8 (int4_gemm_w4a8) for large token counts. + # The two oneDNN backends are int4-only; ARK also serves int2. + # Which one is fastest is device-dependent, so this is left as an opt-in. + "VLLM_XPU_INC_WNA16_BACKEND": env_with_choices( + "VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"] + ), # Enable simple KV offload. "VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool( int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0")) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_w4a8_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_w4a8_linear.py new file mode 100644 index 0000000000..01665d40ad --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_w4a8_linear.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from torch.nn.parameter import Parameter + +from .inc_wna16_linear import INCXPULinearMethod + + +class INCXPUW4A8LinearMethod(INCXPULinearMethod): + """XPU linear method for INC int4 weights with dynamic int8 activations. + + Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` — + ``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so + no extra repacking is needed. Activations are dynamically quantized + per-token to symmetric int8, which keeps the GEMM on the int8 datapath + instead of upconverting the weights to the activation dtype. That is a win + for compute-bound shapes (large token counts, e.g. diffusion), where w4a16 + is dominated by dequantization rather than by weight bandwidth. + + Both scale tensors must be float16: the kernel reads them as fp16 + regardless of the activation dtype, so bf16 scales are silently + misinterpreted. The activations themselves may be bf16 or fp16. + + Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is + device-dependent, so it is not the default: it was measured faster on B70 + (Xe2), where ARK cannot use its XMX int8 path and falls back to fp16. + """ + + # int8 activations only pay off once the GEMM is compute-bound. Below this + # token count the per-token quantization overhead dominates and w4a16 is + # faster, so fall back to it per-call. + _MIN_TOKENS_FOR_INT8 = 512 + + # int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor + # parallelism shards these, so an aligned model dimension can still yield an + # unaligned partition. + _DIM_ALIGNMENT = 8 + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + unaligned = [ + (name, size) + for name, size in ( + ("input", input_size_per_partition), + ("output", output_size_per_partition), + ) + if size % self._DIM_ALIGNMENT != 0 + ] + if unaligned: + raise NotImplementedError( + "VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out " + f"sizes that are multiples of {self._DIM_ALIGNMENT}, got " + + ", ".join(f"{name}={size}" for name, size in unaligned) + + f". Partition shape: ({input_size_per_partition}, " + f"{output_size_per_partition})." + ) + super().create_weights( + layer=layer, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + input_size=input_size, + output_size=output_size, + params_dtype=params_dtype, + **extra_weight_attrs, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer) + # The kernel reads both scale tensors as fp16; keep an fp16 copy for the + # w4a8 path and leave ``scales`` untouched for the w4a16 fallback. + layer.scales_fp16 = Parameter( + layer.scales.data.to(torch.float16), requires_grad=False + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + reshaped_x = x.reshape(-1, x.shape[-1]) + if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8: + return super().apply_weights(layer, x, bias) + + from vllm._xpu_ops import xpu_ops as ops + + out_shape = x.shape[:-1] + (layer.qweight.shape[1],) + quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref( + reshaped_x, True, 8 + ) + out = torch.ops._xpu_C.int4_gemm_w4a8( + quant_x, + x_scale.to(torch.float16), + x_zero, + layer.qweight, + layer.scales_fp16, + layer.qzeros, + self.group_size, + None, + bias, + ) + return out.to(x.dtype).reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py index f83cc22e8e..77888db1eb 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING +from vllm import envs from vllm.logger import init_logger from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig @@ -21,6 +22,32 @@ logger = init_logger(__name__) XPU_WNA16_SUPPORTED_BITS = {2, 4} +# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the +# oneDNN int4 GEMMs rather than by ARK. These only cover int4. +XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8") + + +def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None: + """Raise unless ``int4_gemm_w4a8`` can serve this layer. + + The backend is requested explicitly, so an unusable configuration is an + error rather than something to silently fall back from. + """ + import torch + + if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"): + raise NotImplementedError( + "VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, " + "which this build of vllm-xpu-kernels does not provide. " + f"Layer: {prefix}." + ) + if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0: + raise NotImplementedError( + "VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a " + f"positive multiple of 32, got {layer_config.group_size}. " + f"Layer: {prefix}." + ) + class INCWna16Scheme(INCScheme): @staticmethod @@ -38,12 +65,31 @@ class INCWna16Scheme(INCScheme): if current_platform.is_xpu(): if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym: from .inc_ark_ops import get_ark_state + from .inc_w4a8_linear import INCXPUW4A8LinearMethod from .inc_wna16_linear import ( INCARKLinearMethod, INCXPULinearMethod, ) + backend = envs.VLLM_XPU_INC_WNA16_BACKEND + if backend in XPU_ONEDNN_BACKENDS: + if layer_config.bits != 4: + raise NotImplementedError( + f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports " + f"int4, got int{layer_config.bits}. Layer: {prefix}." + ) + if backend == "w4a8": + _check_xpu_w4a8_supported(layer_config, prefix) + return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config)) + return INCLinearMethod(INCXPULinearMethod(layer_config)) + is_ark_available, ark_error, _, _ = get_ark_state() + if backend == "ark" and not is_ark_available: + raise NotImplementedError( + "VLLM_XPU_INC_WNA16_BACKEND=ark was requested but " + f"auto_round_kernel is unavailable: " + f"{ark_error or 'unknown error'}. Layer: {prefix}." + ) if is_ark_available: return INCLinearMethod(INCARKLinearMethod(layer_config)) elif layer_config.bits == 2: