[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)
Signed-off-by: R <Ganesh.R@amd.com> Signed-off-by: Ganesh R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4d6874d137
commit
e12291d733
@@ -24,6 +24,9 @@ steps:
|
||||
- tests/kernels/test_onednn.py
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
- tests/kernels/quantization/test_zen_da8w4.py
|
||||
- vllm/model_executor/kernels/linear/mixed_precision/zentorch.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py
|
||||
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
- tests/kernels/mamba/test_cpu_short_conv.py
|
||||
- tests/kernels/mamba/test_causal_conv1d.py
|
||||
@@ -46,6 +49,7 @@ steps:
|
||||
tests/kernels/test_onednn.py \
|
||||
tests/kernels/test_awq_int4_to_int8.py \
|
||||
tests/kernels/quantization/test_cpu_fp8_scaled_mm.py \
|
||||
tests/kernels/quantization/test_zen_da8w4.py \
|
||||
tests/kernels/mamba/cpu/test_cpu_gdn_ops.py \
|
||||
tests/kernels/mamba/test_causal_conv1d.py \
|
||||
tests/kernels/mamba/test_mamba_ssm.py \
|
||||
|
||||
@@ -0,0 +1,661 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
|
||||
|
||||
The zentorch ops are mocked with reference implementations when zentorch is not
|
||||
installed, so the layout and dispatch contracts are covered in CI.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
|
||||
|
||||
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
_import_unpack_from_int32,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
GROUP_SIZE = 128
|
||||
IN_FEATURES = 512
|
||||
OUT_FEATURES = 128
|
||||
|
||||
|
||||
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
|
||||
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
|
||||
words = packed.view(torch.int32)
|
||||
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
|
||||
for i in range(8):
|
||||
nibble = (words >> (4 * i)) & 0xF
|
||||
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
|
||||
return out
|
||||
|
||||
|
||||
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
|
||||
n, k = unpacked.shape
|
||||
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
|
||||
out = torch.zeros(n, k // 8, dtype=torch.int32)
|
||||
for i in range(8):
|
||||
out |= (values[:, :, i] & 0xF) << (4 * i)
|
||||
return out
|
||||
|
||||
|
||||
def _quantize_per_group(
|
||||
weight: torch.Tensor, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
|
||||
out_features, in_features = weight.shape
|
||||
grouped = weight.float().reshape(
|
||||
out_features, in_features // group_size, group_size
|
||||
)
|
||||
scale = grouped.abs().amax(dim=-1) / 8.0
|
||||
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
|
||||
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _dequantize_per_group(
|
||||
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
|
||||
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
|
||||
|
||||
|
||||
def _dynamic_quant_matmul(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
|
||||
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
|
||||
out = (x_q.float() @ weight.t()) * scale
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_zentorch_ops():
|
||||
"""Register reference zentorch ops when zentorch is not installed."""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
|
||||
yield None
|
||||
return
|
||||
|
||||
calls: dict[str, tuple] = {}
|
||||
|
||||
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
|
||||
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
|
||||
in_features = weight.view(torch.int32).shape[1] * 8
|
||||
group_size = in_features // weight_scales.shape[0]
|
||||
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
|
||||
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
|
||||
weight_deq = _unpack_s4(weight, in_features) * scales
|
||||
out = _dynamic_quant_matmul(input, weight_deq, bias)
|
||||
return out.to(input.dtype)
|
||||
|
||||
def _fused_moe(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales=None,
|
||||
w2_scales=None,
|
||||
zentorch_op_name="",
|
||||
):
|
||||
calls["fused_moe"] = (
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
)
|
||||
output.zero_()
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
|
||||
lib_def.define(
|
||||
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
|
||||
"Tensor weight_scales, Tensor? bias=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
|
||||
)
|
||||
lib_def.define(
|
||||
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
|
||||
"Tensor topk_id, bool skip_weighted, str act, "
|
||||
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
|
||||
)
|
||||
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
|
||||
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
|
||||
lib_impl.impl("zentorch_fused_moe", _fused_moe)
|
||||
|
||||
yield calls
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
def _make_config(
|
||||
act_type: torch.dtype = torch.bfloat16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
zero_points: bool = False,
|
||||
group_size: int = GROUP_SIZE,
|
||||
) -> MPLinearLayerConfig:
|
||||
return MPLinearLayerConfig(
|
||||
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
weight_type=weight_type,
|
||||
act_type=act_type,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
|
||||
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
|
||||
quantized, scale = _quantize_per_group(weight, group_size)
|
||||
|
||||
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
|
||||
packed = pack_to_int32(quantized, 4, packed_dim=1)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
|
||||
layer.weight_packed.input_dim = 1
|
||||
layer.weight_packed.packed_dim = 1
|
||||
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
|
||||
layer.weight_zero_point = None
|
||||
return layer, _dequantize_per_group(quantized, scale, group_size)
|
||||
|
||||
|
||||
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
|
||||
return ZentorchWNA16LinearKernel(
|
||||
config, "weight_packed", "weight_scale", "weight_zero_point"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dense DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config_kwargs,reason",
|
||||
[
|
||||
({"act_type": torch.float32}, "f32 activations are rejected"),
|
||||
(
|
||||
{"weight_type": scalar_types.uint4, "zero_points": True},
|
||||
"asymmetric int4 is unsupported",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config(**config_kwargs))
|
||||
assert not kernel._maybe_process_da8w4_weights(layer), reason
|
||||
|
||||
|
||||
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"envs.VLLM_CPU_INT4_W4A8",
|
||||
False,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._zentorch_da8w4
|
||||
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
|
||||
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
|
||||
assert layer._zentorch_da8w4_packed.dtype == torch.int8
|
||||
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
|
||||
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
|
||||
assert layer._zentorch_da8w4_scale.shape == (
|
||||
IN_FEATURES // GROUP_SIZE,
|
||||
OUT_FEATURES,
|
||||
)
|
||||
assert layer._zentorch_da8w4_packed.is_contiguous()
|
||||
assert layer._zentorch_da8w4_scale.is_contiguous()
|
||||
# The checkpoint parameters are released once repacked.
|
||||
assert layer.weight_packed.numel() == 0
|
||||
assert layer.weight_scale.numel() == 0
|
||||
|
||||
|
||||
def test_da8w4_apply_weights_matches_dequantized_reference(
|
||||
mock_zentorch_ops, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, weight_deq = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
|
||||
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
|
||||
out = kernel.apply_weights(layer, x, bias)
|
||||
|
||||
expected = _dynamic_quant_matmul(x, weight_deq, bias)
|
||||
assert out.dtype == torch.bfloat16
|
||||
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
|
||||
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: "zentorch_dynamic_qlinear" not in ops,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_can_implement_requires_zen_cpu(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: False,
|
||||
)
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
|
||||
assert not ok
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
|
||||
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
|
||||
CPUWNA16LinearKernel,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
|
||||
"current_platform.is_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
unaligned = dataclasses.replace(
|
||||
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
|
||||
)
|
||||
assert unaligned.partition_weight_shape[1] % 32 != 0
|
||||
|
||||
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
|
||||
assert not ok, "the parent is expected to turn this shape away"
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
|
||||
assert ok, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[
|
||||
(-1, True), # per-channel, one group spanning K
|
||||
(GROUP_SIZE, True),
|
||||
(IN_FEATURES, True),
|
||||
(6, False), # even, so the parent's multiple-of-2 rule would allow it
|
||||
(0, False),
|
||||
],
|
||||
)
|
||||
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
|
||||
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(
|
||||
_make_config(group_size=group_size)
|
||||
)
|
||||
assert ok is expected_ok, reason
|
||||
if not expected_ok:
|
||||
assert reason is not None
|
||||
assert "must divide input size" in reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MoE DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NUM_EXPERTS = 4
|
||||
HIDDEN = 256
|
||||
INTERMEDIATE = 128
|
||||
|
||||
|
||||
def _make_moe_weights(group_size: int = 32):
|
||||
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
|
||||
torch.manual_seed(0)
|
||||
w13_q, w13_s, w2_q, w2_s = [], [], [], []
|
||||
for _ in range(NUM_EXPERTS):
|
||||
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
|
||||
q13, s13 = _quantize_per_group(w13, group_size)
|
||||
# CT is N-first and packs along the input dim: [N, K//8] per expert.
|
||||
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
|
||||
w13_s.append(s13)
|
||||
|
||||
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
|
||||
q2, s2 = _quantize_per_group(w2, group_size)
|
||||
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
|
||||
w2_s.append(s2)
|
||||
return (
|
||||
torch.stack(w13_q),
|
||||
torch.stack(w2_q),
|
||||
torch.stack(w13_s),
|
||||
torch.stack(w2_s),
|
||||
)
|
||||
|
||||
|
||||
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
|
||||
backends = int_wna16._get_priority_backends()
|
||||
assert backends[0] == WNA16MoEBackend.ZEN_CPU
|
||||
# The generic CPU backend stays available as a fallback.
|
||||
assert WNA16MoEBackend.CPU in backends
|
||||
|
||||
|
||||
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
|
||||
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
|
||||
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
|
||||
|
||||
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
|
||||
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
assert w13_out.dtype == w2_out.dtype == torch.int32
|
||||
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
|
||||
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
|
||||
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
|
||||
# Symmetric checkpoints carry no zero points into the kernel.
|
||||
assert converted[8] is None and converted[9] is None
|
||||
|
||||
|
||||
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
|
||||
"""The repacked weight must dequantize back to the checkpoint values."""
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
|
||||
expected = _import_unpack_from_int32()(
|
||||
w13,
|
||||
4,
|
||||
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
|
||||
packed_dim=1,
|
||||
)
|
||||
for expert in range(NUM_EXPERTS):
|
||||
torch.testing.assert_close(
|
||||
_unpack_s4(w13_out[expert], HIDDEN),
|
||||
expected[expert].float(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[(32, True), (128, True), (-1, False), (2, False)],
|
||||
)
|
||||
def test_zen_cpu_group_size_gating(group_size, expected_ok):
|
||||
quant_config = type("Args", (), {"group_size": group_size})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"may_have_zp,may_have_bias,expected_ok",
|
||||
[(True, False, False), (False, True, True), (True, True, False)],
|
||||
)
|
||||
def test_zen_cpu_rejects_zero_points_but_takes_bias(
|
||||
may_have_zp, may_have_bias, expected_ok
|
||||
):
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=may_have_zp,
|
||||
may_have_bias=may_have_bias,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
def test_zen_cpu_rejects_moe_wna16_layout():
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
quant_config = MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
has_zp=False,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=None,
|
||||
full_config={},
|
||||
)
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_disabled_by_env(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_backend_maps_to_experts_class():
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
|
||||
ZentorchExpertsInt4
|
||||
]
|
||||
|
||||
|
||||
def test_zen_experts_support_predicates():
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
|
||||
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
|
||||
for act in MoEActivation:
|
||||
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
|
||||
assert ZentorchExpertsInt4._supports_activation(act) == expected
|
||||
assert ZentorchExpertsInt4.requires_interleaved_w13
|
||||
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
|
||||
assert ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": False})()
|
||||
)
|
||||
assert not ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": True})()
|
||||
)
|
||||
|
||||
|
||||
def test_zen_experts_take_custom_routing():
|
||||
"""Models with their own router (gemma-4) reach select_experts through the
|
||||
config captured off the layer, since apply() cannot be handed the callable."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_routing_method(
|
||||
RoutingMethodType.Custom, None, None
|
||||
)
|
||||
|
||||
def routing_fn(**kwargs):
|
||||
raise AssertionError("not called")
|
||||
|
||||
layer = _StubMoELayer(
|
||||
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
|
||||
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
|
||||
MoEActivation.SILU,
|
||||
)
|
||||
layer.renormalize = True
|
||||
layer.custom_routing_function = routing_fn
|
||||
|
||||
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
|
||||
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
|
||||
|
||||
assert experts.custom_routing_function is routing_fn
|
||||
assert experts.renormalize is True
|
||||
|
||||
|
||||
def _swigluoai_perm(activation, experts_cls=None):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
swigluoai_w13_interleave_perm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
return swigluoai_w13_interleave_perm(
|
||||
experts_cls or ZentorchExpertsInt4,
|
||||
activation,
|
||||
2 * INTERMEDIATE,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def test_swigluoai_perm_only_for_zen_swigluoai():
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
|
||||
# Other activations keep the half-split layout the loader leaves.
|
||||
assert _swigluoai_perm(MoEActivation.SILU) is None
|
||||
# object stands in for any kernel that never set requires_interleaved_w13.
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
|
||||
|
||||
|
||||
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
|
||||
"""The permutation is a gather on w13's output-channel axis, which is dim 1
|
||||
for the N-first packed weights and their group scales."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
|
||||
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
|
||||
|
||||
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
permuted = int_wna16._process_weights_zen_cpu(
|
||||
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
|
||||
)[0]
|
||||
|
||||
for expert in range(NUM_EXPERTS):
|
||||
before = _unpack_s4(baseline[expert], HIDDEN)
|
||||
after = _unpack_s4(permuted[expert], HIDDEN)
|
||||
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
|
||||
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
|
||||
|
||||
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
|
||||
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])
|
||||
+2
-1
@@ -912,7 +912,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
|
||||
int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1"))
|
||||
),
|
||||
# (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ.
|
||||
# (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ, and
|
||||
# on Zen CPUs whether to serve int4 checkpoints as DA8W4 rather than W4A16.
|
||||
"VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))),
|
||||
# If the env var is set, Ray Compiled Graph uses the specified
|
||||
# channel type to communicate between workers belonging to
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
|
||||
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
|
||||
|
||||
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
|
||||
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
|
||||
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .cpu import CPUWNA16LinearKernel
|
||||
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
|
||||
from .MPLinearKernel import MPLinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
|
||||
|
||||
|
||||
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
|
||||
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
|
||||
``zentorch_woq_linear`` (W4A16)."""
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
ok, reason = super().can_implement(c)
|
||||
if not ok:
|
||||
return ok, reason
|
||||
|
||||
if not current_platform.is_zen_cpu():
|
||||
return False, "ZentorchWNA16 requires an AMD Zen CPU."
|
||||
|
||||
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
|
||||
# constraint the zentorch repack does not share.
|
||||
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
"CPUWNA16, supported types are: "
|
||||
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
|
||||
# -1 is per-channel, one group spanning K. Anything else has to divide
|
||||
# K, which is stricter than the parent's multiple-of-2 rule.
|
||||
if c.group_size != -1:
|
||||
in_features = c.partition_weight_shape[0]
|
||||
if c.group_size <= 0 or in_features % c.group_size != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) must divide input size "
|
||||
f"({in_features})",
|
||||
)
|
||||
|
||||
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
|
||||
return (
|
||||
False,
|
||||
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
num_groups = weight_scale.shape[1]
|
||||
return num_groups > 0 and in_features % num_groups == 0
|
||||
|
||||
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
|
||||
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
|
||||
|
||||
Returns False without mutating the layer when it is not eligible, so
|
||||
the caller falls through to the W4A16 paths.
|
||||
"""
|
||||
if not envs.VLLM_CPU_INT4_W4A8:
|
||||
return False
|
||||
|
||||
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
|
||||
return False
|
||||
|
||||
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
|
||||
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
|
||||
return False
|
||||
if self.config.act_type != torch.bfloat16:
|
||||
return False
|
||||
|
||||
if not self._zentorch_woq_eligible(layer):
|
||||
return False
|
||||
|
||||
weight_q = getattr(layer, self.w_q_name)
|
||||
weight_s = getattr(layer, self.w_s_name)
|
||||
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
|
||||
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
|
||||
|
||||
bits = self.config.weight_type.mantissa
|
||||
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
|
||||
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
|
||||
in_features = weight_packed.shape[1] * pack_factor
|
||||
|
||||
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
|
||||
# pack two s4 values per byte.
|
||||
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
|
||||
return False
|
||||
|
||||
if self.w_zp_name is not None:
|
||||
setattr(layer, self.w_zp_name, None)
|
||||
|
||||
unpack_from_int32 = _import_unpack_from_int32()
|
||||
|
||||
weight_unpacked = unpack_from_int32(
|
||||
weight_packed,
|
||||
bits,
|
||||
torch.Size([out_features, in_features]),
|
||||
packed_dim=weight_q.packed_dim,
|
||||
)
|
||||
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
|
||||
# same byte stream as s4 [N, K/2]; the kernel takes either view.
|
||||
layer._zentorch_da8w4_packed = (
|
||||
torch.ops.zentorch.zentorch_woq_repack_weight.default(
|
||||
weight_unpacked.to(torch.int8).contiguous()
|
||||
).view(torch.int8)
|
||||
)
|
||||
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
|
||||
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
|
||||
|
||||
for param_name in (self.w_q_name, self.w_s_name):
|
||||
param = getattr(layer, param_name, None)
|
||||
if param is not None and hasattr(param, "data"):
|
||||
param.data = torch.empty(0)
|
||||
|
||||
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
|
||||
layer._zentorch_da8w4 = True
|
||||
layer._zentorch_processed_weights = True
|
||||
logger.info_once(
|
||||
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
|
||||
"(weight_type=%s, group_size=%d)",
|
||||
self.config.weight_type,
|
||||
in_features // num_groups,
|
||||
)
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Repack CT GPTQ weights into the zentorch WOQ layout.
|
||||
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
|
||||
|
||||
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
|
||||
via ``super()`` when the layer doesn't satisfy
|
||||
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
if getattr(layer, "_zentorch_processed_weights", False):
|
||||
return
|
||||
|
||||
if self._maybe_process_da8w4_weights(layer):
|
||||
return
|
||||
|
||||
if not self._zentorch_woq_eligible(layer):
|
||||
logger.info_once(
|
||||
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
|
||||
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if getattr(layer, "_zentorch_da8w4", False):
|
||||
# The kernel reads bias through a raw pointer, so it must be
|
||||
# contiguous; the weight and scales already are.
|
||||
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
|
||||
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
|
||||
layer._zentorch_da8w4_packed,
|
||||
layer._zentorch_da8w4_scale,
|
||||
bias.contiguous() if bias is not None else None,
|
||||
)
|
||||
|
||||
if getattr(layer, "_zentorch_processed_weights", False):
|
||||
return torch.ops.zentorch.zentorch_woq_linear.default(
|
||||
x,
|
||||
|
||||
@@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# ===========================================================================
|
||||
# Weight layout
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def swigluoai_w13_interleave_perm(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
activation: MoEActivation,
|
||||
two_i: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
"""Index taking a half-split w13 ``[gate | up]`` to the interleaved order
|
||||
ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed.
|
||||
Callers apply it along whichever axis holds their output channels."""
|
||||
if not getattr(experts_cls, "requires_interleaved_w13", False):
|
||||
return None
|
||||
if activation != MoEActivation.SWIGLUOAI:
|
||||
return None
|
||||
|
||||
i = two_i // 2
|
||||
return torch.stack(
|
||||
[torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)],
|
||||
dim=1,
|
||||
).flatten()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Unquantized (BF16/FP16/FP32) MoE
|
||||
# ===========================================================================
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Zentorch fused MoE experts for AMD Zen CPUs."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
_moe_activation_to_str,
|
||||
has_zentorch_op,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
)
|
||||
|
||||
|
||||
class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"""DA8W4 (W4A8) group-quantized monolithic MoE experts."""
|
||||
|
||||
# swiglu_oai_mul reads gate/up interleaved, not in the half-split order the
|
||||
# weight loader leaves behind.
|
||||
requires_interleaved_w13 = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
# Defaults hold until process_weights_after_loading sees the layer.
|
||||
self.renormalize = moe_config.routing_method in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
)
|
||||
self.custom_routing_function: Callable | None = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Capture the router config that the monolithic apply() signature
|
||||
cannot carry; Custom routing has no other source for it."""
|
||||
self.renormalize = layer.renormalize
|
||||
self.custom_routing_function = layer.custom_routing_function
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"])
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
return False, "kernel requires bfloat16 activations"
|
||||
# The grouped GEMM needs tokens spread over at least two experts, which
|
||||
# top_k=1 cannot satisfy once decode drops to a single token.
|
||||
if moe_config.experts_per_token < 2:
|
||||
return False, "kernel requires experts_per_token >= 2"
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
# apply() passes select_experts' global ids straight to the kernel, so
|
||||
# every expert has to be resident on the rank.
|
||||
return not moe_parallel_config.use_ep
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) in [
|
||||
(kInt4Static, None),
|
||||
(kInt4Static32, None),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.Custom,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
"ZentorchExpertsInt4 does not support "
|
||||
"apply_router_weight_on_input=True."
|
||||
)
|
||||
|
||||
if (
|
||||
self.moe_config.routing_method == RoutingMethodType.Custom
|
||||
and self.custom_routing_function is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"ZentorchExpertsInt4 needs the model's custom_routing_function "
|
||||
"for RoutingMethodType.Custom."
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=self.custom_routing_function,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
output = torch.empty_like(hidden_states)
|
||||
torch.ops.zentorch.zentorch_fused_moe(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
self.w1_bias,
|
||||
self.w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # skip_weighted
|
||||
_moe_activation_to_str(activation),
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
__all__ = ["ZentorchExpertsInt4"]
|
||||
@@ -12,6 +12,7 @@ from compressed_tensors.quantization import (
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
@@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum):
|
||||
BATCHED_MARLIN = "BATCHED_MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
CPU = "CPU"
|
||||
ZEN_CPU = "ZEN_CPU"
|
||||
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
|
||||
TRITON = "TRITON"
|
||||
XPU = "XPU"
|
||||
@@ -98,6 +100,12 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [CPUExpertsInt4]
|
||||
elif backend == WNA16MoEBackend.ZEN_CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
return [ZentorchExpertsInt4]
|
||||
elif backend == WNA16MoEBackend.EMULATION:
|
||||
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
|
||||
Int4EmulationTritonExperts,
|
||||
@@ -117,7 +125,7 @@ def backend_to_kernel_cls(
|
||||
def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
"""Get available backends in priority order based on platform and config."""
|
||||
if current_platform.is_cpu():
|
||||
return [WNA16MoEBackend.CPU]
|
||||
return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU]
|
||||
if current_platform.is_xpu():
|
||||
return [WNA16MoEBackend.XPU]
|
||||
|
||||
@@ -160,6 +168,27 @@ def _backend_incompatibility_reason(
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
if backend == WNA16MoEBackend.ZEN_CPU:
|
||||
if not envs.VLLM_CPU_INT4_W4A8:
|
||||
return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path"
|
||||
if may_have_zp:
|
||||
return "zero points are not supported"
|
||||
if isinstance(quant_config, MoeWNA16Config):
|
||||
return "the MoeWNA16 weight layout is not supported"
|
||||
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
|
||||
return "GPTQ activation ordering is not supported"
|
||||
if (
|
||||
isinstance(quant_config, QuantizationArgs)
|
||||
and quant_config.actorder == "group"
|
||||
):
|
||||
return "group activation ordering is not supported"
|
||||
group_size = getattr(quant_config, "group_size", None)
|
||||
if group_size is None or group_size <= 0:
|
||||
return "DA8W4 requires group-quantized weights"
|
||||
# AOCL sym_quant requires the group size to be a multiple of 4.
|
||||
if group_size % 4 != 0:
|
||||
return f"group size {group_size} is not a multiple of 4"
|
||||
|
||||
if backend == WNA16MoEBackend.TRITON:
|
||||
if may_have_bias:
|
||||
return "expert bias is not supported"
|
||||
@@ -403,10 +432,14 @@ def make_wna16_moe_kernel(
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsWNA16,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
|
||||
# grouped/indexed experts, and Int4EmulationTritonExperts
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4,
|
||||
# ZentorchExpertsInt4, the Humming grouped/indexed experts, and
|
||||
# Int4EmulationTritonExperts
|
||||
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
|
||||
MarlinExperts,
|
||||
BatchedMarlinExperts,
|
||||
@@ -414,6 +447,7 @@ def make_wna16_moe_kernel(
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
XPUExpertsWNA16,
|
||||
CPUExpertsInt4,
|
||||
ZentorchExpertsInt4,
|
||||
Int4EmulationTritonExperts,
|
||||
Rdna3WNA16Experts,
|
||||
)
|
||||
@@ -1034,6 +1068,64 @@ def _process_weights_cpu(
|
||||
)
|
||||
|
||||
|
||||
def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
|
||||
"""Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
_import_unpack_from_int32,
|
||||
)
|
||||
|
||||
num_experts, out_features, _ = packed.shape
|
||||
# packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1.
|
||||
unpacked = _import_unpack_from_int32()(
|
||||
packed,
|
||||
4,
|
||||
torch.Size([num_experts, out_features, in_features]),
|
||||
packed_dim=1,
|
||||
)
|
||||
# CT is already N-first, which is what the repack consumes.
|
||||
repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default
|
||||
return torch.stack([repack_op(w) for w in unpacked])
|
||||
|
||||
|
||||
def _process_weights_zen_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, # w13_qweight
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
torch.Tensor | None, # w2_input_global_scale
|
||||
torch.Tensor | None, # w13_bias
|
||||
torch.Tensor | None, # w2_bias
|
||||
]:
|
||||
"""Zen CPU INT4 DA8W4 (W4A8) weight post-processing."""
|
||||
# CT WNA16 buffers are N-first, so the scales are [E, N, G].
|
||||
hidden_size = w2_scale.shape[1]
|
||||
intermediate_size = w13_scale.shape[1] // 2
|
||||
|
||||
# The kernel reads per-group scales as [E, G, N], so N-first needs a swap.
|
||||
return (
|
||||
_zen_repack_s4(w13.data, hidden_size),
|
||||
_zen_repack_s4(w2.data, intermediate_size),
|
||||
w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
|
||||
w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
|
||||
None, # w13_qzeros (symmetric)
|
||||
None, # w2_qzeros (symmetric)
|
||||
None, # w13_input_global_scale
|
||||
None, # w2_input_global_scale
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
|
||||
def _process_weights_xpu(
|
||||
layer: torch.nn.Module,
|
||||
quant_config: QuantizationConfig,
|
||||
@@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif backend == WNA16MoEBackend.ZEN_CPU:
|
||||
return _process_weights_zen_cpu(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
|
||||
return _process_weights_flashinfer(
|
||||
w13,
|
||||
|
||||
+66
-2
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
# Per-expert biases, as gpt-oss carries.
|
||||
if self.moe.has_bias:
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
|
||||
# Monolithic experts route inside apply(), so like the unquantized
|
||||
# CPU path they need the router config that only the layer carries.
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
swigluoai_w13_interleave_perm,
|
||||
)
|
||||
|
||||
# Reorder before the conversion below, which repacks w13 for the kernel.
|
||||
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
|
||||
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
|
||||
perm = swigluoai_w13_interleave_perm(
|
||||
self.experts_cls,
|
||||
self.moe.activation,
|
||||
layer.w13_weight_packed.size(1),
|
||||
layer.w13_weight_packed.device,
|
||||
)
|
||||
if perm is not None:
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w13_weight_packed",
|
||||
layer.w13_weight_packed.data[:, perm].contiguous(),
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w13_weight_scale",
|
||||
layer.w13_weight_scale.data[:, perm].contiguous(),
|
||||
)
|
||||
if self.moe.has_bias:
|
||||
replace_parameter(
|
||||
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
|
||||
)
|
||||
|
||||
# Process weights using the shared oracle infrastructure
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
w2_input_global_scale,
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = converted
|
||||
|
||||
# Replace common parameters
|
||||
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
if w13_bias is not None:
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
if w2_bias is not None:
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant.
|
||||
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
|
||||
if (
|
||||
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
|
||||
@@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
|
||||
"w2_bias": "gpt_oss_w2",
|
||||
"w13_weight_scale": "gpt_oss_w13",
|
||||
"w2_weight_scale": "gpt_oss_w2",
|
||||
"w13_weight_packed": "gpt_oss_w13",
|
||||
"w2_weight_packed": "gpt_oss_w2",
|
||||
}
|
||||
|
||||
|
||||
@@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts):
|
||||
) -> None:
|
||||
tp_rank = self.moe_config.moe_parallel_config.tp_rank
|
||||
is_w13 = shard_id == "gpt_oss_w13"
|
||||
is_weight = weight_name.endswith("_weight")
|
||||
is_weight = weight_name.endswith(("_weight", "_weight_packed"))
|
||||
is_partitioned_scale = weight_name.endswith("_weight_scale")
|
||||
|
||||
if is_weight:
|
||||
@@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts):
|
||||
) or quant_method_name in (
|
||||
"CompressedTensorsW4A4Nvfp4MoEMethod",
|
||||
"Nvfp4OnlineMoEMethod",
|
||||
"CompressedTensorsWNA16MoEMethod",
|
||||
):
|
||||
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user